def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.version = GLANCE_VERSION
if self.version == 2:
self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username']
else:
self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username']
def __init__(self):
super(Nova, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.nib = None
self._cloud_plugin_content = []
def decorated_function(*args, **kwargs):
if(ApplicationConfiguration().configuration['debug']):
log.debug('Handling %s HTTP %s REQUEST for resource at %s: %s' % (request.headers.get('Content-Type'),
request.method,
request.path,
request.body.read()))
return f(*args, **kwargs)
def __init__(self):
super(IndirectionCloud, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.pim = PersistentImageManager.default_manager()
self.res_mgr = ReservationManager()
def setUp(self):
logging.basicConfig(
level=logging.NOTSET,
format=
'%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s',
filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(
ApplicationConfiguration().configuration["warehouse"])
self.template_xml = "<template>This is a test template. There is not much to it.</template>"
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger("%s.%s" % (__name__, self.__class__.__name__))
self.version = GLANCE_VERSION
if self.version == 2:
self.credentials_attrs = ["auth_url", "password", "tenant", "username"]
else:
self.credentials_attrs = ["auth_url", "password", "strategy", "tenant", "username"]
def __init__(self):
super(TinMan, self).__init__()
self.cloud_plugin_content = []
config_obj = ApplicationConfiguration()
self.app_config = config_obj.configuration
self.res_mgr = ReservationManager()
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.parameters = None
self.install_script_object = None
self.guest = None
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.version = GLANCE_VERSION
if self.version == 2:
self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username']
elif self.version == 1:
self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username']
def decorated_function(*args, **kwargs):
if (ApplicationConfiguration().configuration['debug']):
request_body = request.body.read()
if ('credentials' in request_body):
marker = 'provider_credentials'
starting_index = request_body.find(marker)
ending_index = request_body.rfind(marker) + len(marker)
sensitive = request_body[starting_index:ending_index]
request_body = request_body.replace(sensitive, 'REDACTED')
log.debug('Handling %s HTTP %s REQUEST for resource at %s: %s' %
(request.headers.get('Content-Type'), request.method,
request.path, request_body))
return f(*args, **kwargs)
def __init__(self, url=None):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
appconfig = ApplicationConfiguration().configuration
self.http = httplib2.Http()
if(not url):
url = appconfig['warehouse']
self.log.debug("Property (url) not specified. Pulling from application configuration: %s" % (url, ))
self.image_bucket = appconfig['image_bucket']
self.build_bucket = appconfig['build_bucket']
self.target_image_bucket = appconfig['target_bucket']
self.template_bucket = appconfig['template_bucket']
self.icicle_bucket = appconfig['icicle_bucket']
self.provider_image_bucket = appconfig['provider_bucket']
self.warehouse_credentials = {'key':appconfig.get('warehouse_key'), 'secret':appconfig.get('warehouse_secret')}
self.warehouse_oauth = True if(self.warehouse_credentials['key'] and self.warehouse_credentials['secret']) else False
self.url = url.rstrip('/')
self.log.debug("Created Image Warehouse instance %s" % (self, ))
def __init__(self):
super(FedoraOS, self).__init__()
self.cloud_plugin_content = []
config_obj = ApplicationConfiguration()
self.app_config = config_obj.configuration
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
if "ec2" in config_obj.jeos_images:
self.ec2_jeos_amis = config_obj.jeos_images['ec2']
else:
self.log.warning(
"No JEOS amis defined for ec2. Snapshot builds will not be possible."
)
self.ec2_jeos_amis = {}
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(Rackspace, self).__init__()
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
config_obj = ApplicationConfiguration()
self.app_config = config_obj.configuration
self.oz_config = ConfigParser.SafeConfigParser()
self.oz_config.read("/etc/oz/oz.cfg")
self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"])
self.active_image = None
if "rackspace" in config_obj.jeos_images:
self.rackspace_jeos_amis = config_obj.jeos_images['rackspace']
else:
self.log.warning("No JEOS images defined for Rackspace. Snapshot builds will not be possible.")
self.rackspace_jeos_amis = {}
def __init__(self, *args, **kwargs):
config = json.loads(open('/etc/imagefactory/imagefactory.conf').read())
config['plugins'] = '/etc/imagefactory/plugins.d'
config['timeout'] = 3600
ApplicationConfiguration(configuration=config)
plugin_mgr = PluginManager('/etc/imagefactory/plugins.d')
plugin_mgr.load()
logfile = os.path.join(kwargs['workdir'], 'imgfac.log')
self.fhandler = logging.FileHandler(logfile)
self.tlog = logging.getLogger()
self.tlog.setLevel(logging.DEBUG)
self.tlog.addHandler(self.fhandler)
pass
def __init__(self, *args, **kwargs):
config = json.loads(open('/etc/imagefactory/imagefactory.conf').read())
config['plugins'] = '/etc/imagefactory/plugins.d'
config['timeout'] = 3600
config['rhevm_image_format'] = 'qcow2'
ApplicationConfiguration(configuration=config)
plugin_mgr = PluginManager('/etc/imagefactory/plugins.d')
plugin_mgr.load()
self.fhandler = logging.StreamHandler(sys.stdout)
self.tlog = logging.getLogger()
self.tlog.setLevel(logging.DEBUG)
self.tlog.addHandler(self.fhandler)
verbosemode = kwargs.get('verbosemode', False)
if verbosemode:
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
self.tlog.addHandler(ch)
pass
def __new__(cls, *p, **k):
if cls.instance is None:
i = super(ReservationManager, cls).__new__(cls, *p, **k)
# initialize here, not in __init__()
i.log = logging.getLogger('%s.%s' %
(__name__, i.__class__.__name__))
i.default_minimum = cls.DEFAULT_MINIMUM
i._mounts = dict()
i.appconfig = ApplicationConfiguration().configuration
i._queues = dict(local=BoundedSemaphore(
i.appconfig.get('max_concurrent_local_sessions', 1)),
ec2=BoundedSemaphore(
i.appconfig.get('max_concurrent_ec2_sessions',
1)))
i._named_locks = {}
i._named_locks_lock = BoundedSemaphore()
# Initialize based on PID to prevent conflicts from multiple CLI runs on the same machine
# TODO: This is TinMan/Oz specific - move it to the plugin
i._listen_port = cls.MIN_PORT + (os.getpid() %
(cls.MAX_PORT - cls.MIN_PORT))
i._listen_port_lock = BoundedSemaphore()
cls.instance = i
return cls.instance
class OpenStack(object):
zope.interface.implements(CloudDelegate)
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.version = GLANCE_VERSION
if self.version == 2:
self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username']
self.openrc_credentials_attrs = { 'OS_AUTH_URL': 'auth_url',
'OS_TENANT_NAME': 'tenant',
'OS_USERNAME': '******'}
else:
self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username']
def activity(self, activity):
# Simple helper function
# Activity should be a one line human-readable string indicating the task in progress
# We log it at DEBUG and also set it as the status_detail on our active image
self.log.debug(activity)
self.active_image.status_detail['activity'] = activity
def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters):
# Our target_image is already a raw KVM image. All we need to do is upload to glance
self.builder = builder
self.active_image = self.builder.provider_image
self.openstack_decode_credentials(credentials)
provider_data = self.get_dynamic_provider_data(provider)
if provider_data is None:
raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided")
# Image is always here and it is the target_image datafile
input_image = self.builder.target_image.data
# If the template species a name, use that, otherwise create a name
# using provider_image.identifier.
template = Template(self.builder.provider_image.template)
if template.name:
image_name = template.name
else:
image_name = 'ImageFactory created image - %s' % (self.builder.provider_image.identifier)
if self.check_qcow_size(input_image):
self.log.debug("Uploading image to glance, detected qcow format")
disk_format='qcow2'
else:
self.log.debug("Uploading image to glance, assuming raw format")
disk_format='raw'
# Support openstack grizzly keystone authentication and glance upload
if self.version == 2:
if self.credentials_token is None:
self.credentials_token = self.keystone_authenticate(**self.credentials_dict)
provider_data['name'] = image_name
provider_data['disk_format'] = disk_format
image_id = self.glance_upload_v2(input_image, self.credentials_token, **provider_data)
else:
# Also support backward compatible for folsom
image_id = self.glance_upload(input_image, creds = self.credentials_dict, token = self.credentials_token,
host=provider_data['glance-host'], port=provider_data['glance-port'],
name=image_name, disk_format=disk_format)
self.builder.provider_image.identifier_on_provider = image_id
if 'username' in self.credentials_dict:
self.builder.provider_image.provider_account_identifier = self.credentials_dict['username']
self.percent_complete=100
def openstack_decode_credentials(self, credentials):
self.activity("Preparing OpenStack credentials")
# TODO: Validate these - in particular, ensure that if some nodes are missing at least
# a minimal acceptable set of auth is present
try:
doc = libxml2.parseDoc(credentials)
except libxml2.parserError:
self._populate_credentials_dict_from_openrc(credentials)
return
self.credentials_dict = { }
for authprop in self.credentials_attrs:
value = self._get_xml_node(doc, authprop)
if value is not None:
self.credentials_dict[authprop] = value
self.credentials_token = self._get_xml_node(doc, 'token')
def _get_xml_node(self, doc, credtype):
nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype))
# OpenStack supports multiple auth schemes so not all nodes are required
if len(nodes) < 1:
return None
return nodes[0].content
def _populate_credentials_dict_from_openrc(self, credentials_str):
credentials_list = credentials_str.split('\n')
self.credentials_dict = {}
for authprop in self.openrc_credentials_attrs.keys():
corresponding_authprop = self.openrc_credentials_attrs[authprop]
matching = [line for line in credentials_list if authprop in line]
if matching == []:
raise ImageFactoryException("Credentials file does not have required field - " + authprop)
self.credentials_dict[corresponding_authprop] = matching[0][matching[0].index('=') + 1 : ].strip('"')
self.credentials_dict['password'] = getpass.getpass(prompt='Please enter your OpenStack Password:\n')
self.credentials_token = None
def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters):
# TODO: Implement snapshot builds
raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM")
def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
return True
def builder_will_create_target_image(self, builder, target, image_id, template, parameters):
pass
def builder_did_create_target_image(self, builder, target, image_id, template, parameters):
self.target=target
self.builder=builder
self.modify_oz_filesystem()
# OS plugin has already provided the initial file for us to work with
# which we can currently assume is a raw image
input_image = builder.target_image.data
# Support conversion to alternate preferred image format
# Currently only handle qcow2, but the size reduction of
# using this avoids the performance penalty of uploading
# (and launching) raw disk images on slow storage
if self.app_config.get('openstack_image_format', 'raw') == 'qcow2':
# Prevent double convert if the image is already qcow2
if self.check_qcow_size(input_image) is not None:
self.log.debug("No conversion require. Image already in qcow2 format.")
return
self.log.debug("Converting RAW image to compressed qcow2 format")
rc = os.system("qemu-img convert -c -O qcow2 %s %s" %
(input_image, input_image + ".tmp.qcow2"))
if rc == 0:
os.unlink(input_image)
os.rename(input_image + ".tmp.qcow2", input_image)
else:
raise ImageFactoryException("qemu-img convert failed!")
def modify_oz_filesystem(self):
self.log.debug("Doing further Factory specific modification of Oz image")
guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data)
remove_net_persist(guestfs_handle)
create_cloud_info(guestfs_handle, self.target)
shutdown_and_close(guestfs_handle)
def get_dynamic_provider_data(self, provider):
try:
xml_et = fromstring(provider)
return xml_et.attrib
except Exception as e:
self.log.debug('Testing provider for XML: %s' % e)
pass
try:
jload = json.loads(provider)
return jload
except ValueError as e:
self.log.debug('Testing provider for JSON: %s' % e)
pass
return None
def keystone_authenticate(self, **kwargs):
user = kwargs.get('username')
pwd = kwargs.get('password')
tenant = kwargs.get('tenant')
url = kwargs.get('auth_url', 'http://127.0.0.1:5000/v2.0')
keystone = client.Client(username=user, password=pwd, tenant_name=tenant, auth_url=url)
keystone.authenticate()
return keystone.auth_token
def glance_upload(self, image_filename, creds = {'auth_url': None, 'password': None, 'strategy': 'noauth', 'tenant': None, 'username': None},
host = "0.0.0.0", port = "9292", token = None, name = 'Factory Test Image', disk_format = 'raw'):
image_meta = {'container_format': 'bare',
'disk_format': disk_format,
'is_public': True,
'min_disk': 0,
'min_ram': 0,
'name': name,
'properties': {'distro': 'rhel'}}
c = glance_client.Client(host=host, port=port,
auth_tok=token, creds=creds)
image_data = open(image_filename, "r")
image_meta = c.add_image(image_meta, image_data)
image_data.close()
return image_meta['id']
def glance_upload_v2(self, image, auth_token=None, **kwargs):
if image is None:
raise ImageFactoryException("No image is provided")
glance_host = kwargs.setdefault("glance-host", "127.0.0.1")
glance_port = kwargs.setdefault("glance-port", "9292")
glance_url = "http://%s:%s" % (glance_host, glance_port)
image_data = open(image, "r")
image_meta = {
'container_format': kwargs.setdefault('container_format', 'bare'),
'disk_format': kwargs.setdefault('disk_format', 'raw'),
'is_public': kwargs.setdefault('is_public', False),
'min_disk': kwargs.setdefault('min_disk', 0),
'min_ram': kwargs.setdefault('min_ram', 0),
'name': kwargs.setdefault('name', 'Factory Test Image'),
'data': image_data,
'properties': kwargs.setdefault('properties', '{}')
}
c = glance_client.Client('1', glance_url, token=auth_token)
image_meta = c.images.create(**image_meta)
image_data.close()
return image_meta.id
# FIXME : cut/paste from RHEVMHelper.py, should refactor into a common utility class
def check_qcow_size(self, filename):
# Detect if an image is in qcow format
# If it is, return the size of the underlying disk image
# If it isn't, return None
# For interested parties, this is the QCOW header struct in C
# struct qcow_header {
# uint32_t magic;
# uint32_t version;
# uint64_t backing_file_offset;
# uint32_t backing_file_size;
# uint32_t cluster_bits;
# uint64_t size; /* in bytes */
# uint32_t crypt_method;
# uint32_t l1_size;
# uint64_t l1_table_offset;
# uint64_t refcount_table_offset;
# uint32_t refcount_table_clusters;
# uint32_t nb_snapshots;
# uint64_t snapshots_offset;
# };
# And in Python struct format string-ese
qcow_struct=">IIQIIQIIQQIIQ" # > means big-endian
qcow_magic = 0x514649FB # 'Q' 'F' 'I' 0xFB
f = open(filename,"r")
pack = f.read(struct.calcsize(qcow_struct))
f.close()
unpack = struct.unpack(qcow_struct, pack)
if unpack[0] == qcow_magic:
return unpack[5]
else:
return None
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
class OpenStack(object):
zope.interface.implements(CloudDelegate)
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
def activity(self, activity):
# Simple helper function
# Activity should be a one line human-readable string indicating the task in progress
# We log it at DEBUG and also set it as the status_detail on our active image
self.log.debug(activity)
self.active_image.status_detail['activity'] = activity
def push_image_to_provider(self, builder, provider, credentials, target,
target_image, parameters):
# Our target_image is already a raw KVM image. All we need to do is upload to glance
self.builder = builder
self.active_image = self.builder.provider_image
self.openstack_decode_credentials(credentials)
provider_data = self.get_dynamic_provider_data(provider)
if provider_data is None:
raise ImageFactoryException(
"OpenStack KVM instance not found in XML or JSON provided")
# Image is always here and it is the target_image datafile
input_image = self.builder.target_image.data
# If the template species a name, use that, otherwise create a name
# using provider_image.identifier.
template = Template(self.builder.provider_image.template)
if template.name:
image_name = template.name
else:
image_name = 'ImageFactory created image - %s' % (
self.builder.provider_image.identifier)
if self.check_qcow_size(input_image):
self.log.debug("Uploading image to glance, detected qcow format")
disk_format = 'qcow2'
else:
self.log.debug("Uploading image to glance, assuming raw format")
disk_format = 'raw'
image_id = glance_upload(input_image,
creds=self.credentials_dict,
token=self.credentials_token,
host=provider_data['glance-host'],
port=provider_data['glance-port'],
name=image_name,
disk_format=disk_format)
self.builder.provider_image.identifier_on_provider = image_id
if 'username' in self.credentials_dict:
self.builder.provider_image.provider_account_identifier = self.credentials_dict[
'username']
self.percent_complete = 100
def openstack_decode_credentials(self, credentials):
self.activity("Preparing OpenStack credentials")
# TODO: Validate these - in particular, ensure that if some nodes are missing at least
# a minimal acceptable set of auth is present
doc = libxml2.parseDoc(credentials)
self.credentials_dict = {}
for authprop in [
'auth_url', 'password', 'strategy', 'tenant', 'username'
]:
self.credentials_dict[authprop] = self._get_xml_node(doc, authprop)
self.credentials_token = self._get_xml_node(doc, 'token')
def _get_xml_node(self, doc, credtype):
nodes = doc.xpathEval(
"//provider_credentials/openstack_credentials/%s" % (credtype))
# OpenStack supports multiple auth schemes so not all nodes are required
if len(nodes) < 1:
return None
return nodes[0].content
def snapshot_image_on_provider(self, builder, provider, credentials,
template, parameters):
# TODO: Implement snapshot builds
raise ImageFactoryException(
"Snapshot builds not currently supported on OpenStack KVM")
def builder_should_create_target_image(self, builder, target, image_id,
template, parameters):
return True
def builder_will_create_target_image(self, builder, target, image_id,
template, parameters):
pass
def builder_did_create_target_image(self, builder, target, image_id,
template, parameters):
self.target = target
self.builder = builder
self.modify_oz_filesystem()
# OS plugin has already provided the initial file for us to work with
# which we can currently assume is a raw image
input_image = builder.target_image.data
# Support conversion to alternate preferred image format
# Currently only handle qcow2, but the size reduction of
# using this avoids the performance penalty of uploading
# (and launching) raw disk images on slow storage
if self.app_config.get('openstack_image_format', 'raw') == 'qcow2':
self.log.debug("Converting RAW image to compressed qcow2 format")
rc = os.system("qemu-img convert -c -O qcow2 %s %s" %
(input_image, input_image + ".tmp.qcow2"))
if rc == 0:
os.unlink(input_image)
os.rename(input_image + ".tmp.qcow2", input_image)
else:
raise ImageFactoryException("qemu-img convert failed!")
def modify_oz_filesystem(self):
self.log.debug(
"Doing further Factory specific modification of Oz image")
guestfs_handle = launch_inspect_and_mount(
self.builder.target_image.data)
remove_net_persist(guestfs_handle)
create_cloud_info(guestfs_handle, self.target)
shutdown_and_close(guestfs_handle)
def get_dynamic_provider_data(self, provider):
try:
xml_et = fromstring(provider)
return xml_et.attrib
except Exception as e:
self.log.debug('Testing provider for XML: %s' % e)
pass
try:
jload = json.loads(provider)
return jload
except ValueError as e:
self.log.debug('Testing provider for JSON: %s' % e)
pass
return None
# FIXME : cut/paste from RHEVMHelper.py, should refactor into a common utility class
def check_qcow_size(self, filename):
# Detect if an image is in qcow format
# If it is, return the size of the underlying disk image
# If it isn't, return None
# For interested parties, this is the QCOW header struct in C
# struct qcow_header {
# uint32_t magic;·
# uint32_t version;
# uint64_t backing_file_offset;
# uint32_t backing_file_size;
# uint32_t cluster_bits;
# uint64_t size; /* in bytes */
# uint32_t crypt_method;
# uint32_t l1_size;
# uint64_t l1_table_offset;
# uint64_t refcount_table_offset;
# uint32_t refcount_table_clusters;
# uint32_t nb_snapshots;
# uint64_t snapshots_offset;
# };
# And in Python struct format string-ese
qcow_struct = ">IIQIIQIIQQIIQ" # > means big-endian
qcow_magic = 0x514649FB # 'Q' 'F' 'I' 0xFB
f = open(filename, "r")
pack = f.read(struct.calcsize(qcow_struct))
f.close()
unpack = struct.unpack(qcow_struct, pack)
if unpack[0] == qcow_magic:
return unpack[5]
else:
return None
def __init__(self, key):
consumers = ApplicationConfiguration().configuration['clients']
self.key = key
self.secret = consumers.get(key) if consumers else None
class OpenStack(object):
zope.interface.implements(CloudDelegate)
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger("%s.%s" % (__name__, self.__class__.__name__))
self.version = GLANCE_VERSION
if self.version == 2:
self.credentials_attrs = ["auth_url", "password", "tenant", "username"]
else:
self.credentials_attrs = ["auth_url", "password", "strategy", "tenant", "username"]
def activity(self, activity):
# Simple helper function
# Activity should be a one line human-readable string indicating the task in progress
# We log it at DEBUG and also set it as the status_detail on our active image
self.log.debug(activity)
self.active_image.status_detail["activity"] = activity
def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters):
# Our target_image is already a raw KVM image. All we need to do is upload to glance
self.builder = builder
self.active_image = self.builder.provider_image
self.openstack_decode_credentials(credentials)
provider_data = self.get_dynamic_provider_data(provider)
if provider_data is None:
raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided")
# Image is always here and it is the target_image datafile
input_image = self.builder.target_image.data
# If the template species a name, use that, otherwise create a name
# using provider_image.identifier.
template = Template(self.builder.provider_image.template)
if template.name:
image_name = template.name
else:
image_name = "ImageFactory created image - %s" % (self.builder.provider_image.identifier)
if check_qcow_size(input_image):
self.log.debug("Uploading image to glance, detected qcow format")
disk_format = "qcow2"
else:
self.log.debug("Uploading image to glance, assuming raw format")
disk_format = "raw"
# Support openstack grizzly keystone authentication and glance upload
if self.version == 2:
if self.credentials_token is None:
self.credentials_token = self.keystone_authenticate(**self.credentials_dict)
provider_data["name"] = image_name
provider_data["disk_format"] = disk_format
image_id = self.glance_upload_v2(input_image, self.credentials_token, **provider_data)
else:
# Also support backward compatible for folsom
image_id = self.glance_upload(
input_image,
creds=self.credentials_dict,
token=self.credentials_token,
host=provider_data["glance-host"],
port=provider_data["glance-port"],
name=image_name,
disk_format=disk_format,
)
self.builder.provider_image.identifier_on_provider = image_id
if "username" in self.credentials_dict:
self.builder.provider_image.provider_account_identifier = self.credentials_dict["username"]
self.percent_complete = 100
def openstack_decode_credentials(self, credentials):
self.activity("Preparing OpenStack credentials")
# TODO: Validate these - in particular, ensure that if some nodes are missing at least
# a minimal acceptable set of auth is present
doc = libxml2.parseDoc(credentials)
self.credentials_dict = {}
for authprop in self.credentials_attrs:
value = self._get_xml_node(doc, authprop)
if value is not None:
self.credentials_dict[authprop] = value
self.credentials_token = self._get_xml_node(doc, "token")
def _get_xml_node(self, doc, credtype):
nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype))
# OpenStack supports multiple auth schemes so not all nodes are required
if len(nodes) < 1:
return None
return nodes[0].content
def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters):
# TODO: Implement snapshot builds
raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM")
def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
return True
def builder_will_create_target_image(self, builder, target, image_id, template, parameters):
pass
def builder_did_create_target_image(self, builder, target, image_id, template, parameters):
self.target = target
self.builder = builder
self.modify_oz_filesystem()
# OS plugin has already provided the initial file for us to work with
# which we can currently assume is a raw image
input_image = builder.target_image.data
# Support conversion to alternate preferred image format
# Currently only handle qcow2, but the size reduction of
# using this avoids the performance penalty of uploading
# (and launching) raw disk images on slow storage
if self.app_config.get("openstack_image_format", "raw") == "qcow2":
# None of the existing input base_image plugins produce compressed qcow2 output
# the step below is either going from raw to compressed qcow2 or
# uncompressed qcow2 to compressed qcow2
self.log.debug("Converting image to compressed qcow2 format")
tmp_output = input_image + ".tmp.qcow2"
convert_cmd = qemu_convert_cmd(input_image, tmp_output, True)
(stdout, stderr, retcode) = subprocess_check_output(convert_cmd)
os.unlink(input_image)
os.rename(tmp_output, input_image)
def modify_oz_filesystem(self):
self.log.debug("Doing further Factory specific modification of Oz image")
guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data)
remove_net_persist(guestfs_handle)
create_cloud_info(guestfs_handle, self.target)
shutdown_and_close(guestfs_handle)
def get_dynamic_provider_data(self, provider):
try:
xml_et = fromstring(provider)
return xml_et.attrib
except Exception as e:
self.log.debug("Testing provider for XML: %s" % e)
pass
try:
jload = json.loads(provider)
return jload
except ValueError as e:
self.log.debug("Testing provider for JSON: %s" % e)
pass
return None
def keystone_authenticate(self, **kwargs):
user = kwargs.get("username")
pwd = kwargs.get("password")
tenant = kwargs.get("tenant")
url = kwargs.get("auth_url", "http://127.0.0.1:5000/v2.0")
keystone = client.Client(username=user, password=pwd, tenant_name=tenant, auth_url=url)
keystone.authenticate()
return keystone.auth_token
def glance_upload(
self,
image_filename,
creds={"auth_url": None, "password": None, "strategy": "noauth", "tenant": None, "username": None},
host="0.0.0.0",
port="9292",
token=None,
name="Factory Test Image",
disk_format="raw",
):
image_meta = {
"container_format": "bare",
"disk_format": disk_format,
"is_public": True,
"min_disk": 0,
"min_ram": 0,
"name": name,
"properties": {"distro": "rhel"},
}
c = glance_client.Client(host=host, port=port, auth_tok=token, creds=creds)
image_data = open(image_filename, "r")
image_meta = c.add_image(image_meta, image_data)
image_data.close()
return image_meta["id"]
def glance_upload_v2(self, image, auth_token=None, **kwargs):
if image is None:
raise ImageFactoryException("No image is provided")
glance_host = kwargs.setdefault("glance-host", "127.0.0.1")
glance_port = kwargs.setdefault("glance-port", "9292")
glance_url = "http://%s:%s" % (glance_host, glance_port)
image_data = open(image, "r")
image_meta = {
"container_format": kwargs.setdefault("container_format", "bare"),
"disk_format": kwargs.setdefault("disk_format", "raw"),
"is_public": kwargs.setdefault("is_public", False),
"min_disk": kwargs.setdefault("min_disk", 0),
"min_ram": kwargs.setdefault("min_ram", 0),
"name": kwargs.setdefault("name", "Factory Test Image"),
"data": image_data,
}
c = glance_client.Client("1", glance_url, token=auth_token)
image_meta = c.images.create(**image_meta)
image_data.close()
return image_meta.id
def testSingleton(self):
self.assertTrue(
id(ApplicationConfiguration()) == id(ApplicationConfiguration()))
class OpenStack(object):
zope.interface.implements(CloudDelegate)
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStack, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.version = GLANCE_VERSION
if self.version == 2:
self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username']
else:
self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username']
def activity(self, activity):
# Simple helper function
# Activity should be a one line human-readable string indicating the task in progress
# We log it at DEBUG and also set it as the status_detail on our active image
self.log.debug(activity)
self.active_image.status_detail['activity'] = activity
def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters):
# Our target_image is already a raw KVM image. All we need to do is upload to glance
self.builder = builder
self.active_image = self.builder.provider_image
self.openstack_decode_credentials(credentials)
provider_data = self.get_dynamic_provider_data(provider)
if provider_data is None:
raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided")
# Image is always here and it is the target_image datafile
input_image = self.builder.target_image.data
# If the template species a name, use that, otherwise create a name
# using provider_image.identifier.
template = Template(self.builder.provider_image.template)
if template.name:
image_name = template.name
else:
image_name = 'ImageFactory created image - %s' % (self.builder.provider_image.identifier)
if check_qcow_size(input_image):
self.log.debug("Uploading image to glance, detected qcow format")
disk_format='qcow2'
else:
self.log.debug("Uploading image to glance, assuming raw format")
disk_format='raw'
# Support openstack grizzly keystone authentication and glance upload
if self.version == 2:
if self.credentials_token is None:
self.credentials_token = self.keystone_authenticate(**self.credentials_dict)
provider_data['name'] = image_name
provider_data['disk_format'] = disk_format
image_id = self.glance_upload_v2(input_image, self.credentials_token, **provider_data)
else:
# Also support backward compatible for folsom
image_id = self.glance_upload(input_image, creds = self.credentials_dict, token = self.credentials_token,
host=provider_data['glance-host'], port=provider_data['glance-port'],
name=image_name, disk_format=disk_format)
self.builder.provider_image.identifier_on_provider = image_id
if 'username' in self.credentials_dict:
self.builder.provider_image.provider_account_identifier = self.credentials_dict['username']
self.percent_complete=100
def openstack_decode_credentials(self, credentials):
self.activity("Preparing OpenStack credentials")
# TODO: Validate these - in particular, ensure that if some nodes are missing at least
# a minimal acceptable set of auth is present
doc = libxml2.parseDoc(credentials)
self.credentials_dict = { }
for authprop in self.credentials_attrs:
value = self._get_xml_node(doc, authprop)
if value is not None:
self.credentials_dict[authprop] = value
self.credentials_token = self._get_xml_node(doc, 'token')
def _get_xml_node(self, doc, credtype):
nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype))
# OpenStack supports multiple auth schemes so not all nodes are required
if len(nodes) < 1:
return None
return nodes[0].content
def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters):
# TODO: Implement snapshot builds
raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM")
def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
return True
def builder_will_create_target_image(self, builder, target, image_id, template, parameters):
pass
def builder_did_create_target_image(self, builder, target, image_id, template, parameters):
self.target=target
self.builder=builder
self.modify_oz_filesystem()
# OS plugin has already provided the initial file for us to work with
# which we can currently assume is a raw image
input_image = builder.target_image.data
# Support conversion to alternate preferred image format
# Currently only handle qcow2, but the size reduction of
# using this avoids the performance penalty of uploading
# (and launching) raw disk images on slow storage
if self.app_config.get('openstack_image_format', 'raw') == 'qcow2':
# None of the existing input base_image plugins produce compressed qcow2 output
# the step below is either going from raw to compressed qcow2 or
# uncompressed qcow2 to compressed qcow2
self.log.debug("Converting image to compressed qcow2 format")
tmp_output = input_image + ".tmp.qcow2"
convert_cmd = qemu_convert_cmd(input_image, tmp_output, True)
(stdout, stderr, retcode) = subprocess_check_output(convert_cmd)
os.unlink(input_image)
os.rename(tmp_output, input_image)
def modify_oz_filesystem(self):
self.log.debug("Doing further Factory specific modification of Oz image")
guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data)
remove_net_persist(guestfs_handle)
create_cloud_info(guestfs_handle, self.target)
shutdown_and_close(guestfs_handle)
def get_dynamic_provider_data(self, provider):
try:
xml_et = fromstring(provider)
return xml_et.attrib
except Exception as e:
self.log.debug('Testing provider for XML: %s' % e)
pass
try:
jload = json.loads(provider)
return jload
except ValueError as e:
self.log.debug('Testing provider for JSON: %s' % e)
pass
return None
def keystone_authenticate(self, **kwargs):
user = kwargs.get('username')
pwd = kwargs.get('password')
tenant = kwargs.get('tenant')
url = kwargs.get('auth_url', 'http://127.0.0.1:5000/v2.0')
keystone = client.Client(username=user, password=pwd, tenant_name=tenant, auth_url=url)
keystone.authenticate()
return keystone.auth_token
def glance_upload(self, image_filename, creds = {'auth_url': None, 'password': None, 'strategy': 'noauth', 'tenant': None, 'username': None},
host = "0.0.0.0", port = "9292", token = None, name = 'Factory Test Image', disk_format = 'raw'):
image_meta = {'container_format': 'bare',
'disk_format': disk_format,
'is_public': True,
'min_disk': 0,
'min_ram': 0,
'name': name,
'properties': {'distro': 'rhel'}}
c = glance_client.Client(host=host, port=port,
auth_tok=token, creds=creds)
image_data = open(image_filename, "r")
image_meta = c.add_image(image_meta, image_data)
image_data.close()
return image_meta['id']
def glance_upload_v2(self, image, auth_token=None, **kwargs):
if image is None:
raise ImageFactoryException("No image is provided")
glance_host = kwargs.setdefault("glance-host", "127.0.0.1")
glance_port = kwargs.setdefault("glance-port", "9292")
glance_url = "http://%s:%s" % (glance_host, glance_port)
image_data = open(image, "r")
image_meta = {
'container_format': kwargs.setdefault('container_format', 'bare'),
'disk_format': kwargs.setdefault('disk_format', 'raw'),
'is_public': kwargs.setdefault('is_public', False),
'min_disk': kwargs.setdefault('min_disk', 0),
'min_ram': kwargs.setdefault('min_ram', 0),
'name': kwargs.setdefault('name', 'Factory Test Image'),
'data': image_data,
}
c = glance_client.Client('1', glance_url, token=auth_token)
image_meta = c.images.create(**image_meta)
image_data.close()
return image_meta.id
def decorated_function(*args, **kwargs):
if (not ApplicationConfiguration().configuration['no_oauth']):
validate_two_leg_oauth()
return f(*args, **kwargs)
def __init__(self, key):
consumers = ApplicationConfiguration().configuration['clients']
self.key = key
self.secret = consumers.get(key) if consumers else None
class OpenStackCloud(object):
zope.interface.implements(CloudDelegate)
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStackCloud, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
def activity(self, activity):
# Simple helper function
# Activity should be a one line human-readable string indicating the task in progress
# We log it at DEBUG and also set it as the status_detail on our active image
self.log.debug(activity)
self.active_image.status_detail['activity'] = activity
def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters):
# Our target_image is already a raw KVM image. All we need to do is upload to glance
self.builder = builder
self.active_image = self.builder.provider_image
self.openstack_decode_credentials(credentials)
provider_data = self.get_dynamic_provider_data(provider)
if provider_data is None:
raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided")
# Image is always here and it is the target_image datafile
input_image = self.builder.target_image.data
# If the template species a name, use that, otherwise create a name
# using provider_image.identifier.
template = Template(self.builder.provider_image.template)
if template.name:
image_name = template.name
else:
image_name = 'ImageFactory created image - %s' % (self.builder.provider_image.identifier)
if self.check_qcow_size(input_image):
self.log.debug("Uploading image to glance, detected qcow format")
disk_format='qcow2'
else:
self.log.debug("Uploading image to glance, assuming raw format")
disk_format='raw'
image_id = glance_upload(input_image, creds = self.credentials_dict, token = self.credentials_token,
host=provider_data['glance-host'], port=provider_data['glance-port'],
name=image_name, disk_format=disk_format)
self.builder.provider_image.identifier_on_provider = image_id
if 'username' in self.credentials_dict:
self.builder.provider_image.provider_account_identifier = self.credentials_dict['username']
self.percent_complete=100
def openstack_decode_credentials(self, credentials):
self.activity("Preparing OpenStack credentials")
# TODO: Validate these - in particular, ensure that if some nodes are missing at least
# a minimal acceptable set of auth is present
doc = libxml2.parseDoc(credentials)
self.credentials_dict = { }
for authprop in [ 'auth_url', 'password', 'strategy', 'tenant', 'username']:
self.credentials_dict[authprop] = self._get_xml_node(doc, authprop)
self.credentials_token = self._get_xml_node(doc, 'token')
def _get_xml_node(self, doc, credtype):
nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype))
# OpenStack supports multiple auth schemes so not all nodes are required
if len(nodes) < 1:
return None
return nodes[0].content
def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters):
# TODO: Implement snapshot builds
raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM")
def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
return True
def builder_will_create_target_image(self, builder, target, image_id, template, parameters):
pass
def builder_did_create_target_image(self, builder, target, image_id, template, parameters):
self.target=target
self.builder=builder
self.modify_oz_filesystem()
# OS plugin has already provided the initial file for us to work with
# which we can currently assume is a raw image
input_image = builder.target_image.data
# Support conversion to alternate preferred image format
# Currently only handle qcow2, but the size reduction of
# using this avoids the performance penalty of uploading
# (and launching) raw disk images on slow storage
if self.app_config.get('openstack_image_format', 'raw') == 'qcow2':
self.log.debug("Converting RAW image to compressed qcow2 format")
rc = os.system("qemu-img convert -c -O qcow2 %s %s" %
(input_image, input_image + ".tmp.qcow2"))
if rc == 0:
os.unlink(input_image)
os.rename(input_image + ".tmp.qcow2", input_image)
else:
raise ImageFactoryException("qemu-img convert failed!")
def modify_oz_filesystem(self):
self.log.debug("Doing further Factory specific modification of Oz image")
guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data)
remove_net_persist(guestfs_handle)
create_cloud_info(guestfs_handle, self.target)
shutdown_and_close(guestfs_handle)
def get_dynamic_provider_data(self, provider):
try:
xml_et = fromstring(provider)
return xml_et.attrib
except Exception as e:
self.log.debug('Testing provider for XML: %s' % e)
pass
try:
jload = json.loads(provider)
return jload
except ValueError as e:
self.log.debug('Testing provider for JSON: %s' % e)
pass
return None
# FIXME : cut/paste from RHEVMHelper.py, should refactor into a common utility class
def check_qcow_size(self, filename):
# Detect if an image is in qcow format
# If it is, return the size of the underlying disk image
# If it isn't, return None
# For interested parties, this is the QCOW header struct in C
# struct qcow_header {
# uint32_t magic;·
# uint32_t version;
# uint64_t backing_file_offset;
# uint32_t backing_file_size;
# uint32_t cluster_bits;
# uint64_t size; /* in bytes */
# uint32_t crypt_method;
# uint32_t l1_size;
# uint64_t l1_table_offset;
# uint64_t refcount_table_offset;
# uint32_t refcount_table_clusters;
# uint32_t nb_snapshots;
# uint64_t snapshots_offset;
# };
# And in Python struct format string-ese
qcow_struct=">IIQIIQIIQQIIQ" # > means big-endian
qcow_magic = 0x514649FB # 'Q' 'F' 'I' 0xFB
f = open(filename,"r")
pack = f.read(struct.calcsize(qcow_struct))
f.close()
unpack = struct.unpack(qcow_struct, pack)
if unpack[0] == qcow_magic:
return unpack[5]
else:
return None
def __init__(self):
super(vSphere, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
def __init__(self):
# Note that we are now missing ( template, target, config_block = None):
super(OpenStackCloud, self).__init__()
self.app_config = ApplicationConfiguration().configuration
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
