def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.version = GLANCE_VERSION if self.version == 2: self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username'] else: self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username']
def __init__(self): super(Nova, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.nib = None self._cloud_plugin_content = []
def decorated_function(*args, **kwargs): if(ApplicationConfiguration().configuration['debug']): log.debug('Handling %s HTTP %s REQUEST for resource at %s: %s' % (request.headers.get('Content-Type'), request.method, request.path, request.body.read())) return f(*args, **kwargs)
def __init__(self): super(IndirectionCloud, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.pim = PersistentImageManager.default_manager() self.res_mgr = ReservationManager()
def setUp(self): logging.basicConfig( level=logging.NOTSET, format= '%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse( ApplicationConfiguration().configuration["warehouse"]) self.template_xml = "<template>This is a test template. There is not much to it.</template>"
def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger("%s.%s" % (__name__, self.__class__.__name__)) self.version = GLANCE_VERSION if self.version == 2: self.credentials_attrs = ["auth_url", "password", "tenant", "username"] else: self.credentials_attrs = ["auth_url", "password", "strategy", "tenant", "username"]
def __init__(self): super(TinMan, self).__init__() self.cloud_plugin_content = [] config_obj = ApplicationConfiguration() self.app_config = config_obj.configuration self.res_mgr = ReservationManager() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.parameters = None self.install_script_object = None self.guest = None
def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.version = GLANCE_VERSION if self.version == 2: self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username'] elif self.version == 1: self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username']
def decorated_function(*args, **kwargs): if (ApplicationConfiguration().configuration['debug']): request_body = request.body.read() if ('credentials' in request_body): marker = 'provider_credentials' starting_index = request_body.find(marker) ending_index = request_body.rfind(marker) + len(marker) sensitive = request_body[starting_index:ending_index] request_body = request_body.replace(sensitive, 'REDACTED') log.debug('Handling %s HTTP %s REQUEST for resource at %s: %s' % (request.headers.get('Content-Type'), request.method, request.path, request_body)) return f(*args, **kwargs)
def __init__(self, url=None): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) appconfig = ApplicationConfiguration().configuration self.http = httplib2.Http() if(not url): url = appconfig['warehouse'] self.log.debug("Property (url) not specified. Pulling from application configuration: %s" % (url, )) self.image_bucket = appconfig['image_bucket'] self.build_bucket = appconfig['build_bucket'] self.target_image_bucket = appconfig['target_bucket'] self.template_bucket = appconfig['template_bucket'] self.icicle_bucket = appconfig['icicle_bucket'] self.provider_image_bucket = appconfig['provider_bucket'] self.warehouse_credentials = {'key':appconfig.get('warehouse_key'), 'secret':appconfig.get('warehouse_secret')} self.warehouse_oauth = True if(self.warehouse_credentials['key'] and self.warehouse_credentials['secret']) else False self.url = url.rstrip('/') self.log.debug("Created Image Warehouse instance %s" % (self, ))
def __init__(self): super(FedoraOS, self).__init__() self.cloud_plugin_content = [] config_obj = ApplicationConfiguration() self.app_config = config_obj.configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) if "ec2" in config_obj.jeos_images: self.ec2_jeos_amis = config_obj.jeos_images['ec2'] else: self.log.warning( "No JEOS amis defined for ec2. Snapshot builds will not be possible." ) self.ec2_jeos_amis = {}
def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(Rackspace, self).__init__() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) config_obj = ApplicationConfiguration() self.app_config = config_obj.configuration self.oz_config = ConfigParser.SafeConfigParser() self.oz_config.read("/etc/oz/oz.cfg") self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"]) self.active_image = None if "rackspace" in config_obj.jeos_images: self.rackspace_jeos_amis = config_obj.jeos_images['rackspace'] else: self.log.warning("No JEOS images defined for Rackspace. Snapshot builds will not be possible.") self.rackspace_jeos_amis = {}
def __init__(self, *args, **kwargs): config = json.loads(open('/etc/imagefactory/imagefactory.conf').read()) config['plugins'] = '/etc/imagefactory/plugins.d' config['timeout'] = 3600 ApplicationConfiguration(configuration=config) plugin_mgr = PluginManager('/etc/imagefactory/plugins.d') plugin_mgr.load() logfile = os.path.join(kwargs['workdir'], 'imgfac.log') self.fhandler = logging.FileHandler(logfile) self.tlog = logging.getLogger() self.tlog.setLevel(logging.DEBUG) self.tlog.addHandler(self.fhandler) pass
def __init__(self, *args, **kwargs): config = json.loads(open('/etc/imagefactory/imagefactory.conf').read()) config['plugins'] = '/etc/imagefactory/plugins.d' config['timeout'] = 3600 config['rhevm_image_format'] = 'qcow2' ApplicationConfiguration(configuration=config) plugin_mgr = PluginManager('/etc/imagefactory/plugins.d') plugin_mgr.load() self.fhandler = logging.StreamHandler(sys.stdout) self.tlog = logging.getLogger() self.tlog.setLevel(logging.DEBUG) self.tlog.addHandler(self.fhandler) verbosemode = kwargs.get('verbosemode', False) if verbosemode: ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) self.tlog.addHandler(ch) pass
def __new__(cls, *p, **k): if cls.instance is None: i = super(ReservationManager, cls).__new__(cls, *p, **k) # initialize here, not in __init__() i.log = logging.getLogger('%s.%s' % (__name__, i.__class__.__name__)) i.default_minimum = cls.DEFAULT_MINIMUM i._mounts = dict() i.appconfig = ApplicationConfiguration().configuration i._queues = dict(local=BoundedSemaphore( i.appconfig.get('max_concurrent_local_sessions', 1)), ec2=BoundedSemaphore( i.appconfig.get('max_concurrent_ec2_sessions', 1))) i._named_locks = {} i._named_locks_lock = BoundedSemaphore() # Initialize based on PID to prevent conflicts from multiple CLI runs on the same machine # TODO: This is TinMan/Oz specific - move it to the plugin i._listen_port = cls.MIN_PORT + (os.getpid() % (cls.MAX_PORT - cls.MIN_PORT)) i._listen_port_lock = BoundedSemaphore() cls.instance = i return cls.instance
class OpenStack(object): zope.interface.implements(CloudDelegate) def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.version = GLANCE_VERSION if self.version == 2: self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username'] self.openrc_credentials_attrs = { 'OS_AUTH_URL': 'auth_url', 'OS_TENANT_NAME': 'tenant', 'OS_USERNAME': '******'} else: self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username'] def activity(self, activity): # Simple helper function # Activity should be a one line human-readable string indicating the task in progress # We log it at DEBUG and also set it as the status_detail on our active image self.log.debug(activity) self.active_image.status_detail['activity'] = activity def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters): # Our target_image is already a raw KVM image. All we need to do is upload to glance self.builder = builder self.active_image = self.builder.provider_image self.openstack_decode_credentials(credentials) provider_data = self.get_dynamic_provider_data(provider) if provider_data is None: raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided") # Image is always here and it is the target_image datafile input_image = self.builder.target_image.data # If the template species a name, use that, otherwise create a name # using provider_image.identifier. template = Template(self.builder.provider_image.template) if template.name: image_name = template.name else: image_name = 'ImageFactory created image - %s' % (self.builder.provider_image.identifier) if self.check_qcow_size(input_image): self.log.debug("Uploading image to glance, detected qcow format") disk_format='qcow2' else: self.log.debug("Uploading image to glance, assuming raw format") disk_format='raw' # Support openstack grizzly keystone authentication and glance upload if self.version == 2: if self.credentials_token is None: self.credentials_token = self.keystone_authenticate(**self.credentials_dict) provider_data['name'] = image_name provider_data['disk_format'] = disk_format image_id = self.glance_upload_v2(input_image, self.credentials_token, **provider_data) else: # Also support backward compatible for folsom image_id = self.glance_upload(input_image, creds = self.credentials_dict, token = self.credentials_token, host=provider_data['glance-host'], port=provider_data['glance-port'], name=image_name, disk_format=disk_format) self.builder.provider_image.identifier_on_provider = image_id if 'username' in self.credentials_dict: self.builder.provider_image.provider_account_identifier = self.credentials_dict['username'] self.percent_complete=100 def openstack_decode_credentials(self, credentials): self.activity("Preparing OpenStack credentials") # TODO: Validate these - in particular, ensure that if some nodes are missing at least # a minimal acceptable set of auth is present try: doc = libxml2.parseDoc(credentials) except libxml2.parserError: self._populate_credentials_dict_from_openrc(credentials) return self.credentials_dict = { } for authprop in self.credentials_attrs: value = self._get_xml_node(doc, authprop) if value is not None: self.credentials_dict[authprop] = value self.credentials_token = self._get_xml_node(doc, 'token') def _get_xml_node(self, doc, credtype): nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype)) # OpenStack supports multiple auth schemes so not all nodes are required if len(nodes) < 1: return None return nodes[0].content def _populate_credentials_dict_from_openrc(self, credentials_str): credentials_list = credentials_str.split('\n') self.credentials_dict = {} for authprop in self.openrc_credentials_attrs.keys(): corresponding_authprop = self.openrc_credentials_attrs[authprop] matching = [line for line in credentials_list if authprop in line] if matching == []: raise ImageFactoryException("Credentials file does not have required field - " + authprop) self.credentials_dict[corresponding_authprop] = matching[0][matching[0].index('=') + 1 : ].strip('"') self.credentials_dict['password'] = getpass.getpass(prompt='Please enter your OpenStack Password:\n') self.credentials_token = None def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters): # TODO: Implement snapshot builds raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM") def builder_should_create_target_image(self, builder, target, image_id, template, parameters): return True def builder_will_create_target_image(self, builder, target, image_id, template, parameters): pass def builder_did_create_target_image(self, builder, target, image_id, template, parameters): self.target=target self.builder=builder self.modify_oz_filesystem() # OS plugin has already provided the initial file for us to work with # which we can currently assume is a raw image input_image = builder.target_image.data # Support conversion to alternate preferred image format # Currently only handle qcow2, but the size reduction of # using this avoids the performance penalty of uploading # (and launching) raw disk images on slow storage if self.app_config.get('openstack_image_format', 'raw') == 'qcow2': # Prevent double convert if the image is already qcow2 if self.check_qcow_size(input_image) is not None: self.log.debug("No conversion require. Image already in qcow2 format.") return self.log.debug("Converting RAW image to compressed qcow2 format") rc = os.system("qemu-img convert -c -O qcow2 %s %s" % (input_image, input_image + ".tmp.qcow2")) if rc == 0: os.unlink(input_image) os.rename(input_image + ".tmp.qcow2", input_image) else: raise ImageFactoryException("qemu-img convert failed!") def modify_oz_filesystem(self): self.log.debug("Doing further Factory specific modification of Oz image") guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data) remove_net_persist(guestfs_handle) create_cloud_info(guestfs_handle, self.target) shutdown_and_close(guestfs_handle) def get_dynamic_provider_data(self, provider): try: xml_et = fromstring(provider) return xml_et.attrib except Exception as e: self.log.debug('Testing provider for XML: %s' % e) pass try: jload = json.loads(provider) return jload except ValueError as e: self.log.debug('Testing provider for JSON: %s' % e) pass return None def keystone_authenticate(self, **kwargs): user = kwargs.get('username') pwd = kwargs.get('password') tenant = kwargs.get('tenant') url = kwargs.get('auth_url', 'http://127.0.0.1:5000/v2.0') keystone = client.Client(username=user, password=pwd, tenant_name=tenant, auth_url=url) keystone.authenticate() return keystone.auth_token def glance_upload(self, image_filename, creds = {'auth_url': None, 'password': None, 'strategy': 'noauth', 'tenant': None, 'username': None}, host = "0.0.0.0", port = "9292", token = None, name = 'Factory Test Image', disk_format = 'raw'): image_meta = {'container_format': 'bare', 'disk_format': disk_format, 'is_public': True, 'min_disk': 0, 'min_ram': 0, 'name': name, 'properties': {'distro': 'rhel'}} c = glance_client.Client(host=host, port=port, auth_tok=token, creds=creds) image_data = open(image_filename, "r") image_meta = c.add_image(image_meta, image_data) image_data.close() return image_meta['id'] def glance_upload_v2(self, image, auth_token=None, **kwargs): if image is None: raise ImageFactoryException("No image is provided") glance_host = kwargs.setdefault("glance-host", "127.0.0.1") glance_port = kwargs.setdefault("glance-port", "9292") glance_url = "http://%s:%s" % (glance_host, glance_port) image_data = open(image, "r") image_meta = { 'container_format': kwargs.setdefault('container_format', 'bare'), 'disk_format': kwargs.setdefault('disk_format', 'raw'), 'is_public': kwargs.setdefault('is_public', False), 'min_disk': kwargs.setdefault('min_disk', 0), 'min_ram': kwargs.setdefault('min_ram', 0), 'name': kwargs.setdefault('name', 'Factory Test Image'), 'data': image_data, 'properties': kwargs.setdefault('properties', '{}') } c = glance_client.Client('1', glance_url, token=auth_token) image_meta = c.images.create(**image_meta) image_data.close() return image_meta.id # FIXME : cut/paste from RHEVMHelper.py, should refactor into a common utility class def check_qcow_size(self, filename): # Detect if an image is in qcow format # If it is, return the size of the underlying disk image # If it isn't, return None # For interested parties, this is the QCOW header struct in C # struct qcow_header { # uint32_t magic; # uint32_t version; # uint64_t backing_file_offset; # uint32_t backing_file_size; # uint32_t cluster_bits; # uint64_t size; /* in bytes */ # uint32_t crypt_method; # uint32_t l1_size; # uint64_t l1_table_offset; # uint64_t refcount_table_offset; # uint32_t refcount_table_clusters; # uint32_t nb_snapshots; # uint64_t snapshots_offset; # }; # And in Python struct format string-ese qcow_struct=">IIQIIQIIQQIIQ" # > means big-endian qcow_magic = 0x514649FB # 'Q' 'F' 'I' 0xFB f = open(filename,"r") pack = f.read(struct.calcsize(qcow_struct)) f.close() unpack = struct.unpack(qcow_struct, pack) if unpack[0] == qcow_magic: return unpack[5] else: return None
def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
class OpenStack(object): zope.interface.implements(CloudDelegate) def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) def activity(self, activity): # Simple helper function # Activity should be a one line human-readable string indicating the task in progress # We log it at DEBUG and also set it as the status_detail on our active image self.log.debug(activity) self.active_image.status_detail['activity'] = activity def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters): # Our target_image is already a raw KVM image. All we need to do is upload to glance self.builder = builder self.active_image = self.builder.provider_image self.openstack_decode_credentials(credentials) provider_data = self.get_dynamic_provider_data(provider) if provider_data is None: raise ImageFactoryException( "OpenStack KVM instance not found in XML or JSON provided") # Image is always here and it is the target_image datafile input_image = self.builder.target_image.data # If the template species a name, use that, otherwise create a name # using provider_image.identifier. template = Template(self.builder.provider_image.template) if template.name: image_name = template.name else: image_name = 'ImageFactory created image - %s' % ( self.builder.provider_image.identifier) if self.check_qcow_size(input_image): self.log.debug("Uploading image to glance, detected qcow format") disk_format = 'qcow2' else: self.log.debug("Uploading image to glance, assuming raw format") disk_format = 'raw' image_id = glance_upload(input_image, creds=self.credentials_dict, token=self.credentials_token, host=provider_data['glance-host'], port=provider_data['glance-port'], name=image_name, disk_format=disk_format) self.builder.provider_image.identifier_on_provider = image_id if 'username' in self.credentials_dict: self.builder.provider_image.provider_account_identifier = self.credentials_dict[ 'username'] self.percent_complete = 100 def openstack_decode_credentials(self, credentials): self.activity("Preparing OpenStack credentials") # TODO: Validate these - in particular, ensure that if some nodes are missing at least # a minimal acceptable set of auth is present doc = libxml2.parseDoc(credentials) self.credentials_dict = {} for authprop in [ 'auth_url', 'password', 'strategy', 'tenant', 'username' ]: self.credentials_dict[authprop] = self._get_xml_node(doc, authprop) self.credentials_token = self._get_xml_node(doc, 'token') def _get_xml_node(self, doc, credtype): nodes = doc.xpathEval( "//provider_credentials/openstack_credentials/%s" % (credtype)) # OpenStack supports multiple auth schemes so not all nodes are required if len(nodes) < 1: return None return nodes[0].content def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters): # TODO: Implement snapshot builds raise ImageFactoryException( "Snapshot builds not currently supported on OpenStack KVM") def builder_should_create_target_image(self, builder, target, image_id, template, parameters): return True def builder_will_create_target_image(self, builder, target, image_id, template, parameters): pass def builder_did_create_target_image(self, builder, target, image_id, template, parameters): self.target = target self.builder = builder self.modify_oz_filesystem() # OS plugin has already provided the initial file for us to work with # which we can currently assume is a raw image input_image = builder.target_image.data # Support conversion to alternate preferred image format # Currently only handle qcow2, but the size reduction of # using this avoids the performance penalty of uploading # (and launching) raw disk images on slow storage if self.app_config.get('openstack_image_format', 'raw') == 'qcow2': self.log.debug("Converting RAW image to compressed qcow2 format") rc = os.system("qemu-img convert -c -O qcow2 %s %s" % (input_image, input_image + ".tmp.qcow2")) if rc == 0: os.unlink(input_image) os.rename(input_image + ".tmp.qcow2", input_image) else: raise ImageFactoryException("qemu-img convert failed!") def modify_oz_filesystem(self): self.log.debug( "Doing further Factory specific modification of Oz image") guestfs_handle = launch_inspect_and_mount( self.builder.target_image.data) remove_net_persist(guestfs_handle) create_cloud_info(guestfs_handle, self.target) shutdown_and_close(guestfs_handle) def get_dynamic_provider_data(self, provider): try: xml_et = fromstring(provider) return xml_et.attrib except Exception as e: self.log.debug('Testing provider for XML: %s' % e) pass try: jload = json.loads(provider) return jload except ValueError as e: self.log.debug('Testing provider for JSON: %s' % e) pass return None # FIXME : cut/paste from RHEVMHelper.py, should refactor into a common utility class def check_qcow_size(self, filename): # Detect if an image is in qcow format # If it is, return the size of the underlying disk image # If it isn't, return None # For interested parties, this is the QCOW header struct in C # struct qcow_header { # uint32_t magic;· # uint32_t version; # uint64_t backing_file_offset; # uint32_t backing_file_size; # uint32_t cluster_bits; # uint64_t size; /* in bytes */ # uint32_t crypt_method; # uint32_t l1_size; # uint64_t l1_table_offset; # uint64_t refcount_table_offset; # uint32_t refcount_table_clusters; # uint32_t nb_snapshots; # uint64_t snapshots_offset; # }; # And in Python struct format string-ese qcow_struct = ">IIQIIQIIQQIIQ" # > means big-endian qcow_magic = 0x514649FB # 'Q' 'F' 'I' 0xFB f = open(filename, "r") pack = f.read(struct.calcsize(qcow_struct)) f.close() unpack = struct.unpack(qcow_struct, pack) if unpack[0] == qcow_magic: return unpack[5] else: return None
def __init__(self, key): consumers = ApplicationConfiguration().configuration['clients'] self.key = key self.secret = consumers.get(key) if consumers else None
class OpenStack(object): zope.interface.implements(CloudDelegate) def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger("%s.%s" % (__name__, self.__class__.__name__)) self.version = GLANCE_VERSION if self.version == 2: self.credentials_attrs = ["auth_url", "password", "tenant", "username"] else: self.credentials_attrs = ["auth_url", "password", "strategy", "tenant", "username"] def activity(self, activity): # Simple helper function # Activity should be a one line human-readable string indicating the task in progress # We log it at DEBUG and also set it as the status_detail on our active image self.log.debug(activity) self.active_image.status_detail["activity"] = activity def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters): # Our target_image is already a raw KVM image. All we need to do is upload to glance self.builder = builder self.active_image = self.builder.provider_image self.openstack_decode_credentials(credentials) provider_data = self.get_dynamic_provider_data(provider) if provider_data is None: raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided") # Image is always here and it is the target_image datafile input_image = self.builder.target_image.data # If the template species a name, use that, otherwise create a name # using provider_image.identifier. template = Template(self.builder.provider_image.template) if template.name: image_name = template.name else: image_name = "ImageFactory created image - %s" % (self.builder.provider_image.identifier) if check_qcow_size(input_image): self.log.debug("Uploading image to glance, detected qcow format") disk_format = "qcow2" else: self.log.debug("Uploading image to glance, assuming raw format") disk_format = "raw" # Support openstack grizzly keystone authentication and glance upload if self.version == 2: if self.credentials_token is None: self.credentials_token = self.keystone_authenticate(**self.credentials_dict) provider_data["name"] = image_name provider_data["disk_format"] = disk_format image_id = self.glance_upload_v2(input_image, self.credentials_token, **provider_data) else: # Also support backward compatible for folsom image_id = self.glance_upload( input_image, creds=self.credentials_dict, token=self.credentials_token, host=provider_data["glance-host"], port=provider_data["glance-port"], name=image_name, disk_format=disk_format, ) self.builder.provider_image.identifier_on_provider = image_id if "username" in self.credentials_dict: self.builder.provider_image.provider_account_identifier = self.credentials_dict["username"] self.percent_complete = 100 def openstack_decode_credentials(self, credentials): self.activity("Preparing OpenStack credentials") # TODO: Validate these - in particular, ensure that if some nodes are missing at least # a minimal acceptable set of auth is present doc = libxml2.parseDoc(credentials) self.credentials_dict = {} for authprop in self.credentials_attrs: value = self._get_xml_node(doc, authprop) if value is not None: self.credentials_dict[authprop] = value self.credentials_token = self._get_xml_node(doc, "token") def _get_xml_node(self, doc, credtype): nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype)) # OpenStack supports multiple auth schemes so not all nodes are required if len(nodes) < 1: return None return nodes[0].content def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters): # TODO: Implement snapshot builds raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM") def builder_should_create_target_image(self, builder, target, image_id, template, parameters): return True def builder_will_create_target_image(self, builder, target, image_id, template, parameters): pass def builder_did_create_target_image(self, builder, target, image_id, template, parameters): self.target = target self.builder = builder self.modify_oz_filesystem() # OS plugin has already provided the initial file for us to work with # which we can currently assume is a raw image input_image = builder.target_image.data # Support conversion to alternate preferred image format # Currently only handle qcow2, but the size reduction of # using this avoids the performance penalty of uploading # (and launching) raw disk images on slow storage if self.app_config.get("openstack_image_format", "raw") == "qcow2": # None of the existing input base_image plugins produce compressed qcow2 output # the step below is either going from raw to compressed qcow2 or # uncompressed qcow2 to compressed qcow2 self.log.debug("Converting image to compressed qcow2 format") tmp_output = input_image + ".tmp.qcow2" convert_cmd = qemu_convert_cmd(input_image, tmp_output, True) (stdout, stderr, retcode) = subprocess_check_output(convert_cmd) os.unlink(input_image) os.rename(tmp_output, input_image) def modify_oz_filesystem(self): self.log.debug("Doing further Factory specific modification of Oz image") guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data) remove_net_persist(guestfs_handle) create_cloud_info(guestfs_handle, self.target) shutdown_and_close(guestfs_handle) def get_dynamic_provider_data(self, provider): try: xml_et = fromstring(provider) return xml_et.attrib except Exception as e: self.log.debug("Testing provider for XML: %s" % e) pass try: jload = json.loads(provider) return jload except ValueError as e: self.log.debug("Testing provider for JSON: %s" % e) pass return None def keystone_authenticate(self, **kwargs): user = kwargs.get("username") pwd = kwargs.get("password") tenant = kwargs.get("tenant") url = kwargs.get("auth_url", "http://127.0.0.1:5000/v2.0") keystone = client.Client(username=user, password=pwd, tenant_name=tenant, auth_url=url) keystone.authenticate() return keystone.auth_token def glance_upload( self, image_filename, creds={"auth_url": None, "password": None, "strategy": "noauth", "tenant": None, "username": None}, host="0.0.0.0", port="9292", token=None, name="Factory Test Image", disk_format="raw", ): image_meta = { "container_format": "bare", "disk_format": disk_format, "is_public": True, "min_disk": 0, "min_ram": 0, "name": name, "properties": {"distro": "rhel"}, } c = glance_client.Client(host=host, port=port, auth_tok=token, creds=creds) image_data = open(image_filename, "r") image_meta = c.add_image(image_meta, image_data) image_data.close() return image_meta["id"] def glance_upload_v2(self, image, auth_token=None, **kwargs): if image is None: raise ImageFactoryException("No image is provided") glance_host = kwargs.setdefault("glance-host", "127.0.0.1") glance_port = kwargs.setdefault("glance-port", "9292") glance_url = "http://%s:%s" % (glance_host, glance_port) image_data = open(image, "r") image_meta = { "container_format": kwargs.setdefault("container_format", "bare"), "disk_format": kwargs.setdefault("disk_format", "raw"), "is_public": kwargs.setdefault("is_public", False), "min_disk": kwargs.setdefault("min_disk", 0), "min_ram": kwargs.setdefault("min_ram", 0), "name": kwargs.setdefault("name", "Factory Test Image"), "data": image_data, } c = glance_client.Client("1", glance_url, token=auth_token) image_meta = c.images.create(**image_meta) image_data.close() return image_meta.id
def testSingleton(self): self.assertTrue( id(ApplicationConfiguration()) == id(ApplicationConfiguration()))
class OpenStack(object): zope.interface.implements(CloudDelegate) def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStack, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.version = GLANCE_VERSION if self.version == 2: self.credentials_attrs = [ 'auth_url', 'password', 'tenant', 'username'] else: self.credentials_attrs = [ 'auth_url', 'password', 'strategy', 'tenant', 'username'] def activity(self, activity): # Simple helper function # Activity should be a one line human-readable string indicating the task in progress # We log it at DEBUG and also set it as the status_detail on our active image self.log.debug(activity) self.active_image.status_detail['activity'] = activity def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters): # Our target_image is already a raw KVM image. All we need to do is upload to glance self.builder = builder self.active_image = self.builder.provider_image self.openstack_decode_credentials(credentials) provider_data = self.get_dynamic_provider_data(provider) if provider_data is None: raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided") # Image is always here and it is the target_image datafile input_image = self.builder.target_image.data # If the template species a name, use that, otherwise create a name # using provider_image.identifier. template = Template(self.builder.provider_image.template) if template.name: image_name = template.name else: image_name = 'ImageFactory created image - %s' % (self.builder.provider_image.identifier) if check_qcow_size(input_image): self.log.debug("Uploading image to glance, detected qcow format") disk_format='qcow2' else: self.log.debug("Uploading image to glance, assuming raw format") disk_format='raw' # Support openstack grizzly keystone authentication and glance upload if self.version == 2: if self.credentials_token is None: self.credentials_token = self.keystone_authenticate(**self.credentials_dict) provider_data['name'] = image_name provider_data['disk_format'] = disk_format image_id = self.glance_upload_v2(input_image, self.credentials_token, **provider_data) else: # Also support backward compatible for folsom image_id = self.glance_upload(input_image, creds = self.credentials_dict, token = self.credentials_token, host=provider_data['glance-host'], port=provider_data['glance-port'], name=image_name, disk_format=disk_format) self.builder.provider_image.identifier_on_provider = image_id if 'username' in self.credentials_dict: self.builder.provider_image.provider_account_identifier = self.credentials_dict['username'] self.percent_complete=100 def openstack_decode_credentials(self, credentials): self.activity("Preparing OpenStack credentials") # TODO: Validate these - in particular, ensure that if some nodes are missing at least # a minimal acceptable set of auth is present doc = libxml2.parseDoc(credentials) self.credentials_dict = { } for authprop in self.credentials_attrs: value = self._get_xml_node(doc, authprop) if value is not None: self.credentials_dict[authprop] = value self.credentials_token = self._get_xml_node(doc, 'token') def _get_xml_node(self, doc, credtype): nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype)) # OpenStack supports multiple auth schemes so not all nodes are required if len(nodes) < 1: return None return nodes[0].content def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters): # TODO: Implement snapshot builds raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM") def builder_should_create_target_image(self, builder, target, image_id, template, parameters): return True def builder_will_create_target_image(self, builder, target, image_id, template, parameters): pass def builder_did_create_target_image(self, builder, target, image_id, template, parameters): self.target=target self.builder=builder self.modify_oz_filesystem() # OS plugin has already provided the initial file for us to work with # which we can currently assume is a raw image input_image = builder.target_image.data # Support conversion to alternate preferred image format # Currently only handle qcow2, but the size reduction of # using this avoids the performance penalty of uploading # (and launching) raw disk images on slow storage if self.app_config.get('openstack_image_format', 'raw') == 'qcow2': # None of the existing input base_image plugins produce compressed qcow2 output # the step below is either going from raw to compressed qcow2 or # uncompressed qcow2 to compressed qcow2 self.log.debug("Converting image to compressed qcow2 format") tmp_output = input_image + ".tmp.qcow2" convert_cmd = qemu_convert_cmd(input_image, tmp_output, True) (stdout, stderr, retcode) = subprocess_check_output(convert_cmd) os.unlink(input_image) os.rename(tmp_output, input_image) def modify_oz_filesystem(self): self.log.debug("Doing further Factory specific modification of Oz image") guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data) remove_net_persist(guestfs_handle) create_cloud_info(guestfs_handle, self.target) shutdown_and_close(guestfs_handle) def get_dynamic_provider_data(self, provider): try: xml_et = fromstring(provider) return xml_et.attrib except Exception as e: self.log.debug('Testing provider for XML: %s' % e) pass try: jload = json.loads(provider) return jload except ValueError as e: self.log.debug('Testing provider for JSON: %s' % e) pass return None def keystone_authenticate(self, **kwargs): user = kwargs.get('username') pwd = kwargs.get('password') tenant = kwargs.get('tenant') url = kwargs.get('auth_url', 'http://127.0.0.1:5000/v2.0') keystone = client.Client(username=user, password=pwd, tenant_name=tenant, auth_url=url) keystone.authenticate() return keystone.auth_token def glance_upload(self, image_filename, creds = {'auth_url': None, 'password': None, 'strategy': 'noauth', 'tenant': None, 'username': None}, host = "0.0.0.0", port = "9292", token = None, name = 'Factory Test Image', disk_format = 'raw'): image_meta = {'container_format': 'bare', 'disk_format': disk_format, 'is_public': True, 'min_disk': 0, 'min_ram': 0, 'name': name, 'properties': {'distro': 'rhel'}} c = glance_client.Client(host=host, port=port, auth_tok=token, creds=creds) image_data = open(image_filename, "r") image_meta = c.add_image(image_meta, image_data) image_data.close() return image_meta['id'] def glance_upload_v2(self, image, auth_token=None, **kwargs): if image is None: raise ImageFactoryException("No image is provided") glance_host = kwargs.setdefault("glance-host", "127.0.0.1") glance_port = kwargs.setdefault("glance-port", "9292") glance_url = "http://%s:%s" % (glance_host, glance_port) image_data = open(image, "r") image_meta = { 'container_format': kwargs.setdefault('container_format', 'bare'), 'disk_format': kwargs.setdefault('disk_format', 'raw'), 'is_public': kwargs.setdefault('is_public', False), 'min_disk': kwargs.setdefault('min_disk', 0), 'min_ram': kwargs.setdefault('min_ram', 0), 'name': kwargs.setdefault('name', 'Factory Test Image'), 'data': image_data, } c = glance_client.Client('1', glance_url, token=auth_token) image_meta = c.images.create(**image_meta) image_data.close() return image_meta.id
def decorated_function(*args, **kwargs): if (not ApplicationConfiguration().configuration['no_oauth']): validate_two_leg_oauth() return f(*args, **kwargs)
class OpenStackCloud(object): zope.interface.implements(CloudDelegate) def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStackCloud, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) def activity(self, activity): # Simple helper function # Activity should be a one line human-readable string indicating the task in progress # We log it at DEBUG and also set it as the status_detail on our active image self.log.debug(activity) self.active_image.status_detail['activity'] = activity def push_image_to_provider(self, builder, provider, credentials, target, target_image, parameters): # Our target_image is already a raw KVM image. All we need to do is upload to glance self.builder = builder self.active_image = self.builder.provider_image self.openstack_decode_credentials(credentials) provider_data = self.get_dynamic_provider_data(provider) if provider_data is None: raise ImageFactoryException("OpenStack KVM instance not found in XML or JSON provided") # Image is always here and it is the target_image datafile input_image = self.builder.target_image.data # If the template species a name, use that, otherwise create a name # using provider_image.identifier. template = Template(self.builder.provider_image.template) if template.name: image_name = template.name else: image_name = 'ImageFactory created image - %s' % (self.builder.provider_image.identifier) if self.check_qcow_size(input_image): self.log.debug("Uploading image to glance, detected qcow format") disk_format='qcow2' else: self.log.debug("Uploading image to glance, assuming raw format") disk_format='raw' image_id = glance_upload(input_image, creds = self.credentials_dict, token = self.credentials_token, host=provider_data['glance-host'], port=provider_data['glance-port'], name=image_name, disk_format=disk_format) self.builder.provider_image.identifier_on_provider = image_id if 'username' in self.credentials_dict: self.builder.provider_image.provider_account_identifier = self.credentials_dict['username'] self.percent_complete=100 def openstack_decode_credentials(self, credentials): self.activity("Preparing OpenStack credentials") # TODO: Validate these - in particular, ensure that if some nodes are missing at least # a minimal acceptable set of auth is present doc = libxml2.parseDoc(credentials) self.credentials_dict = { } for authprop in [ 'auth_url', 'password', 'strategy', 'tenant', 'username']: self.credentials_dict[authprop] = self._get_xml_node(doc, authprop) self.credentials_token = self._get_xml_node(doc, 'token') def _get_xml_node(self, doc, credtype): nodes = doc.xpathEval("//provider_credentials/openstack_credentials/%s" % (credtype)) # OpenStack supports multiple auth schemes so not all nodes are required if len(nodes) < 1: return None return nodes[0].content def snapshot_image_on_provider(self, builder, provider, credentials, template, parameters): # TODO: Implement snapshot builds raise ImageFactoryException("Snapshot builds not currently supported on OpenStack KVM") def builder_should_create_target_image(self, builder, target, image_id, template, parameters): return True def builder_will_create_target_image(self, builder, target, image_id, template, parameters): pass def builder_did_create_target_image(self, builder, target, image_id, template, parameters): self.target=target self.builder=builder self.modify_oz_filesystem() # OS plugin has already provided the initial file for us to work with # which we can currently assume is a raw image input_image = builder.target_image.data # Support conversion to alternate preferred image format # Currently only handle qcow2, but the size reduction of # using this avoids the performance penalty of uploading # (and launching) raw disk images on slow storage if self.app_config.get('openstack_image_format', 'raw') == 'qcow2': self.log.debug("Converting RAW image to compressed qcow2 format") rc = os.system("qemu-img convert -c -O qcow2 %s %s" % (input_image, input_image + ".tmp.qcow2")) if rc == 0: os.unlink(input_image) os.rename(input_image + ".tmp.qcow2", input_image) else: raise ImageFactoryException("qemu-img convert failed!") def modify_oz_filesystem(self): self.log.debug("Doing further Factory specific modification of Oz image") guestfs_handle = launch_inspect_and_mount(self.builder.target_image.data) remove_net_persist(guestfs_handle) create_cloud_info(guestfs_handle, self.target) shutdown_and_close(guestfs_handle) def get_dynamic_provider_data(self, provider): try: xml_et = fromstring(provider) return xml_et.attrib except Exception as e: self.log.debug('Testing provider for XML: %s' % e) pass try: jload = json.loads(provider) return jload except ValueError as e: self.log.debug('Testing provider for JSON: %s' % e) pass return None # FIXME : cut/paste from RHEVMHelper.py, should refactor into a common utility class def check_qcow_size(self, filename): # Detect if an image is in qcow format # If it is, return the size of the underlying disk image # If it isn't, return None # For interested parties, this is the QCOW header struct in C # struct qcow_header { # uint32_t magic;· # uint32_t version; # uint64_t backing_file_offset; # uint32_t backing_file_size; # uint32_t cluster_bits; # uint64_t size; /* in bytes */ # uint32_t crypt_method; # uint32_t l1_size; # uint64_t l1_table_offset; # uint64_t refcount_table_offset; # uint32_t refcount_table_clusters; # uint32_t nb_snapshots; # uint64_t snapshots_offset; # }; # And in Python struct format string-ese qcow_struct=">IIQIIQIIQQIIQ" # > means big-endian qcow_magic = 0x514649FB # 'Q' 'F' 'I' 0xFB f = open(filename,"r") pack = f.read(struct.calcsize(qcow_struct)) f.close() unpack = struct.unpack(qcow_struct, pack) if unpack[0] == qcow_magic: return unpack[5] else: return None
def __init__(self): super(vSphere, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
def __init__(self): # Note that we are now missing ( template, target, config_block = None): super(OpenStackCloud, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))