class TargetImage(PersistentImage): """ TODO: Docstring for TargetImage """ base_image_id = prop("_base_image_id") target = prop("_target") parameters = prop("_parameters") def __init__(self, image_id=None): super(TargetImage, self).__init__(image_id) self.base_image_id = None self.target = None def metadata(self): self.log.debug("Executing metadata in class (%s) my metadata is (%s)" % (self.__class__, METADATA)) return frozenset(METADATA + super(self.__class__, self).metadata())
class Notification(object): """ TODO: Docstring for Notification """ message = prop("_message") sender = prop("_sender") user_info = prop("_user_info") def __init__(self, message, sender, user_info=None): """ TODO: Fill me in @param message TODO @param sender TODO @param user_info TODO """ self._message = message self._sender = sender self._user_info = user_info
class ProviderImage(PersistentImage): """ TODO: Docstring for ProviderImage """ target_image_id = prop("_target_image_id") provider = prop("_provider") identifier_on_provider = prop("_identifier_on_provider") provider_account_identifier = prop("_provider_account_identifier") credentials = prop("_credentials") parameters = prop("_parameters") def __init__(self, image_id=None): """ TODO: Fill me in @param template TODO @param target_img_id TODO """ super(ProviderImage, self).__init__(image_id) self.target_image_id = None self.provider = None self.credentials = None self.parameters = None def metadata(self): self.log.debug("Executing metadata in class (%s) my metadata is (%s)" % (self.__class__, METADATA)) return frozenset(METADATA + super(self.__class__, self).metadata())
class ApplicationConfiguration(Singleton): configuration = props.prop("_configuration", "The configuration property.") def _singleton_init(self): super(ApplicationConfiguration, self)._singleton_init() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.configuration = self.__parse_arguments() if not 'debug' in self.configuration: # Slightly confusing, I know - For daemon mode we have a debug argument with default False # For cli, we debug by default and have a nodebug argument with default False # Rest of the code assumes a 'debug' value in app_config so set it here self.configuration['debug'] = not self.configuration['nodebug'] self.jeos_images = {} self.__parse_jeos_images() def __init__(self): pass def __new_argument_parser(self, appname): main_description = """Image Factory is an application for creating system images for use on public and private clouds.""" argparser = argparse.ArgumentParser(description=main_description, prog=appname, version=VERSION) argparser.add_argument('--verbose', action='store_true', default=False, help='Set verbose logging.') argparser.add_argument( '--config', default='/etc/imagefactory/imagefactory.conf', help='Configuration file to use. (default: %(default)s)') argparser.add_argument( '--imgdir', default='/tmp', help= 'Build image files in location specified. (default: %(default)s)') argparser.add_argument( '--timeout', type=int, default=3600, help= 'Set the timeout period for image building in seconds. (default: %(default)s)' ) argparser.add_argument( '--tmpdir', default='/tmp', help= 'Use the specified location for temporary files. (default: %(default)s)' ) argparser.add_argument('--plugins', default='/etc/imagefactory/plugins.d', help='Plugin directory. (default: %(default)s)') group_ec2 = argparser.add_argument_group(title='EC2 settings') group_ec2.add_argument( '--ec2-32bit-util', default='m1.small', help='Instance type to use when launching a 32 bit utility instance' ) group_ec2.add_argument( '--ec2-64bit-util', default='m1.large', help='Instance type to use when launching a 64 bit utility instance' ) if (appname == 'imagefactoryd'): argparser.add_argument( '--debug', action='store_true', default=False, help='Set really verbose logging for debugging.') argparser.add_argument( '--foreground', action='store_true', default=False, help= 'Stay in the foreground and avoid launching a daemon. (default: %(default)s)' ) group_rest = argparser.add_argument_group( title='REST service options') group_rest.add_argument( '--port', type=int, default=8075, help= 'Port to attach the RESTful http interface to. (defaul: %(default)s)' ) group_rest.add_argument( '--address', default='0.0.0.0', help='Interface address to listen to. (defaul: %(default)s)') group_rest.add_argument( '--no_ssl', action='store_true', default=False, help='Turn off SSL. (default: %(default)s)') group_rest.add_argument( '--ssl_pem', default='*', help= 'PEM certificate file to use for HTTPS access to the REST interface. (default: A transient certificate is generated at runtime.)' ) group_rest.add_argument( '--no_oauth', action='store_true', default=False, help= 'Use 2 legged OAuth to protect the REST interface. (default: %(default)s)' ) elif (appname == 'imagefactory'): argparser.add_argument( '--nodebug', action='store_true', default=False, help='Turn off the default verbose CLI logging') argparser.add_argument( '--output', choices=('log', 'json'), default='log', help='Choose between log or json output. (default: %(default)s)' ) argparser.add_argument('--raw', action='store_true', default=False, help='Turn off pretty printing.') subparsers = argparser.add_subparsers(title='commands', dest='command') template_help = 'A file containing the TDL for this image.' cmd_base = subparsers.add_parser('base_image', help='Build a generic image.') cmd_base.add_argument('template', type=argparse.FileType(), help=template_help) cmd_base.add_argument('--paramaters') cmd_target = subparsers.add_parser( 'target_image', help='Customize an image for a given cloud.') cmd_target.add_argument( 'target', help= 'The name of the target cloud for which to customize the image.' ) target_group = cmd_target.add_mutually_exclusive_group( required=True) target_group.add_argument( '--id', help='The uuid of the BaseImage to customize.') target_group.add_argument('--template', type=argparse.FileType(), help=template_help) cmd_target.add_argument('--parameters') cmd_provider = subparsers.add_parser( 'provider_image', help='Push an image to a cloud provider.') cmd_provider.add_argument( 'target', help='The target type of the given provider') cmd_provider.add_argument( 'provider', type=argparse.FileType(), help='A file containing the provider description.') cmd_provider.add_argument( 'credentials', type=argparse.FileType(), help='A file containing the provider credentials') provider_group = cmd_provider.add_mutually_exclusive_group( required=True) provider_group.add_argument( '--id', help='The uuid of the TargetImage to push.') provider_group.add_argument('--template', type=argparse.FileType(), help=template_help) cmd_provider.add_argument('--parameters') cmd_list = subparsers.add_parser( 'images', help='List images of a given type or get details of an image.') cmd_list.add_argument( 'fetch_spec', help='JSON formatted string of key/value pairs') cmd_delete = subparsers.add_parser('delete', help='Delete an image.') cmd_delete.add_argument('id', help='UUID of the image to delete') cmd_delete.add_argument( '--provider', type=argparse.FileType(), help='A file containing the provider description.') cmd_delete.add_argument( '--credentials', type=argparse.FileType(), help='A file containing the provider credentials') cmd_delete.add_argument( '--target', help= 'The name of the target cloud for which to customize the image.' ) cmd_delete.add_argument('--parameters') cmd_plugins = subparsers.add_parser( 'plugins', help='List active plugins or get details of a specific plugin.' ) cmd_plugins.add_argument('--id') return argparser def __parse_arguments(self): argparser = self.__new_argument_parser(sys.argv[0].rpartition('/')[2]) if (len(sys.argv) == 1): argparser.print_help() sys.exit() configuration = argparser.parse_args() if (os.path.isfile(configuration.config)): try: def dencode(a_dict, encoding='ascii'): new_dict = {} for k, v in a_dict.items(): ek = k.encode(encoding) if (isinstance(v, unicode)): new_dict[ek] = v.encode(encoding) elif (isinstance(v, dict)): new_dict[ek] = dencode(v) else: new_dict[ek] = v return new_dict config_file = open(configuration.config) uconfig = json.load(config_file) config_file.close() defaults = dencode(uconfig) argparser.set_defaults(**defaults) configuration = argparser.parse_args() except Exception, e: self.log.exception(e) return configuration.__dict__
class Template(object): uuid_pattern = '([0-9a-f]{8})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{12})' identifier = props.prop("_identifier", "The identifier property.") url = props.prop("_url", "The url property.") xml = props.prop("_xml", "The xml property.") @property def name(self): """The property name""" return self._content_at_path('/template/name') @property def os_name(self): """The property os_name""" return self._content_at_path('/template/os/name') @property def os_version(self): """The property os_version""" return self._content_at_path('/template/os/version') @property def os_arch(self): """The property os_arch""" return self._content_at_path('/template/os/arch') @property def install_type(self): """The type of install ('url' or 'iso')""" result = libxml2.parseDoc( self.xml).xpathEval('/template/os/install')[0] if result: return result.prop('type') else: return None @property def install_url(self): """OS install URL""" return self._content_at_path('/template/os/install/url') @property def install_iso(self): """OS install ISO""" return self._content_at_path('/template/os/install/iso') @property def install_location(self): """Either OS install URL or ISO""" return self._content_at_path('/template/os/install/%s' % self.install_type) def __repr__(self): if (self.xml): return self.xml else: return super(Template, self).__repr__ def _content_at_path(self, path): try: return libxml2.parseDoc(self.xml).xpathEval(path)[0].content except Exception as e: self.log.exception('Could not parse document for path (%s):\n%s' % (path, e)) return None def __init__(self, template=None, uuid=None, url=None, xml=None): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.identifier = None self.url = None self.xml = None path = None if (template): template_string = str(template) template_string_type = self.__template_string_type(template_string) if (template_string_type == "UUID"): uuid = template_string elif (template_string_type == "URL"): url = template_string elif (template_string_type == "XML"): xml = template_string elif (template_string_type == "PATH"): path = template_string if (uuid): uuid_string = uuid self.identifier, self.xml = self.__fetch_template_for_uuid( uuid_string) if ((not self.identifier) and (not self.xml)): raise RuntimeError( "Could not create a template with the uuid %s" % (uuid, )) elif (url): self.url = url self.identifier, self.xml = self.__fetch_template_with_url(url) elif (xml): self.xml = xml elif (path): template_file = open(path, "r") file_content = template_file.read() template_file.close() if (self.__string_is_xml_template(file_content)): self.xml = file_content else: raise ValueError( "File %s does not contain properly formatted template xml:\n%s" % (path, self.__abbreviated_template(file_content))) else: raise ValueError( "'template' must be a UUID, URL, XML string or XML document path..." ) def __template_string_type(self, template_string): regex = re.compile(Template.uuid_pattern) match = regex.search(template_string) if (template_string.lower().startswith("http")): return "URL" elif (("<template" in template_string.lower()) and ("</template>" in template_string.lower())): return "XML" elif (match): return "UUID" elif (os.path.exists(template_string)): return "PATH" else: raise ValueError( "'template_string' must be a UUID, URL, or XML document...\n--- TEMPLATE STRING ---\n%s\n-----------------" % (template_string, )) def __string_is_xml_template(self, text): return (("<template" in text.lower()) and ("</template>" in text.lower())) def __abbreviated_template(self, template_string): lines = template_string.splitlines(True) if (len(lines) > 20): return "%s\n...\n...\n...\n%s" % ("".join(lines[0:10]), "".join( lines[-10:len(lines)])) else: return template_string
class ImageWarehouse(object): url = props.prop("_url", "The url property.") image_bucket = props.prop("_image_bucket", "The image_bucket property.") build_bucket = props.prop("_build_bucket", "The build_bucket property.") target_image_bucket = props.prop("_target_image_bucket", "The target_image_bucket property.") template_bucket = props.prop("_template_bucket", "The template_bucket property.") icicle_bucket = props.prop("_icicle_bucket", "The icicle_bucket property.") provider_image_bucket = props.prop("_provider_image_bucket", "The provider_image_bucket property.") def __repr__(self): return "%s - %r" % (super(ImageWarehouse, self).__repr__(), self.__dict__) def __str__(self): return "%s - buckets(%s, %s, %s, %s)" % ( self.url, self.target_image_bucket, self.template_bucket, self.icicle_bucket, self.provider_image_bucket) def __init__(self, url=None): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.http = httplib2.Http() if (not url): url = ApplicationConfiguration().configuration['warehouse'] self.log.debug( "Property (url) not specified. Pulling from application configuration: %s" % (url, )) self.image_bucket = ApplicationConfiguration( ).configuration['image_bucket'] self.build_bucket = ApplicationConfiguration( ).configuration['build_bucket'] self.target_image_bucket = ApplicationConfiguration( ).configuration['target_bucket'] self.template_bucket = ApplicationConfiguration( ).configuration['template_bucket'] self.icicle_bucket = ApplicationConfiguration( ).configuration['icicle_bucket'] self.provider_image_bucket = ApplicationConfiguration( ).configuration['provider_bucket'] if (url.endswith('/')): self.url = url[0:len(url) - 1] else: self.url = url self.log.debug("Created Image Warehouse instance %s" % (self, )) def _http_request(self, url, method, body=None, content_type='text/plain'): try: return self.http.request(url, method, body, headers={'content-type': content_type}) except Exception, e: raise WarehouseError( "Problem encountered trying to reach image warehouse. Please check that iwhd is running and reachable.\nException text: %s" % (e, ))
class BuildJob(object): template = props.prop("_template", "The template property.") target = props.prop("_target", "The target property.") image_id = props.prop("_image_id", "The UUID of the image.") build_id = props.prop("_build_id", "The UUID of the build.") provider = props.prop("_provider", "The provider being pushed to.") status = props.prop("_status", "The status property.") percent_complete = props.prop("_percent_complete", "The percent_complete property.") new_image_id = props.prop("_new_image_id" "The image property.") def __init__(self, template, target, image_id='', build_id=''): super(BuildJob, self).__init__() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.template = template if isinstance( template, Template) else Template(template) self.target = target self.image_id = image_id self.build_id = build_id self.provider = None self.status = "New" self.percent_complete = 0 self._watcher = None self._builder = self._get_builder() self._builder.delegate = self self.new_image_id = self._builder.new_image_id def build_image(self, watcher=None): self._watcher = watcher kwargs = dict(build_id=self.build_id) self._start_builder_thread("build_image", arg_dict=kwargs) def push_image(self, target_image_id, provider, credentials, watcher=None): self._watcher = watcher self.provider = provider kwargs = dict(target_image_id=target_image_id, provider=provider, credentials=credentials) self._start_builder_thread("push_image", arg_dict=kwargs) def abort(self): self._builder.abort() def builder_did_update_status(self, builder, old_status, new_status): self.status = new_status if self.status == "COMPLETED" and self._watcher: self._watcher.completed() self._watcher = None def builder_did_update_percentage(self, builder, original_percentage, new_percentage): self.percent_complete = new_percentage def builder_did_fail(self, builder, failure_type, failure_info): pass def _get_builder(self): builder_class = MockBuilder.MockBuilder if ( self.target != "mock" ): # If target is mock always run mock builder regardless of template os_name = self._xml_node(self.template.xml, '/template/os/name') # Change RHEL-6 to RHEL6, etc. os_name = os_name.translate(None, '-') class_name = "%sBuilder" % (os_name, ) try: module_name = "imagefactory.builders.%s" % (class_name, ) __import__(module_name) builder_class = getattr(sys.modules[module_name], class_name) except AttributeError, e: self.log.exception( "CAUGHT EXCEPTION: %s \n Could not find builder class for %s, returning MockBuilder!", e, os_name) return builder_class(self.template, self.target)
class NotificationCenter(Singleton): """ TODO: Docstring for NotificationCenter """ observers = prop("_observers") def _singleton_init(self, *args, **kwargs): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.observers = defaultdict(set) self.lock = RLock() def add_observer(self, observer, method, message='all', sender=None): """ TODO: Docstring for add_observer @param observer TODO @param method TODO @param message TODO @param sender TODO """ self.lock.acquire() self.observers[message].add((observer, method, sender)) self.lock.release() def remove_observer(self, observer, method, message='all', sender=None): """ TODO: Docstring for remove_observer @param observer TODO @param message TODO @param sender TODO """ self.lock.acquire() _observer = (observer, method, sender) self.observers[message].discard(_observer) if (len(self.observers[message]) == 0): del self.observers[message] self.lock.release() def post_notification(self, notification): """ TODO: Docstring for post_notification @param notification TODO """ self.lock.acquire() _observers = self.observers['all'].union( self.observers[notification.message]) for _observer in _observers: _sender = _observer[2] if ((not _sender) or (_sender == notification.sender)): try: getattr(_observer[0], _observer[1])(notification) except AttributeError as e: self.log.exception( 'Caught exception: posting notification to object (%s) with method (%s)' % (_observer[0], _observer[1])) self.lock.release() def post_notification_with_info(self, message, sender, user_info=None): """ TODO: Docstring for post_notification_with_info @param message TODO @param sender TODO @param user_info TODO """ self.post_notification(Notification(message, sender, user_info))
class Template(object): uuid_pattern = '([0-9a-f]{8})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{12})' identifier = props.prop("_identifier", "The identifier property.") url = props.prop("_url", "The url property.") xml = props.prop("_xml", "The xml property.") def __repr__(self): if(self.xml): return self.xml else: return super(Template, self).__repr__ def __init__(self, template=None, uuid=None, url=None, xml=None): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) self.identifier = None self.url = None self.xml = None path = None if(template): template_string = str(template) template_string_type = self.__template_string_type(template_string) if(template_string_type == "UUID"): uuid = template_string elif(template_string_type == "URL"): url = template_string elif(template_string_type == "XML"): xml = template_string elif(template_string_type == "PATH"): path = template_string if(uuid): uuid_string = uuid self.identifier, self.xml = self.__fetch_template_for_uuid(uuid_string) if((not self.identifier) and (not self.xml)): raise RuntimeError("Could not create a template with the uuid %s" % (uuid, )) elif(url): self.url = url self.identifier, self.xml = self.__fetch_template_with_url(url) elif(xml): self.xml = xml elif(path): template_file = open(path, "r") file_content = template_file.read() template_file.close() if(self.__string_is_xml_template(file_content)): self.xml = file_content else: raise ValueError("File %s does not contain properly formatted template xml:\n%s" % (path, self.__abbreviated_template(file_content))) else: raise ValueError("'template' must be a UUID, URL, XML string or XML document path...") def __template_string_type(self, template_string): regex = re.compile(Template.uuid_pattern) match = regex.search(template_string) if(template_string.lower().startswith("http")): return "URL" elif(("<template>" in template_string.lower()) and ("</template>" in template_string.lower())): return "XML" elif(match): return "UUID" elif(os.path.exists(template_string)): return "PATH" else: raise ValueError("'template_string' must be a UUID, URL, or XML document...\n--- TEMPLATE STRING ---\n%s\n-----------------" % (template_string, )) def __fetch_template_for_uuid(self, uuid_string): xml_string, metadata = self.warehouse.template_with_id(uuid_string) if(xml_string and self.__string_is_xml_template(xml_string)): return uuid_string, xml_string else: self.log.debug("Unable to fetch a valid template given template id %s:\n%s\nWill try fetching template id from a target image with this id..." % (uuid_string, self.__abbreviated_template(xml_string))) template_id, xml_string, metadata = self.warehouse.template_for_target_image_id(uuid_string) if(template_id and xml_string and self.__string_is_xml_template(xml_string)): return template_id, xml_string else: self.log.debug("Unable to fetch a valid template given a target image id %s:\n%s\n" % (uuid_string, self.__abbreviated_template(xml_string))) return None, None def __string_is_xml_template(self, text): return (("<template>" in text.lower()) and ("</template>" in text.lower())) def __fetch_template_with_url(self, url): template_id = None regex = re.compile(Template.uuid_pattern) match = regex.search(url) if (match): template_id = match.group() response_headers, response = httplib2.Http().request(url, "GET", headers={'content-type':'text/plain'}) if(response and self.__string_is_xml_template(response)): return template_id, response else: raise RuntimeError("Recieved status %s fetching a template from %s!\n--- Response Headers:\n%s\n--- Response:\n%s" % (response_headers["status"], url, response_headers, response)) def __abbreviated_template(self, template_string): lines = template_string.splitlines(True) if(len(lines) > 20): return "%s\n...\n...\n...\n%s" % ("".join(lines[0:10]), "".join(lines[-10:len(lines)])) else: return template_string
class FilePersistentImageManager(PersistentImageManager): """ TODO: Docstring for PersistentImageManager """ storage_path = prop("_storage_path") def __init__(self, storage_path=STORAGE_PATH): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) if not os.path.exists(storage_path): self.log.debug("Creating directory (%s) for persistent storage" % (storage_path)) os.makedirs(storage_path) os.chmod( storage_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) elif not os.path.isdir(storage_path): raise ImageFactoryException( "Storage location (%s) already exists and is not a directory - cannot init persistence" % (storage_path)) else: # TODO: verify that we can write to this location pass self.storage_path = storage_path self.metadata_lock = BoundedSemaphore() def _image_from_metadata(self, metadata): # Given the retrieved metadata from mongo, return a PersistentImage type object # with us as the persistent_manager. image_module = __import__(metadata['type'], globals(), locals(), [metadata['type']], -1) image_class = getattr(image_module, metadata['type']) image = image_class(metadata['identifier']) # We don't actually want a 'type' property in the resulting PersistentImage object del metadata['type'] for key in image.metadata().union(metadata.keys()): setattr(image, key, metadata.get(key)) #set ourselves as the manager image.persistent_manager = self return image def _metadata_from_file(self, metadatafile): self.metadata_lock.acquire() try: mdf = open(metadatafile, 'r') metadata = json.load(mdf) mdf.close() finally: self.metadata_lock.release() return metadata def image_with_id(self, image_id): """ TODO: Docstring for image_with_id @param image_id TODO @return TODO """ metadatafile = self.storage_path + '/' + image_id + METADATA_EXT try: metadata = self._metadata_from_file(metadatafile) except Exception as e: self.log.debug('Exception caught: %s' % e) return None return self._image_from_metadata(metadata) def images_from_query(self, query): images = [] for storefileshortname in os.listdir(self.storage_path): storefilename = self.storage_path + '/' + storefileshortname if re.search(METADATA_EXT, storefilename): try: metadata = self._metadata_from_file(storefilename) match = True for querykey in query: if metadata[querykey] != query[querykey]: match = False break if match: images.append(self._image_from_metadata(metadata)) except: self.log.warn( "Could not extract image metadata from file (%s)" % (storefilename)) return images def add_image(self, image): """ TODO: Docstring for add_image @param image TODO @return TODO """ image.persistent_manager = self basename = self.storage_path + '/' + str(image.identifier) metadata_path = basename + METADATA_EXT body_path = basename + BODY_EXT image.data = body_path try: if not os.path.isfile(metadata_path): open(metadata_path, 'w').close() self.log.debug('Created file %s' % metadata_path) if not os.path.isfile(body_path): open(body_path, 'w').close() self.log.debug('Created file %s' % body_path) except IOError as e: self.log.debug('Exception caught: %s' % e) self.save_image(image) def save_image(self, image): """ TODO: Docstring for save_image @param image TODO @return TODO """ image_id = str(image.identifier) metadata_path = self.storage_path + '/' + image_id + METADATA_EXT if not os.path.isfile(metadata_path): raise ImageFactoryException( 'Image %s not managed, use "add_image()" first.' % image_id) try: meta = {'type': type(image).__name__} for mdprop in image.metadata(): meta[mdprop] = getattr(image, mdprop, None) self.metadata_lock.acquire() try: mdf = open(metadata_path, 'w') json.dump(meta, mdf) mdf.close() finally: self.metadata_lock.release() self.log.debug("Saved metadata for image (%s): %s" % (image_id, meta)) except Exception as e: self.log.debug('Exception caught: %s' % e) raise ImageFactoryException('Unable to save image metadata: %s' % e) def delete_image_with_id(self, image_id): """ TODO: Docstring for delete_image_with_id @param image_id TODO @return TODO """ basename = self.storage_path + '/' + image_id metadata_path = basename + METADATA_EXT body_path = basename + BODY_EXT try: os.remove(metadata_path) os.remove(body_path) except Exception as e: self.log.warn('Unable to delete file: %s' % e)
class PersistentImage(object): """ TODO: Docstring for PersistentImage """ ##### PROPERTIES persistence_manager = prop("_persistence_manager") identifier = prop("_identifier") data = prop("_data") template = prop("_template") icicle = prop("_icicle") status_detail = prop("_status_detail") parameters = prop("_parameters") properties = prop("_properties") def status(): doc = "A string value." def fget(self): return self._status def fset(self, value): value = value.upper() if value == self._status: # Do not update or send a notification if nothing has changed return if value in STATUS_STRINGS: old_value = self._status self._status = value notification = Notification(message=NOTIFICATIONS[0], sender=self, user_info=dict( old_status=old_value, new_status=value)) self.notification_center.post_notification(notification) else: raise KeyError('Status (%s) unknown. Use one of %s.' % (value, STATUS_STRINGS)) return locals() status = property(**status()) def percent_complete(): doc = "The percentage through an operation." def fget(self): return self._percent_complete def fset(self, value): old_value = self._percent_complete if value == old_value: # Do not update or send a notification if nothing has changed return self._percent_complete = value notification = Notification(message=NOTIFICATIONS[1], sender=self, user_info=dict( old_percentage=old_value, new_percentage=value)) self.notification_center.post_notification(notification) return locals() percent_complete = property(**percent_complete()) ##### End PROPERTIES def __init__(self, image_id=None): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.notification_center = NotificationCenter() # We have never had use for the UUID object itself - make this a string # TODO: Root out all places where were str() convert this elsewhere self.identifier = image_id if image_id else str(uuid.uuid4()) self.persistence_manager = None self.data = None # 'activity' should be set to a single line indicating, in as much detail as reasonably possible, # what it is that the plugin operating on this image is doing at any given time. # 'error' should remain None unless an exception or other fatal error has occurred. Error may # be a multi-line string self.status_detail = { 'activity': 'Initializing image prior to Cloud/OS customization', 'error': None } # Setting these to None or setting initial value via the properties breaks the prop code above self._status = "NEW" self._percent_complete = 0 self.icicle = None self.parameters = {} self.properties = {} def update(self, percentage=None, status=None, detail=None, error=None): if percentage: self.percent_complete = percentage if status: self.status = status if detail: self.status_detail['activity'] = detail self.status_detail['error'] = error def metadata(self): self.log.debug("Executing metadata in class (%s) my metadata is (%s)" % (self.__class__, METADATA)) return METADATA
class ApplicationConfiguration(Singleton): configuration = props.prop("_configuration", "The configuration property.") def _singleton_init(self): super(ApplicationConfiguration, self)._singleton_init() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.configuration = self.__parse_arguments() def __init__(self): pass def __new_argument_parser(self): main_description = """Image Factory is an application for creating system images to run virtual machines in various public and private \ clouds. The imgfac command can be used to start a daemon providing a QMFv2 agent interface, allowing for \ remote interaction. An alternate method of running imgfac allows for one-off image building and deployment \ and does not connect to a qpidd.""" qmf_description = """Provide a QMFv2 agent interface. (see https://cwiki.apache.org/qpid/qmfv2-project-page.html for more information)""" cli_build_description = """Build specified system and exit.""" cli_push_description = """Push an image and exit.""" warehouse_description = """Options for specifying Image Warehouse (http://aeolusproject.org/imagewarehouse.html) base URL and bucket names.""" argparser = argparse.ArgumentParser(description=main_description, prog='imgfac') argparser.add_argument('--version', action='version', version='%(prog)s 0.1', help='Version info') argparser.add_argument('-v', '--verbose', action='store_true', default=False, help='Set verbose logging.') argparser.add_argument( '--debug', action='store_true', default=False, help='Set really verbose logging for debugging.') argparser.add_argument( '--image', help='UUID of iwhd image object to rebuild or push') argparser.add_argument( '--foreground', action='store_true', default=False, help= 'Stay in the foreground and avoid launching a daemon. (default: %(default)s)' ) argparser.add_argument( '--config', default='/etc/imagefactory.conf', help='Configuration file to use. (default: %(default)s)') argparser.add_argument( '--imgdir', default='/tmp', help= 'Build image files in location specified. (default: %(default)s)') argparser.add_argument( '--timeout', type=int, default=3600, help= 'Set the timeout period for image building in seconds. (default: %(default)s)' ) argparser.add_argument( '--tmpdir', default='/tmp', help= 'Use the specified location for temporary files. (default: %(default)s)' ) group_qmf = argparser.add_argument_group(title='QMF agent', description=qmf_description) group_qmf.add_argument( '--qmf', action='store_true', default=False, help='Turn on QMF agent interface. (default: %(default)s)') group_qmf.add_argument( '--qpidd', default='localhost', help='URL of qpidd to connect to. (default: %(default)s)') group_build = argparser.add_argument_group( title='Image building', description=cli_build_description) group_build.add_argument('--template', help='Template XML file to build from.') group_build.add_argument( '--target', action='append', help= 'Cloud services to target (e.g. ec2, rhevm, vsphere, rackspace, condorcloud, etc.)' ) group_push = argparser.add_argument_group( title='Image pushing', description=cli_push_description) group_push.add_argument( '--provider', action='append', help= 'Cloud service providers to push the image (e.g. ec2-us-east-1, rackspace, etc.)' ) group_push.add_argument( '--credentials', help= 'Cloud provider credentials XML (i.e. <provider_credentials/> document)' ) group_build = argparser.add_argument_group( title='Image importing', description=cli_build_description) group_build.add_argument( '--target-image', help='Target specific identifier for the image to import.') group_build.add_argument( '--image-desc', help='XML document describing the imported image.') group_warehouse = argparser.add_argument_group( title='Image Warehouse', description=warehouse_description) group_warehouse.add_argument( '--warehouse', default='http://localhost:9090/', help= 'URL of the warehouse location to store images. (default: %(default)s)' ) group_warehouse.add_argument( '--image_bucket', help= 'Name of warehouse bucket to look in images. (default: %(default)s)' ) group_warehouse.add_argument( '--build_bucket', help= 'Name of warehouse bucket to look in builds. (default: %(default)s)' ) group_warehouse.add_argument( '--target_bucket', help= 'Name of warehouse bucket to look in for target images. (default: %(default)s)' ) group_warehouse.add_argument( '--template_bucket', help= 'Name of warehouse bucket to look in for templates. (default: %(default)s)' ) group_warehouse.add_argument( '--icicle_bucket', help= 'Name of warehouse bucket to look in for icicles. (default: %(default)s)' ) group_warehouse.add_argument( '--provider_bucket', help= 'Name of warehouse bucket to look in for provider image instances. (default: %(default)s)' ) return argparser def __parse_args(self, defaults=None): if (defaults): self.argparser.set_defaults(**defaults) if (len(sys.argv) == 1): self.argparser.print_help() sys.exit() elif (sys.argv[0].endswith("imgfac.py")): return self.argparser.parse_args() elif (sys.argv[0].endswith("unittest")): return self.argparser.parse_args( '--image_bucket unittests_images --build_bucket unittests_builds --target_bucket unittests_target_images --template_bucket unittests_templates --icicle_bucket unittests_icicles --provider_bucket unittests_provider_images' .split()) else: return self.argparser.parse_args([]) def __parse_arguments(self): self.argparser = self.__new_argument_parser() configuration = self.__parse_args() if (os.path.isfile(configuration.config)): try: config_file = open(configuration.config) uconfig = json.load(config_file) # coerce this dict to ascii for python 2.6 config = {} for k, v in uconfig.items(): config[k.encode('ascii')] = v.encode('ascii') configuration = self.__parse_args(defaults=config) except IOError, e: i.log.exception(e) return configuration.__dict__
class ApplicationConfiguration(Singleton): configuration = props.prop("_configuration", "The configuration property.") def _singleton_init(self, configuration=None): super(ApplicationConfiguration, self)._singleton_init() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.jeos_images = {} if configuration: if not isinstance(configuration, dict): raise Exception( "ApplicationConfiguration configuration argument must be a dict" ) self.log.debug( "ApplicationConfiguration passed a dictionary - ignoring any local config files including JEOS configs" ) self.configuration = configuration else: self.configuration = self.__parse_arguments() self.__parse_jeos_images() if not 'debug' in self.configuration: # This most likely means we are being used as a module/library and are not running CLI or daemon self.configuration['debug'] = False if not 'secondary' in self.configuration: # We use this in the non-daemon context so it needs to be set # TODO: Something cleaner? self.configuration['secondary'] = False def __init__(self, configuration=None): pass def __new_argument_parser(self, appname): main_description = """Image Factory is an application for creating system images for use on public and private clouds.""" argparser = argparse.ArgumentParser(description=main_description, prog=appname) argparser.add_argument('--version', action='version', default=argparse.SUPPRESS, version=VERSION, help='Show the version number and exit') argparser.add_argument('--verbose', action='store_true', default=False, help='Set verbose logging.') argparser.add_argument( '--config', default='/etc/imagefactory/imagefactory.conf', help='Configuration file to use. (default: %(default)s)') argparser.add_argument( '--imgdir', default='/tmp', help= 'Build image files in location specified. (default: %(default)s)') argparser.add_argument( '--timeout', type=int, default=3600, help= 'Set the timeout period for image building in seconds. (default: %(default)s)' ) argparser.add_argument( '--tmpdir', default='/tmp', help= 'Use the specified location for temporary files. (default: %(default)s)' ) argparser.add_argument('--plugins', default='/etc/imagefactory/plugins.d', help='Plugin directory. (default: %(default)s)') group_ec2 = argparser.add_argument_group(title='EC2 settings') group_ec2.add_argument( '--ec2-32bit-util', default='m1.small', help='Instance type to use when launching a 32 bit utility instance' ) group_ec2.add_argument( '--ec2-64bit-util', default='m1.large', help='Instance type to use when launching a 64 bit utility instance' ) if (appname == 'imagefactoryd'): debug_group = argparser.add_mutually_exclusive_group() debug_group.add_argument( '--debug', action='store_true', default=False, help='Set really verbose logging for debugging.') debug_group.add_argument( '--nodebug', dest='debug', action='store_false', help='Turn off the default verbose CLI logging') argparser.add_argument( '--foreground', action='store_true', default=False, help= 'Stay in the foreground and avoid launching a daemon. (default: %(default)s)' ) group_rest = argparser.add_argument_group( title='REST service options') group_rest.add_argument( '--port', type=int, default=8075, help= 'Port to attach the RESTful http interface to. (default: %(default)s)' ) group_rest.add_argument( '--address', default='0.0.0.0', help='Interface address to listen to. (default: %(default)s)') group_rest.add_argument( '--no_ssl', action='store_true', default=False, help='Turn off SSL. (default: %(default)s)') group_rest.add_argument( '--ssl_pem', default='*', help= 'PEM certificate file to use for HTTPS access to the REST interface. (default: A transient certificate is generated at runtime.)' ) group_rest.add_argument( '--no_oauth', action='store_true', default=False, help= 'Use 2 legged OAuth to protect the REST interface. (default: %(default)s)' ) group_rest.add_argument( '--secondary', action='store_true', default=False, help= 'Operate as a secondary/helper factory. (default: %(default)s)' ) elif (appname == 'imagefactory'): debug_group = argparser.add_mutually_exclusive_group() debug_group.add_argument( '--debug', action='store_true', default=True, help='Set really verbose logging for debugging.') debug_group.add_argument( '--nodebug', dest='debug', action='store_false', help='Turn off the default verbose CLI logging') argparser.add_argument( '--output', choices=('log', 'json'), default='log', help='Choose between log or json output. (default: %(default)s)' ) argparser.add_argument('--raw', action='store_true', default=False, help='Turn off pretty printing.') subparsers = argparser.add_subparsers(title='commands', dest='command') template_help = 'A file containing the image template or component outline, compatible with the TDL schema (http://imgfac.org/documentation/tdl).' parameters_help = 'An optional JSON file containing additional parameters to pass to the builders.' cmd_base = subparsers.add_parser('base_image', help='Build a generic image.') cmd_base.add_argument('template', type=argparse.FileType(), help=template_help) cmd_base.add_argument('--parameters', type=argparse.FileType(), help=parameters_help) cmd_target = subparsers.add_parser( 'target_image', help='Customize an image for a given cloud.') cmd_target.add_argument( 'target', help= 'The name of the target cloud for which to customize the image.' ) target_group = cmd_target.add_mutually_exclusive_group( required=True) target_group.add_argument( '--id', help='The uuid of the BaseImage to customize.') target_group.add_argument('--template', type=argparse.FileType(), help=template_help) cmd_target.add_argument('--parameters', type=argparse.FileType(), help=parameters_help) cmd_provider = subparsers.add_parser( 'provider_image', help='Push an image to a cloud provider.') cmd_provider.add_argument( 'target', help='The target type of the given cloud provider') cmd_provider.add_argument( 'provider', help= "A file containing the cloud provider description or a string literal starting with '@' such as '@ec2-us-east-1'." ) cmd_provider.add_argument( 'credentials', type=argparse.FileType(), help='A file containing the cloud provider credentials') provider_group = cmd_provider.add_mutually_exclusive_group( required=True) provider_group.add_argument( '--id', help='The uuid of the TargetImage to push.') provider_group.add_argument('--template', type=argparse.FileType(), help=template_help) cmd_provider.add_argument('--parameters', type=argparse.FileType(), help=parameters_help) cmd_provider.add_argument( '--snapshot', action='store_true', default=False, help='Use snapshot style building. (default: %(default)s)') cmd_list = subparsers.add_parser( 'images', help='List images of a given type or get details of an image.') cmd_list.add_argument( 'fetch_spec', help='JSON formatted string of key/value pairs') cmd_delete = subparsers.add_parser('delete', help='Delete an image.') cmd_delete.add_argument('id', help='UUID of the image to delete') cmd_delete.add_argument( '--provider', help= "A file containing the cloud provider description or a string literal starting with '@' such as '@ec2-us-east-1'." ) cmd_delete.add_argument( '--credentials', type=argparse.FileType(), help='A file containing the cloud provider credentials') cmd_delete.add_argument( '--target', help= 'The name of the target cloud for which to customize the image.' ) cmd_delete.add_argument('--parameters', type=argparse.FileType(), help=parameters_help) cmd_plugins = subparsers.add_parser( 'plugins', help='List active plugins or get details of a specific plugin.' ) cmd_plugins.add_argument('--id') return argparser def __parse_arguments(self): appname = sys.argv[0].rpartition('/')[2] argparser = self.__new_argument_parser(appname) if ((appname == 'imagefactory') and (len(sys.argv) == 1)): argparser.print_help() sys.exit() configuration = argparser.parse_args() if (os.path.isfile(configuration.config)): try: def dencode(a_dict, encoding='ascii'): new_dict = {} for k, v in a_dict.items(): ek = k.encode(encoding) if (isinstance(v, unicode)): new_dict[ek] = v.encode(encoding) elif (isinstance(v, dict)): new_dict[ek] = dencode(v) else: new_dict[ek] = v return new_dict config_file = open(configuration.config) uconfig = json.load(config_file) config_file.close() defaults = dencode(uconfig) argparser.set_defaults(**defaults) configuration = argparser.parse_args() except Exception, e: self.log.exception(e) return configuration.__dict__
class MongoPersistentImageManager(PersistentImageManager): """ TODO: Docstring for PersistentImageManager """ storage_path = prop("_storage_path") def __init__(self, storage_path=STORAGE_PATH): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) if not os.path.exists(storage_path): self.log.debug("Creating directory (%s) for persistent storage" % (storage_path)) os.makedirs(storage_path) elif not os.path.isdir(storage_path): raise ImageFactoryException( "Storage location (%s) already exists and is not a directory - cannot init persistence" % (storage_path)) else: # TODO: verify that we can write to this location pass self.storage_path = storage_path self.con = pymongo.Connection() self.db = self.con[DB_NAME] self.collection = self.db[COLLECTION_NAME] def _to_mongo_meta(self, meta): # Take our view of the metadata and make the mongo view # Use our "identifier" as the mongo "_id" # Explicitly recommended here: http://www.mongodb.org/display/DOCS/Object+IDs # TODO: Pack UUID into BSON representation mongometa = copy(meta) mongometa['_id'] = meta['identifier'] return mongometa def _from_mongo_meta(self, mongometa): # Take mongo metadata and return the internal view meta = copy(mongometa) # This is just a duplicate del meta['_id'] return meta def _image_from_metadata(self, metadata): # Given the retrieved metadata from mongo, return a PersistentImage type object # with us as the persistent_manager. image_module = __import__(metadata['type'], globals(), locals(), [metadata['type']], -1) image_class = getattr(image_module, metadata['type']) image = image_class(metadata['identifier']) # We don't actually want a 'type' property in the resulting PersistentImage object del metadata['type'] for key in image.metadata().union(metadata.keys()): setattr(image, key, metadata.get(key)) #I don't think we want this as it will overwrite the "data" element #read from the store. #self.add_image(image) #just set ourselves as the manager image.persistent_manager = self return image def image_with_id(self, image_id): """ TODO: Docstring for image_with_id @param image_id TODO @return TODO """ try: metadata = self._from_mongo_meta( self.collection.find_one({"_id": image_id})) except Exception as e: self.log.debug('Exception caught: %s' % e) return None if not metadata: raise ImageFactoryException( "Unable to retrieve object metadata in Mongo for ID (%s)" % (image_id)) return self._image_from_metadata(metadata) def images_from_query(self, query): images = [] for image_meta in self.collection.find(query): if "type" in image_meta: images.append(self._image_from_metadata(image_meta)) else: self.log.warn( "Found mongo record with no 'type' key - id (%s)" % (image_meta['_id'])) return images def add_image(self, image): """ Add a PersistentImage-type object to this PersistenImageManager This should only be called with an image that has not yet been added to the store. To retrieve a previously persisted image use image_with_id() or image_query() @param image TODO @return TODO """ metadata = self.collection.find_one({"_id": image.identifier}) if metadata: raise ImageFactoryException( "Image %s already managed, use image_with_id() and save_image()" % (image.identifier)) image.persistent_manager = self basename = self.storage_path + '/' + str(image.identifier) body_path = basename + BODY_EXT image.data = body_path try: if not os.path.isfile(body_path): open(body_path, 'w').close() self.log.debug('Created file %s' % body_path) except IOError as e: self.log.debug('Exception caught: %s' % e) self._save_image(image) def save_image(self, image): """ TODO: Docstring for save_image @param image TODO @return TODO """ image_id = str(image.identifier) metadata = self._from_mongo_meta( self.collection.find_one({"_id": image_id})) if not metadata: raise ImageFactoryException( 'Image %s not managed, use "add_image()" first.' % image_id) self._save_image(image) def _save_image(self, image): try: meta = {'type': type(image).__name__} for mdprop in image.metadata(): meta[mdprop] = getattr(image, mdprop, None) # Set upsert to true - allows this function to do the initial insert for add_image # We check existence for save_image() already self.collection.update({'_id': image.identifier}, self._to_mongo_meta(meta), upsert=True) self.log.debug("Saved metadata for image (%s): %s" % (image.identifier, meta)) except Exception as e: self.log.debug('Exception caught: %s' % e) raise ImageFactoryException('Unable to save image metadata: %s' % e) def delete_image_with_id(self, image_id): """ TODO: Docstring for delete_image_with_id @param image_id TODO @return TODO """ basename = self.storage_path + '/' + image_id body_path = basename + BODY_EXT try: os.remove(body_path) except Exception as e: self.log.warn('Unable to delete file: %s' % e) try: self.collection.remove(image_id) except Exception as e: self.log.warn('Unable to remove mongo record: %s' % e)