class TargetImage(PersistentImage):
""" TODO: Docstring for TargetImage """
base_image_id = prop("_base_image_id")
target = prop("_target")
parameters = prop("_parameters")
def __init__(self, image_id=None):
super(TargetImage, self).__init__(image_id)
self.base_image_id = None
self.target = None
def metadata(self):
self.log.debug("Executing metadata in class (%s) my metadata is (%s)" %
(self.__class__, METADATA))
return frozenset(METADATA + super(self.__class__, self).metadata())
class Notification(object):
""" TODO: Docstring for Notification """
message = prop("_message")
sender = prop("_sender")
user_info = prop("_user_info")
def __init__(self, message, sender, user_info=None):
""" TODO: Fill me in
@param message TODO
@param sender TODO
@param user_info TODO
"""
self._message = message
self._sender = sender
self._user_info = user_info
class ProviderImage(PersistentImage):
""" TODO: Docstring for ProviderImage """
target_image_id = prop("_target_image_id")
provider = prop("_provider")
identifier_on_provider = prop("_identifier_on_provider")
provider_account_identifier = prop("_provider_account_identifier")
credentials = prop("_credentials")
parameters = prop("_parameters")
def __init__(self, image_id=None):
""" TODO: Fill me in
@param template TODO
@param target_img_id TODO
"""
super(ProviderImage, self).__init__(image_id)
self.target_image_id = None
self.provider = None
self.credentials = None
self.parameters = None
def metadata(self):
self.log.debug("Executing metadata in class (%s) my metadata is (%s)" % (self.__class__, METADATA))
return frozenset(METADATA + super(self.__class__, self).metadata())
class ApplicationConfiguration(Singleton):
configuration = props.prop("_configuration", "The configuration property.")
def _singleton_init(self):
super(ApplicationConfiguration, self)._singleton_init()
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.configuration = self.__parse_arguments()
if not 'debug' in self.configuration:
# Slightly confusing, I know - For daemon mode we have a debug argument with default False
# For cli, we debug by default and have a nodebug argument with default False
# Rest of the code assumes a 'debug' value in app_config so set it here
self.configuration['debug'] = not self.configuration['nodebug']
self.jeos_images = {}
self.__parse_jeos_images()
def __init__(self):
pass
def __new_argument_parser(self, appname):
main_description = """Image Factory is an application for creating system images for use on public and private clouds."""
argparser = argparse.ArgumentParser(description=main_description,
prog=appname,
version=VERSION)
argparser.add_argument('--verbose',
action='store_true',
default=False,
help='Set verbose logging.')
argparser.add_argument(
'--config',
default='/etc/imagefactory/imagefactory.conf',
help='Configuration file to use. (default: %(default)s)')
argparser.add_argument(
'--imgdir',
default='/tmp',
help=
'Build image files in location specified. (default: %(default)s)')
argparser.add_argument(
'--timeout',
type=int,
default=3600,
help=
'Set the timeout period for image building in seconds. (default: %(default)s)'
)
argparser.add_argument(
'--tmpdir',
default='/tmp',
help=
'Use the specified location for temporary files. (default: %(default)s)'
)
argparser.add_argument('--plugins',
default='/etc/imagefactory/plugins.d',
help='Plugin directory. (default: %(default)s)')
group_ec2 = argparser.add_argument_group(title='EC2 settings')
group_ec2.add_argument(
'--ec2-32bit-util',
default='m1.small',
help='Instance type to use when launching a 32 bit utility instance'
)
group_ec2.add_argument(
'--ec2-64bit-util',
default='m1.large',
help='Instance type to use when launching a 64 bit utility instance'
)
if (appname == 'imagefactoryd'):
argparser.add_argument(
'--debug',
action='store_true',
default=False,
help='Set really verbose logging for debugging.')
argparser.add_argument(
'--foreground',
action='store_true',
default=False,
help=
'Stay in the foreground and avoid launching a daemon. (default: %(default)s)'
)
group_rest = argparser.add_argument_group(
title='REST service options')
group_rest.add_argument(
'--port',
type=int,
default=8075,
help=
'Port to attach the RESTful http interface to. (defaul: %(default)s)'
)
group_rest.add_argument(
'--address',
default='0.0.0.0',
help='Interface address to listen to. (defaul: %(default)s)')
group_rest.add_argument(
'--no_ssl',
action='store_true',
default=False,
help='Turn off SSL. (default: %(default)s)')
group_rest.add_argument(
'--ssl_pem',
default='*',
help=
'PEM certificate file to use for HTTPS access to the REST interface. (default: A transient certificate is generated at runtime.)'
)
group_rest.add_argument(
'--no_oauth',
action='store_true',
default=False,
help=
'Use 2 legged OAuth to protect the REST interface. (default: %(default)s)'
)
elif (appname == 'imagefactory'):
argparser.add_argument(
'--nodebug',
action='store_true',
default=False,
help='Turn off the default verbose CLI logging')
argparser.add_argument(
'--output',
choices=('log', 'json'),
default='log',
help='Choose between log or json output. (default: %(default)s)'
)
argparser.add_argument('--raw',
action='store_true',
default=False,
help='Turn off pretty printing.')
subparsers = argparser.add_subparsers(title='commands',
dest='command')
template_help = 'A file containing the TDL for this image.'
cmd_base = subparsers.add_parser('base_image',
help='Build a generic image.')
cmd_base.add_argument('template',
type=argparse.FileType(),
help=template_help)
cmd_base.add_argument('--paramaters')
cmd_target = subparsers.add_parser(
'target_image', help='Customize an image for a given cloud.')
cmd_target.add_argument(
'target',
help=
'The name of the target cloud for which to customize the image.'
)
target_group = cmd_target.add_mutually_exclusive_group(
required=True)
target_group.add_argument(
'--id', help='The uuid of the BaseImage to customize.')
target_group.add_argument('--template',
type=argparse.FileType(),
help=template_help)
cmd_target.add_argument('--parameters')
cmd_provider = subparsers.add_parser(
'provider_image', help='Push an image to a cloud provider.')
cmd_provider.add_argument(
'target', help='The target type of the given provider')
cmd_provider.add_argument(
'provider',
type=argparse.FileType(),
help='A file containing the provider description.')
cmd_provider.add_argument(
'credentials',
type=argparse.FileType(),
help='A file containing the provider credentials')
provider_group = cmd_provider.add_mutually_exclusive_group(
required=True)
provider_group.add_argument(
'--id', help='The uuid of the TargetImage to push.')
provider_group.add_argument('--template',
type=argparse.FileType(),
help=template_help)
cmd_provider.add_argument('--parameters')
cmd_list = subparsers.add_parser(
'images',
help='List images of a given type or get details of an image.')
cmd_list.add_argument(
'fetch_spec', help='JSON formatted string of key/value pairs')
cmd_delete = subparsers.add_parser('delete',
help='Delete an image.')
cmd_delete.add_argument('id', help='UUID of the image to delete')
cmd_delete.add_argument(
'--provider',
type=argparse.FileType(),
help='A file containing the provider description.')
cmd_delete.add_argument(
'--credentials',
type=argparse.FileType(),
help='A file containing the provider credentials')
cmd_delete.add_argument(
'--target',
help=
'The name of the target cloud for which to customize the image.'
)
cmd_delete.add_argument('--parameters')
cmd_plugins = subparsers.add_parser(
'plugins',
help='List active plugins or get details of a specific plugin.'
)
cmd_plugins.add_argument('--id')
return argparser
def __parse_arguments(self):
argparser = self.__new_argument_parser(sys.argv[0].rpartition('/')[2])
if (len(sys.argv) == 1):
argparser.print_help()
sys.exit()
configuration = argparser.parse_args()
if (os.path.isfile(configuration.config)):
try:
def dencode(a_dict, encoding='ascii'):
new_dict = {}
for k, v in a_dict.items():
ek = k.encode(encoding)
if (isinstance(v, unicode)):
new_dict[ek] = v.encode(encoding)
elif (isinstance(v, dict)):
new_dict[ek] = dencode(v)
else:
new_dict[ek] = v
return new_dict
config_file = open(configuration.config)
uconfig = json.load(config_file)
config_file.close()
defaults = dencode(uconfig)
argparser.set_defaults(**defaults)
configuration = argparser.parse_args()
except Exception, e:
self.log.exception(e)
return configuration.__dict__
class Template(object):
uuid_pattern = '([0-9a-f]{8})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{12})'
identifier = props.prop("_identifier", "The identifier property.")
url = props.prop("_url", "The url property.")
xml = props.prop("_xml", "The xml property.")
@property
def name(self):
"""The property name"""
return self._content_at_path('/template/name')
@property
def os_name(self):
"""The property os_name"""
return self._content_at_path('/template/os/name')
@property
def os_version(self):
"""The property os_version"""
return self._content_at_path('/template/os/version')
@property
def os_arch(self):
"""The property os_arch"""
return self._content_at_path('/template/os/arch')
@property
def install_type(self):
"""The type of install ('url' or 'iso')"""
result = libxml2.parseDoc(
self.xml).xpathEval('/template/os/install')[0]
if result:
return result.prop('type')
else:
return None
@property
def install_url(self):
"""OS install URL"""
return self._content_at_path('/template/os/install/url')
@property
def install_iso(self):
"""OS install ISO"""
return self._content_at_path('/template/os/install/iso')
@property
def install_location(self):
"""Either OS install URL or ISO"""
return self._content_at_path('/template/os/install/%s' %
self.install_type)
def __repr__(self):
if (self.xml):
return self.xml
else:
return super(Template, self).__repr__
def _content_at_path(self, path):
try:
return libxml2.parseDoc(self.xml).xpathEval(path)[0].content
except Exception as e:
self.log.exception('Could not parse document for path (%s):\n%s' %
(path, e))
return None
def __init__(self, template=None, uuid=None, url=None, xml=None):
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.identifier = None
self.url = None
self.xml = None
path = None
if (template):
template_string = str(template)
template_string_type = self.__template_string_type(template_string)
if (template_string_type == "UUID"):
uuid = template_string
elif (template_string_type == "URL"):
url = template_string
elif (template_string_type == "XML"):
xml = template_string
elif (template_string_type == "PATH"):
path = template_string
if (uuid):
uuid_string = uuid
self.identifier, self.xml = self.__fetch_template_for_uuid(
uuid_string)
if ((not self.identifier) and (not self.xml)):
raise RuntimeError(
"Could not create a template with the uuid %s" % (uuid, ))
elif (url):
self.url = url
self.identifier, self.xml = self.__fetch_template_with_url(url)
elif (xml):
self.xml = xml
elif (path):
template_file = open(path, "r")
file_content = template_file.read()
template_file.close()
if (self.__string_is_xml_template(file_content)):
self.xml = file_content
else:
raise ValueError(
"File %s does not contain properly formatted template xml:\n%s"
% (path, self.__abbreviated_template(file_content)))
else:
raise ValueError(
"'template' must be a UUID, URL, XML string or XML document path..."
)
def __template_string_type(self, template_string):
regex = re.compile(Template.uuid_pattern)
match = regex.search(template_string)
if (template_string.lower().startswith("http")):
return "URL"
elif (("<template" in template_string.lower())
and ("</template>" in template_string.lower())):
return "XML"
elif (match):
return "UUID"
elif (os.path.exists(template_string)):
return "PATH"
else:
raise ValueError(
"'template_string' must be a UUID, URL, or XML document...\n--- TEMPLATE STRING ---\n%s\n-----------------"
% (template_string, ))
def __string_is_xml_template(self, text):
return (("<template" in text.lower())
and ("</template>" in text.lower()))
def __abbreviated_template(self, template_string):
lines = template_string.splitlines(True)
if (len(lines) > 20):
return "%s\n...\n...\n...\n%s" % ("".join(lines[0:10]), "".join(
lines[-10:len(lines)]))
else:
return template_string
class ImageWarehouse(object):
url = props.prop("_url", "The url property.")
image_bucket = props.prop("_image_bucket", "The image_bucket property.")
build_bucket = props.prop("_build_bucket", "The build_bucket property.")
target_image_bucket = props.prop("_target_image_bucket",
"The target_image_bucket property.")
template_bucket = props.prop("_template_bucket",
"The template_bucket property.")
icicle_bucket = props.prop("_icicle_bucket", "The icicle_bucket property.")
provider_image_bucket = props.prop("_provider_image_bucket",
"The provider_image_bucket property.")
def __repr__(self):
return "%s - %r" % (super(ImageWarehouse,
self).__repr__(), self.__dict__)
def __str__(self):
return "%s - buckets(%s, %s, %s, %s)" % (
self.url, self.target_image_bucket, self.template_bucket,
self.icicle_bucket, self.provider_image_bucket)
def __init__(self, url=None):
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.http = httplib2.Http()
if (not url):
url = ApplicationConfiguration().configuration['warehouse']
self.log.debug(
"Property (url) not specified. Pulling from application configuration: %s"
% (url, ))
self.image_bucket = ApplicationConfiguration(
).configuration['image_bucket']
self.build_bucket = ApplicationConfiguration(
).configuration['build_bucket']
self.target_image_bucket = ApplicationConfiguration(
).configuration['target_bucket']
self.template_bucket = ApplicationConfiguration(
).configuration['template_bucket']
self.icicle_bucket = ApplicationConfiguration(
).configuration['icicle_bucket']
self.provider_image_bucket = ApplicationConfiguration(
).configuration['provider_bucket']
if (url.endswith('/')):
self.url = url[0:len(url) - 1]
else:
self.url = url
self.log.debug("Created Image Warehouse instance %s" % (self, ))
def _http_request(self, url, method, body=None, content_type='text/plain'):
try:
return self.http.request(url,
method,
body,
headers={'content-type': content_type})
except Exception, e:
raise WarehouseError(
"Problem encountered trying to reach image warehouse. Please check that iwhd is running and reachable.\nException text: %s"
% (e, ))
class BuildJob(object):
template = props.prop("_template", "The template property.")
target = props.prop("_target", "The target property.")
image_id = props.prop("_image_id", "The UUID of the image.")
build_id = props.prop("_build_id", "The UUID of the build.")
provider = props.prop("_provider", "The provider being pushed to.")
status = props.prop("_status", "The status property.")
percent_complete = props.prop("_percent_complete",
"The percent_complete property.")
new_image_id = props.prop("_new_image_id" "The image property.")
def __init__(self, template, target, image_id='', build_id=''):
super(BuildJob, self).__init__()
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.template = template if isinstance(
template, Template) else Template(template)
self.target = target
self.image_id = image_id
self.build_id = build_id
self.provider = None
self.status = "New"
self.percent_complete = 0
self._watcher = None
self._builder = self._get_builder()
self._builder.delegate = self
self.new_image_id = self._builder.new_image_id
def build_image(self, watcher=None):
self._watcher = watcher
kwargs = dict(build_id=self.build_id)
self._start_builder_thread("build_image", arg_dict=kwargs)
def push_image(self, target_image_id, provider, credentials, watcher=None):
self._watcher = watcher
self.provider = provider
kwargs = dict(target_image_id=target_image_id,
provider=provider,
credentials=credentials)
self._start_builder_thread("push_image", arg_dict=kwargs)
def abort(self):
self._builder.abort()
def builder_did_update_status(self, builder, old_status, new_status):
self.status = new_status
if self.status == "COMPLETED" and self._watcher:
self._watcher.completed()
self._watcher = None
def builder_did_update_percentage(self, builder, original_percentage,
new_percentage):
self.percent_complete = new_percentage
def builder_did_fail(self, builder, failure_type, failure_info):
pass
def _get_builder(self):
builder_class = MockBuilder.MockBuilder
if (
self.target != "mock"
): # If target is mock always run mock builder regardless of template
os_name = self._xml_node(self.template.xml, '/template/os/name')
# Change RHEL-6 to RHEL6, etc.
os_name = os_name.translate(None, '-')
class_name = "%sBuilder" % (os_name, )
try:
module_name = "imagefactory.builders.%s" % (class_name, )
__import__(module_name)
builder_class = getattr(sys.modules[module_name], class_name)
except AttributeError, e:
self.log.exception(
"CAUGHT EXCEPTION: %s \n Could not find builder class for %s, returning MockBuilder!",
e, os_name)
return builder_class(self.template, self.target)
class NotificationCenter(Singleton):
""" TODO: Docstring for NotificationCenter """
observers = prop("_observers")
def _singleton_init(self, *args, **kwargs):
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.observers = defaultdict(set)
self.lock = RLock()
def add_observer(self, observer, method, message='all', sender=None):
"""
TODO: Docstring for add_observer
@param observer TODO
@param method TODO
@param message TODO
@param sender TODO
"""
self.lock.acquire()
self.observers[message].add((observer, method, sender))
self.lock.release()
def remove_observer(self, observer, method, message='all', sender=None):
"""
TODO: Docstring for remove_observer
@param observer TODO
@param message TODO
@param sender TODO
"""
self.lock.acquire()
_observer = (observer, method, sender)
self.observers[message].discard(_observer)
if (len(self.observers[message]) == 0):
del self.observers[message]
self.lock.release()
def post_notification(self, notification):
"""
TODO: Docstring for post_notification
@param notification TODO
"""
self.lock.acquire()
_observers = self.observers['all'].union(
self.observers[notification.message])
for _observer in _observers:
_sender = _observer[2]
if ((not _sender) or (_sender == notification.sender)):
try:
getattr(_observer[0], _observer[1])(notification)
except AttributeError as e:
self.log.exception(
'Caught exception: posting notification to object (%s) with method (%s)'
% (_observer[0], _observer[1]))
self.lock.release()
def post_notification_with_info(self, message, sender, user_info=None):
"""
TODO: Docstring for post_notification_with_info
@param message TODO
@param sender TODO
@param user_info TODO
"""
self.post_notification(Notification(message, sender, user_info))
class Template(object):
uuid_pattern = '([0-9a-f]{8})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{12})'
identifier = props.prop("_identifier", "The identifier property.")
url = props.prop("_url", "The url property.")
xml = props.prop("_xml", "The xml property.")
def __repr__(self):
if(self.xml):
return self.xml
else:
return super(Template, self).__repr__
def __init__(self, template=None, uuid=None, url=None, xml=None):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.identifier = None
self.url = None
self.xml = None
path = None
if(template):
template_string = str(template)
template_string_type = self.__template_string_type(template_string)
if(template_string_type == "UUID"):
uuid = template_string
elif(template_string_type == "URL"):
url = template_string
elif(template_string_type == "XML"):
xml = template_string
elif(template_string_type == "PATH"):
path = template_string
if(uuid):
uuid_string = uuid
self.identifier, self.xml = self.__fetch_template_for_uuid(uuid_string)
if((not self.identifier) and (not self.xml)):
raise RuntimeError("Could not create a template with the uuid %s" % (uuid, ))
elif(url):
self.url = url
self.identifier, self.xml = self.__fetch_template_with_url(url)
elif(xml):
self.xml = xml
elif(path):
template_file = open(path, "r")
file_content = template_file.read()
template_file.close()
if(self.__string_is_xml_template(file_content)):
self.xml = file_content
else:
raise ValueError("File %s does not contain properly formatted template xml:\n%s" % (path, self.__abbreviated_template(file_content)))
else:
raise ValueError("'template' must be a UUID, URL, XML string or XML document path...")
def __template_string_type(self, template_string):
regex = re.compile(Template.uuid_pattern)
match = regex.search(template_string)
if(template_string.lower().startswith("http")):
return "URL"
elif(("<template>" in template_string.lower()) and ("</template>" in template_string.lower())):
return "XML"
elif(match):
return "UUID"
elif(os.path.exists(template_string)):
return "PATH"
else:
raise ValueError("'template_string' must be a UUID, URL, or XML document...\n--- TEMPLATE STRING ---\n%s\n-----------------" % (template_string, ))
def __fetch_template_for_uuid(self, uuid_string):
xml_string, metadata = self.warehouse.template_with_id(uuid_string)
if(xml_string and self.__string_is_xml_template(xml_string)):
return uuid_string, xml_string
else:
self.log.debug("Unable to fetch a valid template given template id %s:\n%s\nWill try fetching template id from a target image with this id..." % (uuid_string, self.__abbreviated_template(xml_string)))
template_id, xml_string, metadata = self.warehouse.template_for_target_image_id(uuid_string)
if(template_id and xml_string and self.__string_is_xml_template(xml_string)):
return template_id, xml_string
else:
self.log.debug("Unable to fetch a valid template given a target image id %s:\n%s\n" % (uuid_string, self.__abbreviated_template(xml_string)))
return None, None
def __string_is_xml_template(self, text):
return (("<template>" in text.lower()) and ("</template>" in text.lower()))
def __fetch_template_with_url(self, url):
template_id = None
regex = re.compile(Template.uuid_pattern)
match = regex.search(url)
if (match):
template_id = match.group()
response_headers, response = httplib2.Http().request(url, "GET", headers={'content-type':'text/plain'})
if(response and self.__string_is_xml_template(response)):
return template_id, response
else:
raise RuntimeError("Recieved status %s fetching a template from %s!\n--- Response Headers:\n%s\n--- Response:\n%s" % (response_headers["status"], url, response_headers, response))
def __abbreviated_template(self, template_string):
lines = template_string.splitlines(True)
if(len(lines) > 20):
return "%s\n...\n...\n...\n%s" % ("".join(lines[0:10]), "".join(lines[-10:len(lines)]))
else:
return template_string
class FilePersistentImageManager(PersistentImageManager):
""" TODO: Docstring for PersistentImageManager """
storage_path = prop("_storage_path")
def __init__(self, storage_path=STORAGE_PATH):
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
if not os.path.exists(storage_path):
self.log.debug("Creating directory (%s) for persistent storage" %
(storage_path))
os.makedirs(storage_path)
os.chmod(
storage_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH)
elif not os.path.isdir(storage_path):
raise ImageFactoryException(
"Storage location (%s) already exists and is not a directory - cannot init persistence"
% (storage_path))
else:
# TODO: verify that we can write to this location
pass
self.storage_path = storage_path
self.metadata_lock = BoundedSemaphore()
def _image_from_metadata(self, metadata):
# Given the retrieved metadata from mongo, return a PersistentImage type object
# with us as the persistent_manager.
image_module = __import__(metadata['type'], globals(), locals(),
[metadata['type']], -1)
image_class = getattr(image_module, metadata['type'])
image = image_class(metadata['identifier'])
# We don't actually want a 'type' property in the resulting PersistentImage object
del metadata['type']
for key in image.metadata().union(metadata.keys()):
setattr(image, key, metadata.get(key))
#set ourselves as the manager
image.persistent_manager = self
return image
def _metadata_from_file(self, metadatafile):
self.metadata_lock.acquire()
try:
mdf = open(metadatafile, 'r')
metadata = json.load(mdf)
mdf.close()
finally:
self.metadata_lock.release()
return metadata
def image_with_id(self, image_id):
"""
TODO: Docstring for image_with_id
@param image_id TODO
@return TODO
"""
metadatafile = self.storage_path + '/' + image_id + METADATA_EXT
try:
metadata = self._metadata_from_file(metadatafile)
except Exception as e:
self.log.debug('Exception caught: %s' % e)
return None
return self._image_from_metadata(metadata)
def images_from_query(self, query):
images = []
for storefileshortname in os.listdir(self.storage_path):
storefilename = self.storage_path + '/' + storefileshortname
if re.search(METADATA_EXT, storefilename):
try:
metadata = self._metadata_from_file(storefilename)
match = True
for querykey in query:
if metadata[querykey] != query[querykey]:
match = False
break
if match:
images.append(self._image_from_metadata(metadata))
except:
self.log.warn(
"Could not extract image metadata from file (%s)" %
(storefilename))
return images
def add_image(self, image):
"""
TODO: Docstring for add_image
@param image TODO
@return TODO
"""
image.persistent_manager = self
basename = self.storage_path + '/' + str(image.identifier)
metadata_path = basename + METADATA_EXT
body_path = basename + BODY_EXT
image.data = body_path
try:
if not os.path.isfile(metadata_path):
open(metadata_path, 'w').close()
self.log.debug('Created file %s' % metadata_path)
if not os.path.isfile(body_path):
open(body_path, 'w').close()
self.log.debug('Created file %s' % body_path)
except IOError as e:
self.log.debug('Exception caught: %s' % e)
self.save_image(image)
def save_image(self, image):
"""
TODO: Docstring for save_image
@param image TODO
@return TODO
"""
image_id = str(image.identifier)
metadata_path = self.storage_path + '/' + image_id + METADATA_EXT
if not os.path.isfile(metadata_path):
raise ImageFactoryException(
'Image %s not managed, use "add_image()" first.' % image_id)
try:
meta = {'type': type(image).__name__}
for mdprop in image.metadata():
meta[mdprop] = getattr(image, mdprop, None)
self.metadata_lock.acquire()
try:
mdf = open(metadata_path, 'w')
json.dump(meta, mdf)
mdf.close()
finally:
self.metadata_lock.release()
self.log.debug("Saved metadata for image (%s): %s" %
(image_id, meta))
except Exception as e:
self.log.debug('Exception caught: %s' % e)
raise ImageFactoryException('Unable to save image metadata: %s' %
e)
def delete_image_with_id(self, image_id):
"""
TODO: Docstring for delete_image_with_id
@param image_id TODO
@return TODO
"""
basename = self.storage_path + '/' + image_id
metadata_path = basename + METADATA_EXT
body_path = basename + BODY_EXT
try:
os.remove(metadata_path)
os.remove(body_path)
except Exception as e:
self.log.warn('Unable to delete file: %s' % e)
class PersistentImage(object):
""" TODO: Docstring for PersistentImage """
##### PROPERTIES
persistence_manager = prop("_persistence_manager")
identifier = prop("_identifier")
data = prop("_data")
template = prop("_template")
icicle = prop("_icicle")
status_detail = prop("_status_detail")
parameters = prop("_parameters")
properties = prop("_properties")
def status():
doc = "A string value."
def fget(self):
return self._status
def fset(self, value):
value = value.upper()
if value == self._status:
# Do not update or send a notification if nothing has changed
return
if value in STATUS_STRINGS:
old_value = self._status
self._status = value
notification = Notification(message=NOTIFICATIONS[0],
sender=self,
user_info=dict(
old_status=old_value,
new_status=value))
self.notification_center.post_notification(notification)
else:
raise KeyError('Status (%s) unknown. Use one of %s.' %
(value, STATUS_STRINGS))
return locals()
status = property(**status())
def percent_complete():
doc = "The percentage through an operation."
def fget(self):
return self._percent_complete
def fset(self, value):
old_value = self._percent_complete
if value == old_value:
# Do not update or send a notification if nothing has changed
return
self._percent_complete = value
notification = Notification(message=NOTIFICATIONS[1],
sender=self,
user_info=dict(
old_percentage=old_value,
new_percentage=value))
self.notification_center.post_notification(notification)
return locals()
percent_complete = property(**percent_complete())
##### End PROPERTIES
def __init__(self, image_id=None):
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.notification_center = NotificationCenter()
# We have never had use for the UUID object itself - make this a string
# TODO: Root out all places where were str() convert this elsewhere
self.identifier = image_id if image_id else str(uuid.uuid4())
self.persistence_manager = None
self.data = None
# 'activity' should be set to a single line indicating, in as much detail as reasonably possible,
# what it is that the plugin operating on this image is doing at any given time.
# 'error' should remain None unless an exception or other fatal error has occurred. Error may
# be a multi-line string
self.status_detail = {
'activity': 'Initializing image prior to Cloud/OS customization',
'error': None
}
# Setting these to None or setting initial value via the properties breaks the prop code above
self._status = "NEW"
self._percent_complete = 0
self.icicle = None
self.parameters = {}
self.properties = {}
def update(self, percentage=None, status=None, detail=None, error=None):
if percentage:
self.percent_complete = percentage
if status:
self.status = status
if detail:
self.status_detail['activity'] = detail
self.status_detail['error'] = error
def metadata(self):
self.log.debug("Executing metadata in class (%s) my metadata is (%s)" %
(self.__class__, METADATA))
return METADATA
class ApplicationConfiguration(Singleton):
configuration = props.prop("_configuration", "The configuration property.")
def _singleton_init(self):
super(ApplicationConfiguration, self)._singleton_init()
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.configuration = self.__parse_arguments()
def __init__(self):
pass
def __new_argument_parser(self):
main_description = """Image Factory is an application for creating system images to run virtual machines in various public and private \
clouds. The imgfac command can be used to start a daemon providing a QMFv2 agent interface, allowing for \
remote interaction. An alternate method of running imgfac allows for one-off image building and deployment \
and does not connect to a qpidd."""
qmf_description = """Provide a QMFv2 agent interface. (see https://cwiki.apache.org/qpid/qmfv2-project-page.html for more information)"""
cli_build_description = """Build specified system and exit."""
cli_push_description = """Push an image and exit."""
warehouse_description = """Options for specifying Image Warehouse (http://aeolusproject.org/imagewarehouse.html) base URL and bucket names."""
argparser = argparse.ArgumentParser(description=main_description,
prog='imgfac')
argparser.add_argument('--version',
action='version',
version='%(prog)s 0.1',
help='Version info')
argparser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='Set verbose logging.')
argparser.add_argument(
'--debug',
action='store_true',
default=False,
help='Set really verbose logging for debugging.')
argparser.add_argument(
'--image', help='UUID of iwhd image object to rebuild or push')
argparser.add_argument(
'--foreground',
action='store_true',
default=False,
help=
'Stay in the foreground and avoid launching a daemon. (default: %(default)s)'
)
argparser.add_argument(
'--config',
default='/etc/imagefactory.conf',
help='Configuration file to use. (default: %(default)s)')
argparser.add_argument(
'--imgdir',
default='/tmp',
help=
'Build image files in location specified. (default: %(default)s)')
argparser.add_argument(
'--timeout',
type=int,
default=3600,
help=
'Set the timeout period for image building in seconds. (default: %(default)s)'
)
argparser.add_argument(
'--tmpdir',
default='/tmp',
help=
'Use the specified location for temporary files. (default: %(default)s)'
)
group_qmf = argparser.add_argument_group(title='QMF agent',
description=qmf_description)
group_qmf.add_argument(
'--qmf',
action='store_true',
default=False,
help='Turn on QMF agent interface. (default: %(default)s)')
group_qmf.add_argument(
'--qpidd',
default='localhost',
help='URL of qpidd to connect to. (default: %(default)s)')
group_build = argparser.add_argument_group(
title='Image building', description=cli_build_description)
group_build.add_argument('--template',
help='Template XML file to build from.')
group_build.add_argument(
'--target',
action='append',
help=
'Cloud services to target (e.g. ec2, rhevm, vsphere, rackspace, condorcloud, etc.)'
)
group_push = argparser.add_argument_group(
title='Image pushing', description=cli_push_description)
group_push.add_argument(
'--provider',
action='append',
help=
'Cloud service providers to push the image (e.g. ec2-us-east-1, rackspace, etc.)'
)
group_push.add_argument(
'--credentials',
help=
'Cloud provider credentials XML (i.e. <provider_credentials/> document)'
)
group_build = argparser.add_argument_group(
title='Image importing', description=cli_build_description)
group_build.add_argument(
'--target-image',
help='Target specific identifier for the image to import.')
group_build.add_argument(
'--image-desc', help='XML document describing the imported image.')
group_warehouse = argparser.add_argument_group(
title='Image Warehouse', description=warehouse_description)
group_warehouse.add_argument(
'--warehouse',
default='http://localhost:9090/',
help=
'URL of the warehouse location to store images. (default: %(default)s)'
)
group_warehouse.add_argument(
'--image_bucket',
help=
'Name of warehouse bucket to look in images. (default: %(default)s)'
)
group_warehouse.add_argument(
'--build_bucket',
help=
'Name of warehouse bucket to look in builds. (default: %(default)s)'
)
group_warehouse.add_argument(
'--target_bucket',
help=
'Name of warehouse bucket to look in for target images. (default: %(default)s)'
)
group_warehouse.add_argument(
'--template_bucket',
help=
'Name of warehouse bucket to look in for templates. (default: %(default)s)'
)
group_warehouse.add_argument(
'--icicle_bucket',
help=
'Name of warehouse bucket to look in for icicles. (default: %(default)s)'
)
group_warehouse.add_argument(
'--provider_bucket',
help=
'Name of warehouse bucket to look in for provider image instances. (default: %(default)s)'
)
return argparser
def __parse_args(self, defaults=None):
if (defaults):
self.argparser.set_defaults(**defaults)
if (len(sys.argv) == 1):
self.argparser.print_help()
sys.exit()
elif (sys.argv[0].endswith("imgfac.py")):
return self.argparser.parse_args()
elif (sys.argv[0].endswith("unittest")):
return self.argparser.parse_args(
'--image_bucket unittests_images --build_bucket unittests_builds --target_bucket unittests_target_images --template_bucket unittests_templates --icicle_bucket unittests_icicles --provider_bucket unittests_provider_images'
.split())
else:
return self.argparser.parse_args([])
def __parse_arguments(self):
self.argparser = self.__new_argument_parser()
configuration = self.__parse_args()
if (os.path.isfile(configuration.config)):
try:
config_file = open(configuration.config)
uconfig = json.load(config_file)
# coerce this dict to ascii for python 2.6
config = {}
for k, v in uconfig.items():
config[k.encode('ascii')] = v.encode('ascii')
configuration = self.__parse_args(defaults=config)
except IOError, e:
i.log.exception(e)
return configuration.__dict__
class ApplicationConfiguration(Singleton):
configuration = props.prop("_configuration", "The configuration property.")
def _singleton_init(self, configuration=None):
super(ApplicationConfiguration, self)._singleton_init()
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
self.jeos_images = {}
if configuration:
if not isinstance(configuration, dict):
raise Exception(
"ApplicationConfiguration configuration argument must be a dict"
)
self.log.debug(
"ApplicationConfiguration passed a dictionary - ignoring any local config files including JEOS configs"
)
self.configuration = configuration
else:
self.configuration = self.__parse_arguments()
self.__parse_jeos_images()
if not 'debug' in self.configuration:
# This most likely means we are being used as a module/library and are not running CLI or daemon
self.configuration['debug'] = False
if not 'secondary' in self.configuration:
# We use this in the non-daemon context so it needs to be set
# TODO: Something cleaner?
self.configuration['secondary'] = False
def __init__(self, configuration=None):
pass
def __new_argument_parser(self, appname):
main_description = """Image Factory is an application for creating system images for use on public and private clouds."""
argparser = argparse.ArgumentParser(description=main_description,
prog=appname)
argparser.add_argument('--version',
action='version',
default=argparse.SUPPRESS,
version=VERSION,
help='Show the version number and exit')
argparser.add_argument('--verbose',
action='store_true',
default=False,
help='Set verbose logging.')
argparser.add_argument(
'--config',
default='/etc/imagefactory/imagefactory.conf',
help='Configuration file to use. (default: %(default)s)')
argparser.add_argument(
'--imgdir',
default='/tmp',
help=
'Build image files in location specified. (default: %(default)s)')
argparser.add_argument(
'--timeout',
type=int,
default=3600,
help=
'Set the timeout period for image building in seconds. (default: %(default)s)'
)
argparser.add_argument(
'--tmpdir',
default='/tmp',
help=
'Use the specified location for temporary files. (default: %(default)s)'
)
argparser.add_argument('--plugins',
default='/etc/imagefactory/plugins.d',
help='Plugin directory. (default: %(default)s)')
group_ec2 = argparser.add_argument_group(title='EC2 settings')
group_ec2.add_argument(
'--ec2-32bit-util',
default='m1.small',
help='Instance type to use when launching a 32 bit utility instance'
)
group_ec2.add_argument(
'--ec2-64bit-util',
default='m1.large',
help='Instance type to use when launching a 64 bit utility instance'
)
if (appname == 'imagefactoryd'):
debug_group = argparser.add_mutually_exclusive_group()
debug_group.add_argument(
'--debug',
action='store_true',
default=False,
help='Set really verbose logging for debugging.')
debug_group.add_argument(
'--nodebug',
dest='debug',
action='store_false',
help='Turn off the default verbose CLI logging')
argparser.add_argument(
'--foreground',
action='store_true',
default=False,
help=
'Stay in the foreground and avoid launching a daemon. (default: %(default)s)'
)
group_rest = argparser.add_argument_group(
title='REST service options')
group_rest.add_argument(
'--port',
type=int,
default=8075,
help=
'Port to attach the RESTful http interface to. (default: %(default)s)'
)
group_rest.add_argument(
'--address',
default='0.0.0.0',
help='Interface address to listen to. (default: %(default)s)')
group_rest.add_argument(
'--no_ssl',
action='store_true',
default=False,
help='Turn off SSL. (default: %(default)s)')
group_rest.add_argument(
'--ssl_pem',
default='*',
help=
'PEM certificate file to use for HTTPS access to the REST interface. (default: A transient certificate is generated at runtime.)'
)
group_rest.add_argument(
'--no_oauth',
action='store_true',
default=False,
help=
'Use 2 legged OAuth to protect the REST interface. (default: %(default)s)'
)
group_rest.add_argument(
'--secondary',
action='store_true',
default=False,
help=
'Operate as a secondary/helper factory. (default: %(default)s)'
)
elif (appname == 'imagefactory'):
debug_group = argparser.add_mutually_exclusive_group()
debug_group.add_argument(
'--debug',
action='store_true',
default=True,
help='Set really verbose logging for debugging.')
debug_group.add_argument(
'--nodebug',
dest='debug',
action='store_false',
help='Turn off the default verbose CLI logging')
argparser.add_argument(
'--output',
choices=('log', 'json'),
default='log',
help='Choose between log or json output. (default: %(default)s)'
)
argparser.add_argument('--raw',
action='store_true',
default=False,
help='Turn off pretty printing.')
subparsers = argparser.add_subparsers(title='commands',
dest='command')
template_help = 'A file containing the image template or component outline, compatible with the TDL schema (http://imgfac.org/documentation/tdl).'
parameters_help = 'An optional JSON file containing additional parameters to pass to the builders.'
cmd_base = subparsers.add_parser('base_image',
help='Build a generic image.')
cmd_base.add_argument('template',
type=argparse.FileType(),
help=template_help)
cmd_base.add_argument('--parameters',
type=argparse.FileType(),
help=parameters_help)
cmd_target = subparsers.add_parser(
'target_image', help='Customize an image for a given cloud.')
cmd_target.add_argument(
'target',
help=
'The name of the target cloud for which to customize the image.'
)
target_group = cmd_target.add_mutually_exclusive_group(
required=True)
target_group.add_argument(
'--id', help='The uuid of the BaseImage to customize.')
target_group.add_argument('--template',
type=argparse.FileType(),
help=template_help)
cmd_target.add_argument('--parameters',
type=argparse.FileType(),
help=parameters_help)
cmd_provider = subparsers.add_parser(
'provider_image', help='Push an image to a cloud provider.')
cmd_provider.add_argument(
'target', help='The target type of the given cloud provider')
cmd_provider.add_argument(
'provider',
help=
"A file containing the cloud provider description or a string literal starting with '@' such as '@ec2-us-east-1'."
)
cmd_provider.add_argument(
'credentials',
type=argparse.FileType(),
help='A file containing the cloud provider credentials')
provider_group = cmd_provider.add_mutually_exclusive_group(
required=True)
provider_group.add_argument(
'--id', help='The uuid of the TargetImage to push.')
provider_group.add_argument('--template',
type=argparse.FileType(),
help=template_help)
cmd_provider.add_argument('--parameters',
type=argparse.FileType(),
help=parameters_help)
cmd_provider.add_argument(
'--snapshot',
action='store_true',
default=False,
help='Use snapshot style building. (default: %(default)s)')
cmd_list = subparsers.add_parser(
'images',
help='List images of a given type or get details of an image.')
cmd_list.add_argument(
'fetch_spec', help='JSON formatted string of key/value pairs')
cmd_delete = subparsers.add_parser('delete',
help='Delete an image.')
cmd_delete.add_argument('id', help='UUID of the image to delete')
cmd_delete.add_argument(
'--provider',
help=
"A file containing the cloud provider description or a string literal starting with '@' such as '@ec2-us-east-1'."
)
cmd_delete.add_argument(
'--credentials',
type=argparse.FileType(),
help='A file containing the cloud provider credentials')
cmd_delete.add_argument(
'--target',
help=
'The name of the target cloud for which to customize the image.'
)
cmd_delete.add_argument('--parameters',
type=argparse.FileType(),
help=parameters_help)
cmd_plugins = subparsers.add_parser(
'plugins',
help='List active plugins or get details of a specific plugin.'
)
cmd_plugins.add_argument('--id')
return argparser
def __parse_arguments(self):
appname = sys.argv[0].rpartition('/')[2]
argparser = self.__new_argument_parser(appname)
if ((appname == 'imagefactory') and (len(sys.argv) == 1)):
argparser.print_help()
sys.exit()
configuration = argparser.parse_args()
if (os.path.isfile(configuration.config)):
try:
def dencode(a_dict, encoding='ascii'):
new_dict = {}
for k, v in a_dict.items():
ek = k.encode(encoding)
if (isinstance(v, unicode)):
new_dict[ek] = v.encode(encoding)
elif (isinstance(v, dict)):
new_dict[ek] = dencode(v)
else:
new_dict[ek] = v
return new_dict
config_file = open(configuration.config)
uconfig = json.load(config_file)
config_file.close()
defaults = dencode(uconfig)
argparser.set_defaults(**defaults)
configuration = argparser.parse_args()
except Exception, e:
self.log.exception(e)
return configuration.__dict__
class MongoPersistentImageManager(PersistentImageManager):
""" TODO: Docstring for PersistentImageManager """
storage_path = prop("_storage_path")
def __init__(self, storage_path=STORAGE_PATH):
self.log = logging.getLogger('%s.%s' %
(__name__, self.__class__.__name__))
if not os.path.exists(storage_path):
self.log.debug("Creating directory (%s) for persistent storage" %
(storage_path))
os.makedirs(storage_path)
elif not os.path.isdir(storage_path):
raise ImageFactoryException(
"Storage location (%s) already exists and is not a directory - cannot init persistence"
% (storage_path))
else:
# TODO: verify that we can write to this location
pass
self.storage_path = storage_path
self.con = pymongo.Connection()
self.db = self.con[DB_NAME]
self.collection = self.db[COLLECTION_NAME]
def _to_mongo_meta(self, meta):
# Take our view of the metadata and make the mongo view
# Use our "identifier" as the mongo "_id"
# Explicitly recommended here: http://www.mongodb.org/display/DOCS/Object+IDs
# TODO: Pack UUID into BSON representation
mongometa = copy(meta)
mongometa['_id'] = meta['identifier']
return mongometa
def _from_mongo_meta(self, mongometa):
# Take mongo metadata and return the internal view
meta = copy(mongometa)
# This is just a duplicate
del meta['_id']
return meta
def _image_from_metadata(self, metadata):
# Given the retrieved metadata from mongo, return a PersistentImage type object
# with us as the persistent_manager.
image_module = __import__(metadata['type'], globals(), locals(),
[metadata['type']], -1)
image_class = getattr(image_module, metadata['type'])
image = image_class(metadata['identifier'])
# We don't actually want a 'type' property in the resulting PersistentImage object
del metadata['type']
for key in image.metadata().union(metadata.keys()):
setattr(image, key, metadata.get(key))
#I don't think we want this as it will overwrite the "data" element
#read from the store.
#self.add_image(image)
#just set ourselves as the manager
image.persistent_manager = self
return image
def image_with_id(self, image_id):
"""
TODO: Docstring for image_with_id
@param image_id TODO
@return TODO
"""
try:
metadata = self._from_mongo_meta(
self.collection.find_one({"_id": image_id}))
except Exception as e:
self.log.debug('Exception caught: %s' % e)
return None
if not metadata:
raise ImageFactoryException(
"Unable to retrieve object metadata in Mongo for ID (%s)" %
(image_id))
return self._image_from_metadata(metadata)
def images_from_query(self, query):
images = []
for image_meta in self.collection.find(query):
if "type" in image_meta:
images.append(self._image_from_metadata(image_meta))
else:
self.log.warn(
"Found mongo record with no 'type' key - id (%s)" %
(image_meta['_id']))
return images
def add_image(self, image):
"""
Add a PersistentImage-type object to this PersistenImageManager
This should only be called with an image that has not yet been added to the store.
To retrieve a previously persisted image use image_with_id() or image_query()
@param image TODO
@return TODO
"""
metadata = self.collection.find_one({"_id": image.identifier})
if metadata:
raise ImageFactoryException(
"Image %s already managed, use image_with_id() and save_image()"
% (image.identifier))
image.persistent_manager = self
basename = self.storage_path + '/' + str(image.identifier)
body_path = basename + BODY_EXT
image.data = body_path
try:
if not os.path.isfile(body_path):
open(body_path, 'w').close()
self.log.debug('Created file %s' % body_path)
except IOError as e:
self.log.debug('Exception caught: %s' % e)
self._save_image(image)
def save_image(self, image):
"""
TODO: Docstring for save_image
@param image TODO
@return TODO
"""
image_id = str(image.identifier)
metadata = self._from_mongo_meta(
self.collection.find_one({"_id": image_id}))
if not metadata:
raise ImageFactoryException(
'Image %s not managed, use "add_image()" first.' % image_id)
self._save_image(image)
def _save_image(self, image):
try:
meta = {'type': type(image).__name__}
for mdprop in image.metadata():
meta[mdprop] = getattr(image, mdprop, None)
# Set upsert to true - allows this function to do the initial insert for add_image
# We check existence for save_image() already
self.collection.update({'_id': image.identifier},
self._to_mongo_meta(meta),
upsert=True)
self.log.debug("Saved metadata for image (%s): %s" %
(image.identifier, meta))
except Exception as e:
self.log.debug('Exception caught: %s' % e)
raise ImageFactoryException('Unable to save image metadata: %s' %
e)
def delete_image_with_id(self, image_id):
"""
TODO: Docstring for delete_image_with_id
@param image_id TODO
@return TODO
"""
basename = self.storage_path + '/' + image_id
body_path = basename + BODY_EXT
try:
os.remove(body_path)
except Exception as e:
self.log.warn('Unable to delete file: %s' % e)
try:
self.collection.remove(image_id)
except Exception as e:
self.log.warn('Unable to remove mongo record: %s' % e)
