def setUp(self):
logging.basicConfig(
level=logging.NOTSET,
format=
'%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s',
filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(
ApplicationConfiguration().configuration["warehouse"])
self.template_xml = "<template>This is a test template. There is not much to it.</template>"
class testOauthIwhd(unittest2.TestCase):
def setUp(self):
logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.warehouseKey = self.warehouse.warehouse_credentials['key'] if self.warehouse.warehouse_oauth else 'warehousekey'
self.warehouseSecret = self.warehouse.warehouse_credentials['secret'] if self.warehouse.warehouse_oauth else 'warehousesecret'
def tearDown(self):
del self.warehouseKey
del self.warehouse
del self.warehouseSecret
def testOauthIwhd(self):
#querying iwhd;
#https://www.aeolusproject.org/redmine/projects/aeolus/wiki/OAuth
consumer = oauth.Consumer(key=self.warehouseKey,secret=self.warehouseSecret)
sig_method = oauth.SignatureMethod_HMAC_SHA1()
oauth_version = None
try:
oauth_version = oauth.OAUTH_VERSION
except AttributeError:
oauth_version = oauth.VERSION
params = {'oauth_version':oauth_version,
'oauth_nonce':oauth.generate_nonce(),
'oauth_timestamp':oauth.generate_timestamp(),
'oauth_signature_method':sig_method.name,
'auth_consumer_key':consumer.key}
req = oauth.Request(method='GET', url=self.warehouse.url, parameters=params)
sig = sig_method.signing_base(req, consumer, None)
#print "\n"
#print sig
req['oauth_signature'] = sig
r, c = httplib2.Http().request(self.warehouse.url, 'GET', None, headers=req.to_header())
#print '\nResponse headers: %s\nContent: %s' % (r,c)
self.assertIsNotNone(c)
self.assertIsNotNone(r)
#self.warehouse._oauth_headers(self.warehouse.url, 'GET')
#self.assert_(self.warehouse.create_bucket_at_url("%s/unittests-create_bucket/%s" % (self.warehouse.url, "unittest-koca")))
def testStoreTemplate(self):
# TEMPLATE
template_content = "<template>This is a test template.</template>"
# store the template and let an id get assigned
template_id = self.warehouse.store_template(template_content)
self.assertIsNotNone(template_id)
def __init__(self, template=None, uuid=None, url=None, xml=None):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.identifier = None
self.url = None
self.xml = None
path = None
if(template):
template_string = str(template)
template_string_type = self.__template_string_type(template_string)
if(template_string_type == "UUID"):
uuid = template_string
elif(template_string_type == "URL"):
url = template_string
elif(template_string_type == "XML"):
xml = template_string
elif(template_string_type == "PATH"):
path = template_string
if(uuid):
uuid_string = uuid
self.identifier, self.xml = self.__fetch_template_for_uuid(uuid_string)
if((not self.identifier) and (not self.xml)):
raise RuntimeError("Could not create a template with the uuid %s" % (uuid, ))
elif(url):
self.url = url
self.identifier, self.xml = self.__fetch_template_with_url(url)
elif(xml):
self.xml = xml
elif(path):
template_file = open(path, "r")
file_content = template_file.read()
template_file.close()
if(self.__string_is_xml_template(file_content)):
self.xml = file_content
else:
raise ValueError("File %s does not contain properly formatted template xml:\n%s" % (path, self.__abbreviated_template(file_content)))
else:
raise ValueError("'template' must be a UUID, URL, XML string or XML document path...")
def setUp(self):
logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.metadata = dict(key1="value1", key2="value2", key3="value3")
class testImageWarehouse(unittest.TestCase):
def setUp(self):
logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.metadata = dict(key1="value1", key2="value2", key3="value3")
def tearDown(self):
del self.warehouse
del self.metadata
def testImageWarehouseMethods(self):
"""Tests CRUD operations on target images, templates, icicles, and metadata on those objects..."""
# TARGET IMAGE
# store a target image
target_image_id = str(uuid.uuid4())
file_content = "This is just to test storing a target image in warehouse. There is not much to see here."
file_path = "/tmp/testImageWarehouse_testStoreAndFetchTargetImage.%s" % (target_image_id, )
with open(file_path, 'w') as test_file:
test_file.write(file_content)
test_file.close()
self.warehouse.store_target_image(target_image_id, file_path, metadata=self.metadata)
# now fetch that target image
target_image, metadata = self.warehouse.target_image_with_id(target_image_id, metadata_keys=self.metadata.keys())
# now make the assertions
self.assertEqual(file_content, target_image)
self.assertEqual(self.metadata, metadata)
# TEMPLATE
template_content = "<template>This is a test template. There is not much to see here.</template>"
# store the template and let an id get assigned
template_id = self.warehouse.store_template(template_content)
self.assertTrue(template_id)
# store the template with a specified id
template_id_known = str(uuid.uuid4())
template_id2 = self.warehouse.store_template(template_content, template_id_known)
self.assertEqual(template_id_known, template_id2)
# fetch the templates
fetched_template_content, template_metadata1 = self.warehouse.template_with_id(template_id)
self.assertEqual(template_content, fetched_template_content)
fetched_template_content2, template_metadata2 = self.warehouse.template_with_id(template_id_known)
self.assertEqual(template_content, fetched_template_content2)
# set the template id for a target image and fetch it back
self.warehouse.set_metadata_for_id_of_type(dict(template=template_id), target_image_id, "target_image")
template_id3, fetched_template_content3, template_metadata3 = self.warehouse.template_for_target_image_id(target_image_id)
self.assertEqual(template_id, template_id3)
self.assertEqual(template_content, fetched_template_content3)
# ICICLE
icicle_content = "<icicle>This is a test icicle. There is not much to see here.</icicle>"
# store the icicle and let an id get assigned
icicle_id = self.warehouse.store_icicle(icicle_content)
self.assertTrue(icicle_id)
# store the icicle with a specified id
icicle_id_known = str(uuid.uuid4())
icicle_id2 = self.warehouse.store_icicle(icicle_content, icicle_id_known)
self.assertEqual(icicle_id_known, icicle_id2)
# fetch the icicles
fetched_icicle_content, icicle_metadata1 = self.warehouse.icicle_with_id(icicle_id)
self.assertEqual(icicle_content, fetched_icicle_content)
fetched_icicle_content2, icicle_metadata2 = self.warehouse.icicle_with_id(icicle_id_known)
self.assertEqual(icicle_content, fetched_icicle_content2)
# set the icicle id for a target image and fetch it back
self.warehouse.set_metadata_for_id_of_type(dict(icicle=icicle_id), target_image_id, "target_image")
icicle_id3, fetched_icicle_content3, icicle_metadata3 = self.warehouse.icicle_for_target_image_id(target_image_id)
self.assertEqual(icicle_id, icicle_id3)
self.assertEqual(icicle_content, fetched_icicle_content3)
self.assertTrue(self.warehouse.remove_target_image_with_id(target_image_id))
self.assertTrue(self.warehouse.remove_template_with_id(template_id))
self.assertTrue(self.warehouse.remove_template_with_id(template_id2))
self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id))
self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id2))
os.remove(file_path)
def testImageAndBuildMethods(self):
"""Tests CRUD operations on images, builds and metadata on those objects..."""
image_xml = '<image/>'
image_id = self.warehouse.store_image(None, image_xml, self.metadata)
self.assertTrue(image_id)
image_body, metadata = self.warehouse.object_with_id_of_type(image_id, 'image', self.metadata.keys())
self.assertEqual(image_xml, image_body)
self.assertEqual(self.metadata, metadata)
build_id = self.warehouse.store_build(None, self.metadata)
build_body, metadata = self.warehouse.object_with_id_of_type(build_id, 'build', self.metadata.keys())
self.assertEqual('', build_body)
self.assertEqual(self.metadata, metadata)
ids = self.warehouse.query('build', '$object_type == "build" && $key1 == "value1"')
self.assertTrue(build_id in ids)
def testBucketCreation(self):
# self.assert_(self.warehouse.create_bucket_at_url("%s/unittests-create_bucket/%s" % (self.warehouse.url, str(uuid.uuid4()))))
self.warehouse.create_bucket_at_url("%s/unittests-create_bucket" % (self.warehouse.url, ))
self.assert_(self.warehouse.create_bucket_at_url("%s/unittests-create_bucket/%s" % (self.warehouse.url, time.asctime().replace(' ', '-'))))
def setUp(self):
logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.warehouseKey = self.warehouse.warehouse_credentials['key'] if self.warehouse.warehouse_oauth else 'warehousekey'
self.warehouseSecret = self.warehouse.warehouse_credentials['secret'] if self.warehouse.warehouse_oauth else 'warehousesecret'
class testTemplate(unittest.TestCase):
def setUp(self):
logging.basicConfig(
level=logging.NOTSET,
format=
'%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s',
filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(
ApplicationConfiguration().configuration["warehouse"])
self.template_xml = "<template>This is a test template. There is not much to it.</template>"
def tearDown(self):
del self.warehouse
del self.template_xml
def testTemplateFromUUID(self):
template_id = self.warehouse.store_template(self.template_xml)
template = Template(template_id)
self.assertEqual(template_id, template.identifier)
self.assertEqual(self.template_xml, template.xml)
self.assertFalse(template.url)
def testTemplateFromImageID(self):
template_id = self.warehouse.store_template(self.template_xml)
template = Template(template_id)
target = "mock"
builder = Mock_Builder(self.template_xml, target)
builder.build_image()
metadata = dict(template=template_id,
target=target,
icicle="None",
target_parameters="None")
self.warehouse.store_target_image(builder.new_image_id,
builder.image,
metadata=metadata)
image_template = Template(builder.new_image_id)
self.assertEqual(template_id, image_template.identifier)
self.assertEqual(self.template_xml, image_template.xml)
self.assertFalse(template.url)
def testTemplateFromXML(self):
template = Template(self.template_xml)
self.assertEqual(self.template_xml, template.xml)
self.assertFalse(template.identifier)
self.assertFalse(template.url)
def testTemplateFromURL(self):
template_id = self.warehouse.store_template(self.template_xml)
template_url = "%s/%s/%s" % (
self.warehouse.url, self.warehouse.template_bucket, template_id)
template = Template(template_url)
self.assertEqual(template_url, template.url)
self.assertEqual(template_id, template.identifier)
self.assertEqual(self.template_xml, template.xml)
def testTemplateFromPath(self):
(fd, template_path) = tempfile.mkstemp(prefix="test_image_factory-")
os.write(fd, self.template_xml)
os.close(fd)
template = Template(template_path)
self.assertFalse(template.url)
self.assertFalse(template.identifier)
self.assertEqual(self.template_xml, template.xml)
os.remove(template_path)
def testTemplateStringRepresentation(self):
template = Template(self.template_xml)
self.assertEqual(self.template_xml, repr(template))
self.assertEqual(self.template_xml, str(template))
self.assertEqual(self.template_xml, "%r" % (template, ))
self.assertEqual(self.template_xml, "%s" % (template, ))
def _singleton_init(self):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse'])
self.job_registry = JobRegistry()
class BuildDispatcher(Singleton):
def _singleton_init(self):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse'])
self.job_registry = JobRegistry()
def import_image(self, image_id, build_id, target_identifier, image_desc, target, provider):
image_id = self._ensure_image(image_id, image_desc)
build_id = self._ensure_build(image_id, build_id)
target_image_id = self._ensure_target_image(build_id, target)
provider_image_id = self._ensure_provider_image(target_image_id, provider, target_identifier)
self._commit_build(image_id, build_id)
return (image_id, build_id, target_image_id, provider_image_id)
def build_image_for_targets(self, image_id, build_id, template, targets, job_cls = BuildJob, *args, **kwargs):
if image_id and not targets:
targets = self._targets_for_image_id(image_id)
template = self._load_template(image_id, build_id, template)
image_id = self._ensure_image_with_template(image_id, template)
build_id = self._ensure_build(image_id, build_id)
watcher = BuildWatcher(image_id, build_id, len(targets), self.warehouse)
jobs = []
for target in targets:
job = job_cls(template, target, image_id, build_id, *args, **kwargs)
job.build_image(watcher)
jobs.append(job)
self.job_registry.register(jobs)
return jobs
def push_image_to_providers(self, image_id, build_id, providers, credentials, job_cls = BuildJob, *args, **kwargs):
if not build_id:
build_id = self._latest_unpushed(image_id)
watcher = PushWatcher(image_id, build_id, len(providers), self.warehouse)
jobs = []
for provider in providers:
target = self._map_provider_to_target(provider)
target_image_id = self._target_image_for_build_and_target(build_id, target)
template = self._template_for_target_image_id(target_image_id)
job = job_cls(template, target, image_id, build_id, *args, **kwargs)
job.push_image(target_image_id, provider, credentials, watcher)
jobs.append(job)
self.job_registry.register(jobs)
return jobs
def get_dynamic_provider_data(self, provider):
# Get provider details for RHEV-M or VSphere
# First try to interpret this as an ad-hoc/dynamic provider def
# If this fails, try to find it in one or the other of the config files
# If this all fails return None
# We use this in the builders as well so I have made it "public"
try:
xml_et = fromstring(provider)
return xml_et.attrib
except Exception as e:
self.log.debug('Testing provider for XML: %s' % e)
pass
try:
jload = json.loads(provider)
return jload
except ValueError as e:
self.log.debug('Testing provider for JSON: %s' % e)
pass
rhevm_data = self._return_dynamic_provider_data(provider, "rhevm")
if rhevm_data:
rhevm_data['target'] = "rhevm"
rhevm_data['name'] = provider
return rhevm_data
vsphere_data = self._return_dynamic_provider_data(provider, "vsphere")
if vsphere_data:
vsphere_data['target'] = "vsphere"
vsphere_data['name'] = provider
return vsphere_data
# It is not there
return None
def _xml_node(self, xml, xpath):
nodes = libxml2.parseDoc(xml).xpathEval(xpath)
if not nodes:
return None
return nodes[0].content
def _ensure_image_with_template(self, image_id, template, image_desc=None):
if not (type(template) == Template):
template = Template(template)
if not image_desc:
name = self._xml_node(template.xml, '/template/name')
if name:
image_desc = '<image><name>%s</name></image>' % name
else:
image_desc = '<image/>'
if not template.identifier:
template.identifier = self.warehouse.store_template(template.xml)
return self._ensure_image(image_id, image_desc, template.identifier)
def _ensure_image(self, image_id, image_desc, template_id=None):
if image_id:
return image_id
elif(template_id):
return self.warehouse.store_image(None, image_desc, dict(template=template_id))
else:
return self.warehouse.store_image(None, image_desc)
def _ensure_build(self, image_id, build_id):
if build_id:
return build_id
return self.warehouse.store_build(None, dict(image = image_id))
def _ensure_target_image(self, build_id, target):
target_image_id = self._target_image_for_build_and_target(build_id, target)
if target_image_id:
return target_image_id
return self.warehouse.store_target_image(None, None, dict(build=build_id, target=target))
def _ensure_provider_image(self, target_image_id, provider, target_identifier):
provider_image_id = self._provider_image_for_target_image_and_provider(target_image_id, provider)
if provider_image_id:
self._set_provider_image_attr(provider_image_id, 'target_identifier', target_identifier)
else:
metadata = dict(target_image=target_image_id, provider=provider, target_identifier=target_identifier)
return self.warehouse.create_provider_image(None, None, metadata)
def _load_template(self, image_id, build_id, template):
if not template:
if build_id:
template = self._template_for_build_id(build_id)
if not template and image_id:
template = self._template_for_image_id(image_id)
return Template(template)
def _commit_build(self, image_id, build_id):
parent_id = self._latest_build(image_id)
if parent_id:
self._set_build_parent(build_id, parent_id)
self._set_latest_build(image_id, build_id)
def _latest_build(self, image_id):
return self.warehouse.metadata_for_id_of_type(['latest_build'], image_id, 'image')['latest_build']
def _latest_unpushed(self, image_id):
return self.warehouse.metadata_for_id_of_type(['latest_unpushed'], image_id, 'image')['latest_unpushed']
def _set_latest_build(self, image_id, build_id):
self.warehouse.set_metadata_for_id_of_type({'latest_build' : build_id}, image_id, 'image')
def _set_build_parent(self, build_id, parent_id):
self.warehouse.set_metadata_for_id_of_type({'parent' : parent_id}, build_id, 'build')
def _targets_for_build_id(self, build_id):
targets = []
for target_image_id in self._target_images_for_build(build_id):
targets.append(self.warehouse.metadata_for_id_of_type(['target'], target_image_id, 'target_image')['target'])
return targets
def _targets_for_image_id(self, image_id):
build_id = self._latest_build(image_id)
if not build_id:
build_id = self._latest_unpushed(image_id)
return self._targets_for_build_id(build_id) if build_id else []
def _target_images_for_build(self, build_id):
return self.warehouse.query("target_image", "$build == \"%s\"" % (build_id,))
def _target_image_for_build_and_target(self, build_id, target):
results = self.warehouse.query("target_image", "$build == \"%s\" && $target == \"%s\"" % (build_id, target))
return results[0] if results else None
def _provider_image_for_target_image_and_provider(self, target_image_id, provider):
results = self.warehouse.query("provider_image", "$target_image == \"%s\" && $provider == \"%s\"" % (target_image_id, provider))
return results[0] if results else None
def _set_provider_image_attr(self, provider_image_id, attr, value):
self.warehouse.set_metadata_for_id_of_type({attr : value},
provider_image_id,
"provider_image")
def _template_for_target_image_id(self, target_image_id):
return self.warehouse.metadata_for_id_of_type(['template'], target_image_id, 'target_image')['template']
def _template_for_build_id(self, build_id):
target_image_ids = self._target_images_for_build(build_id)
return self._template_for_target_image_id(target_image_ids[0]) if target_image_ids else None
def _template_for_image_id(self, image_id):
build_id = self._latest_build(image_id)
if not build_id:
build_id = self._latest_unpushed(image_id)
return self._template_for_build_id(build_id) if build_id else None
def _return_dynamic_provider_data(self, provider, filebase):
provider_json = '/etc/imagefactory/%s.json' % (filebase)
if not os.path.exists(provider_json):
return False
provider_sites = {}
f = open(provider_json, 'r')
try:
provider_sites = json.loads(f.read())
finally:
f.close()
if provider in provider_sites:
return provider_sites[provider]
else:
return None
# FIXME: this is a hack; conductor is the only one who really
# knows this mapping, so perhaps it should provide it?
# e.g. pass a provider => target dict into push_image
# rather than just a list of providers. Perhaps just use
# this heuristic for the command line?
#
# provider semantics, per target:
# - ec2: region, one of ec2-us-east-1, ec2-us-west-1, ec2-ap-southeast-1, ec2-ap-northeast-1, ec2-eu-west-1
# - condorcloud: ignored
# - mock: any provider with 'mock' prefix
# - rackspace: provider is rackspace
# UPDATE - Sept 13, 2011 for dynamic providers
# - vpshere: encoded in provider string or a key in /etc/vmware.json
# - rhevm: encoded in provider string or a key in /etc/rhevm.json
#
def _map_provider_to_target(self, provider):
# Check for dynamic providers first
provider_data = self.get_dynamic_provider_data(provider)
if provider_data:
try:
return provider_data['target']
except KeyError as e:
self.log.debug('Provider data does not specify target!\n%s' % provider_data)
raise Exception('Provider data does not specify target!\n%s' % provider_data)
elif provider.startswith('ec2-'):
return 'ec2'
elif provider == 'rackspace':
return 'rackspace'
elif provider.startswith('mock'):
return 'mock'
else:
self.log.warn('No matching provider found for %s, using "condorcloud" by default.' % (provider))
return 'condorcloud' # condorcloud ignores provider
class Template(object):
uuid_pattern = '([0-9a-f]{8})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{12})'
identifier = props.prop("_identifier", "The identifier property.")
url = props.prop("_url", "The url property.")
xml = props.prop("_xml", "The xml property.")
@property
def os_name(self):
"""The property os_name"""
return self._content_at_path('/template/os/name')
@property
def os_version(self):
"""The property os_version"""
return self._content_at_path('/template/os/version')
@property
def os_arch(self):
"""The property os_arch"""
return self._content_at_path('/template/os/arch')
def __repr__(self):
if(self.xml):
return self.xml
else:
return super(Template, self).__repr__
def _content_at_path(self, path):
try:
return libxml2.parseDoc(self.xml).xpathEval(path)[0].content
except Exception as e:
self.log.exception('Could not parse document for path (%s):\n%s' % (path, e))
return None
def __init__(self, template=None, uuid=None, url=None, xml=None):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.identifier = None
self.url = None
self.xml = None
path = None
if(template):
template_string = str(template)
template_string_type = self.__template_string_type(template_string)
if(template_string_type == "UUID"):
uuid = template_string
elif(template_string_type == "URL"):
url = template_string
elif(template_string_type == "XML"):
xml = template_string
elif(template_string_type == "PATH"):
path = template_string
if(uuid):
uuid_string = uuid
self.identifier, self.xml = self.__fetch_template_for_uuid(uuid_string)
if((not self.identifier) and (not self.xml)):
raise RuntimeError("Could not create a template with the uuid %s" % (uuid, ))
elif(url):
self.url = url
self.identifier, self.xml = self.__fetch_template_with_url(url)
elif(xml):
self.xml = xml
elif(path):
template_file = open(path, "r")
file_content = template_file.read()
template_file.close()
if(self.__string_is_xml_template(file_content)):
self.xml = file_content
else:
raise ValueError("File %s does not contain properly formatted template xml:\n%s" % (path, self.__abbreviated_template(file_content)))
else:
raise ValueError("'template' must be a UUID, URL, XML string or XML document path...")
def __template_string_type(self, template_string):
regex = re.compile(Template.uuid_pattern)
match = regex.search(template_string)
if(template_string.lower().startswith("http")):
return "URL"
elif(("<template" in template_string.lower()) and ("</template>" in template_string.lower())):
return "XML"
elif(match):
return "UUID"
elif(os.path.exists(template_string)):
return "PATH"
else:
raise ValueError("'template_string' must be a UUID, URL, or XML document...\n--- TEMPLATE STRING ---\n%s\n-----------------" % (template_string, ))
def __fetch_template_for_uuid(self, uuid_string):
xml_string, metadata = self.warehouse.template_with_id(uuid_string)
if(xml_string and self.__string_is_xml_template(xml_string)):
return uuid_string, xml_string
else:
self.log.debug("Unable to fetch a valid template given template id %s:\n%s\nWill try fetching template id from a target image with this id..." % (uuid_string, self.__abbreviated_template(xml_string)))
template_id, xml_string, metadata = self.warehouse.template_for_target_image_id(uuid_string)
if(template_id and xml_string and self.__string_is_xml_template(xml_string)):
return template_id, xml_string
else:
self.log.debug("Unable to fetch a valid template given a target image id %s:\n%s\n" % (uuid_string, self.__abbreviated_template(xml_string)))
return None, None
def __string_is_xml_template(self, text):
return (("<template" in text.lower()) and ("</template>" in text.lower()))
def __fetch_template_with_url(self, url):
template_id = None
regex = re.compile(Template.uuid_pattern)
match = regex.search(url)
if (match):
template_id = match.group()
if (not url.startswith(self.warehouse.url)):
response_headers, response = httplib2.Http().request(url, "GET", headers={'content-type':'text/plain'})
else:
response_headers, response = self.warehouse.request(url, "GET")
if(response and self.__string_is_xml_template(response)):
return template_id, response
else:
raise RuntimeError("Recieved status %s fetching a template from %s!\n--- Response Headers:\n%s\n--- Response:\n%s" % (response_headers["status"], url, response_headers, response))
def __abbreviated_template(self, template_string):
lines = template_string.splitlines(True)
if(len(lines) > 20):
return "%s\n...\n...\n...\n%s" % ("".join(lines[0:10]), "".join(lines[-10:len(lines)]))
else:
return template_string
def setUp(self):
logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.template_xml = "<template>This is a test template. There is not much to it.</template>"
class testTemplate(unittest2.TestCase):
def setUp(self):
logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log')
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
self.template_xml = "<template>This is a test template. There is not much to it.</template>"
def tearDown(self):
del self.warehouse
del self.template_xml
def testTemplateFromUUID(self):
template_id = self.warehouse.store_template(self.template_xml)
template = Template(template_id)
self.assertEqual(template_id, template.identifier)
self.assertEqual(self.template_xml, template.xml)
self.assertIsNone(template.url)
def testTemplateFromImageID(self):
template_id = self.warehouse.store_template(self.template_xml)
template = Template(template_id)
target = "mock"
builder = Mock_Builder(self.template_xml, target)
builder.build_image()
metadata = dict(template=template_id, target=target, icicle="None", target_parameters="None")
self.warehouse.store_target_image(builder.new_image_id, builder.image, metadata=metadata)
image_template = Template(builder.new_image_id)
self.assertEqual(template_id, image_template.identifier)
self.assertEqual(self.template_xml, image_template.xml)
self.assertIsNone(template.url)
def testTemplateFromXML(self):
template = Template(self.template_xml)
self.assertEqual(self.template_xml, template.xml)
self.assertIsNone(template.identifier)
self.assertIsNone(template.url)
def testTemplateFromURL(self):
template_id = self.warehouse.store_template(self.template_xml)
template_url = "%s/%s/%s" % (self.warehouse.url, self.warehouse.template_bucket, template_id)
template = Template(template_url)
self.assertEqual(template_url, template.url)
self.assertEqual(template_id, template.identifier)
self.assertEqual(self.template_xml, template.xml)
def testTemplateFromPath(self):
(fd, template_path) = tempfile.mkstemp(prefix = "test_image_factory-")
os.write(fd, self.template_xml)
os.close(fd)
template = Template(template_path)
self.assertIsNone(template.url)
self.assertIsNone(template.identifier)
self.assertEqual(self.template_xml, template.xml)
os.remove(template_path)
def testTemplateStringRepresentation(self):
template = Template(self.template_xml)
self.assertEqual(self.template_xml, repr(template))
self.assertEqual(self.template_xml, str(template))
self.assertEqual(self.template_xml, "%r" % (template, ))
self.assertEqual(self.template_xml, "%s" % (template, ))
def _singleton_init(self):
super(ImageFactory, self)._singleton_init()
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.qmf_object = Data(ImageFactory.qmf_schema)
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
class ImageFactory(Singleton):
# QMF schema for ImageFactory
qmf_schema = Schema(SCHEMA_TYPE_DATA, "com.redhat.imagefactory", "ImageFactory")
# method for building an image
_build_image_method = SchemaMethod("image", desc="Build a new image for a given target cloud")
_build_image_method.addArgument(SchemaProperty("template", SCHEMA_DATA_STRING, direction=DIR_IN, desc="string of xml, uuid, or url"))
_build_image_method.addArgument(SchemaProperty("target", SCHEMA_DATA_STRING, direction=DIR_IN, desc="name of the cloud to target"))
_build_image_method.addArgument(SchemaProperty("build_adaptor", SCHEMA_DATA_MAP, direction=DIR_OUT, desc="the QMF address of the build_adaptor instantiated"))
qmf_schema.addMethod(_build_image_method)
# method for building images
_build_images_method = SchemaMethod("build_image", desc="Build an image for the given target clouds")
_build_images_method.addArgument(SchemaProperty("image", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of an image previously built"))
_build_images_method.addArgument(SchemaProperty("build", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of a previous build of the image"))
_build_images_method.addArgument(SchemaProperty("template", SCHEMA_DATA_STRING, direction=DIR_IN, desc="string of xml, uuid, or url"))
_build_images_method.addArgument(SchemaProperty("targets", SCHEMA_DATA_LIST, direction=DIR_IN, desc="names of the clouds to target"))
_build_images_method.addArgument(SchemaProperty("build_adaptors", SCHEMA_DATA_LIST, direction=DIR_OUT, desc="the QMF addresses of the build_adaptors instantiated"))
qmf_schema.addMethod(_build_images_method)
# method for creating a provider_image from an image
_push_image_method = SchemaMethod("provider_image", desc="Push an image to a provider.")
_push_image_method.addArgument(SchemaProperty("image_id", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the uuid of an image previously built"))
_push_image_method.addArgument(SchemaProperty("provider", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the name of the cloud provider, often a region"))
_push_image_method.addArgument(SchemaProperty("credentials", SCHEMA_DATA_STRING, direction=DIR_IN, desc="an xml string representation of the credentials"))
_push_image_method.addArgument(SchemaProperty("build_adaptor", SCHEMA_DATA_MAP, direction=DIR_OUT, desc="the QMF address of the build_adaptor instantiated"))
qmf_schema.addMethod(_push_image_method)
# method for pushing an image to multiple providers
_push_images_method = SchemaMethod("push_image", desc="Push an image to multiple providers.")
_push_images_method.addArgument(SchemaProperty("image", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of an image previously built"))
_push_images_method.addArgument(SchemaProperty("build", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of a previous build of the image"))
_push_images_method.addArgument(SchemaProperty("providers", SCHEMA_DATA_LIST, direction=DIR_IN, desc="the names of the cloud providers, often regions"))
_push_images_method.addArgument(SchemaProperty("credentials", SCHEMA_DATA_STRING, direction=DIR_IN, desc="an xml string representation of the credentials"))
_push_images_method.addArgument(SchemaProperty("build_adaptors", SCHEMA_DATA_LIST, direction=DIR_OUT, desc="the QMF addresses of the build_adaptors instantiated"))
qmf_schema.addMethod(_push_images_method)
# method for importing a provider image
_import_image_method = SchemaMethod("import_image", desc="Import an image using a target specific image identifier")
_import_image_method.addArgument(SchemaProperty("image", SCHEMA_DATA_STRING, direction=DIR_IN_OUT, desc="the UUID of an image previously built"))
_import_image_method.addArgument(SchemaProperty("build", SCHEMA_DATA_STRING, direction=DIR_IN_OUT, desc="the UUID of a previous build of the image"))
_import_image_method.addArgument(SchemaProperty("target_identifier", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the target specific image ID"))
_import_image_method.addArgument(SchemaProperty("image_desc", SCHEMA_DATA_STRING, direction=DIR_IN, desc="an xml string description of the image"))
_import_image_method.addArgument(SchemaProperty("target", SCHEMA_DATA_STRING, direction=DIR_IN, desc="name of the cloud to target"))
_import_image_method.addArgument(SchemaProperty("provider", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the name of the cloud provider, often a region"))
_import_image_method.addArgument(SchemaProperty("target_image", SCHEMA_DATA_STRING, direction=DIR_OUT, desc="the UUID of the target image object"))
_import_image_method.addArgument(SchemaProperty("provider_image", SCHEMA_DATA_STRING, direction=DIR_OUT, desc="the UUID of the provider image object"))
qmf_schema.addMethod(_import_image_method)
# this method will return a representation of the object's finite state machine
_states_method = SchemaMethod("instance_states", desc = "Returns a dictionary representing the finite state machine for instances.")
_states_method.addArgument(SchemaProperty("class_name", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the name of the class to query for instance states"))
_states_method.addArgument(SchemaProperty("states", SCHEMA_DATA_STRING, direction=DIR_OUT, desc="string representation of a dictionary describing the workflow"))
qmf_schema.addMethod(_states_method)
@classmethod
def object_states(cls):
"""Returns a dictionary representing the finite state machine for instances of this class."""
return {}
qmf_object = props.prop("_qmf_object", "The qmf_object property.")
agent = props.prop("_agent", "The property agent")
def _singleton_init(self):
super(ImageFactory, self)._singleton_init()
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.qmf_object = Data(ImageFactory.qmf_schema)
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
def __init__(self):
pass
def image(self,template,target):
template_object = Template(template=template)
build_adaptor = BuildAdaptor(template_object,target,agent=self.agent)
# Needed to allow us to walk all builders during a shutdown
BuildDispatcher().job_registry.register(build_adaptor)
build_adaptor.build_image()
return build_adaptor
def provider_image(self, image_id, provider, credentials):
target_image_id = image_id
image_metadata = self.warehouse.metadata_for_id_of_type(("template", "target"), target_image_id, "target_image")
template_id = image_metadata["template"]
target = image_metadata["target"]
if (template_id and target):
build_adaptor = BuildAdaptor(Template(uuid=template_id),target,agent=self.agent)
# Needed to allow us to walk all builders during a shutdown
BuildDispatcher().job_registry.register(build_adaptor)
build_adaptor.push_image(target_image_id, provider, credentials)
return build_adaptor
else:
raise RuntimeError("Could not return build_adaptor!\nimage_metadata: %s\ntemplate_id: %s\ntemplate: %s\n" % (image_metadata, template_id, target))
def build_image(self, image, build, template, targets):
return BuildDispatcher().build_image_for_targets(image, build, template, targets, BuildAdaptor, self.agent)
def push_image(self, image, build, providers, credentials):
return BuildDispatcher().push_image_to_providers(image, build, providers, credentials, BuildAdaptor, self.agent)
def import_image(self, image, build, target_identifier, image_desc, target, provider):
ids = BuildDispatcher().import_image(image, build, target_identifier, image_desc, target, provider)
return {"image" : ids[0], "build" : ids[1], "target_image" : ids[2], "provider_image" : ids[3]}
def instance_states(self, class_name):
"""Returns a dictionary representing the finite state machine for instances of the class specified."""
module_name = "imgfac.qmfagent.%s" % (class_name, )
__import__(module_name)
return dict(states=str(getattr(sys.modules[module_name], class_name).object_states()))
class BuildDispatcher(Singleton):
def _singleton_init(self):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse'])
self.job_registry = JobRegistry()
self.builders = dict()
##### BEGIN new BuildDispatcher code for plugins and no warehouse #####
def builder_for_base_image(self, template, parameters=None):
builder = Builder()
builder.build_image_from_template(template)
self.builders[builder.base_image.identifier] = builder
return builder
def builder_for_target_image(self, target, image_id=None, template=None, parameters=None):
builder = Builder()
builder.customize_image_for_target(target, image_id, template, parameters)
self.builders[builder.target_image.identifier] = builder
return builder
def builder_for_provider_image(self, provider, credentials, target, image_id=None, template=None, parameters=None):
builder = Builder()
builder.create_image_on_provider(provider, credentials, target, image_id, template, parameters)
self.builders[builder.provider_image.identifier] = builder
return builder
##### END new BuildDispatcher code
def import_image(self, image_id, build_id, target_identifier, image_desc, target, provider):
image_id = self._ensure_image(image_id, image_desc)
build_id = self._ensure_build(image_id, build_id)
target_image_id = self._ensure_target_image(build_id, target)
provider_image_id = self._ensure_provider_image(target_image_id, provider, target_identifier)
self._commit_build(image_id, build_id)
return (image_id, build_id, target_image_id, provider_image_id)
def build_image_for_targets(self, image_id, build_id, template, targets, job_cls = BuildJob, *args, **kwargs):
if image_id and not targets:
targets = self._targets_for_image_id(image_id)
template = self._load_template(image_id, build_id, template)
image_id = self._ensure_image_with_template(image_id, template)
build_id = self._ensure_build(image_id, build_id)
watcher = BuildWatcher(image_id, build_id, len(targets), self.warehouse)
jobs = []
for target in targets:
job = job_cls(template, target, image_id, build_id, *args, **kwargs)
job.build_image(watcher)
jobs.append(job)
self.job_registry.register(jobs)
return jobs
def push_image_to_providers(self, image_id, build_id, providers, credentials, job_cls = BuildJob, *args, **kwargs):
if not build_id:
build_id = self._latest_unpushed(image_id)
watcher = PushWatcher(image_id, build_id, len(providers), self.warehouse)
jobs = []
for provider in providers:
target = Provider.map_provider_to_target(provider)
target_image_id = self._target_image_for_build_and_target(build_id, target)
template = self._template_for_target_image_id(target_image_id)
job = job_cls(template, target, image_id, build_id, *args, **kwargs)
job.push_image(target_image_id, provider, credentials, watcher)
jobs.append(job)
self.job_registry.register(jobs)
return jobs
def _xml_node(self, xml, xpath):
nodes = libxml2.parseDoc(xml).xpathEval(xpath)
if not nodes:
return None
return nodes[0].content
def _ensure_image_with_template(self, image_id, template, image_desc=None):
if not (type(template) == Template):
template = Template(template)
if not image_desc:
name = self._xml_node(template.xml, '/template/name')
if name:
image_desc = '<image><name>%s</name></image>' % name
else:
image_desc = '<image/>'
if not template.identifier:
template.identifier = self.warehouse.store_template(template.xml)
return self._ensure_image(image_id, image_desc, template.identifier)
def _ensure_image(self, image_id, image_desc, template_id=None):
if image_id:
return image_id
elif(template_id):
return self.warehouse.store_image(None, image_desc, dict(template=template_id))
else:
return self.warehouse.store_image(None, image_desc)
def _ensure_build(self, image_id, build_id):
if build_id:
return build_id
return self.warehouse.store_build(None, dict(image = image_id))
def _ensure_target_image(self, build_id, target):
target_image_id = self._target_image_for_build_and_target(build_id, target)
if target_image_id:
return target_image_id
return self.warehouse.store_target_image(None, None, dict(build=build_id, target=target))
def _ensure_provider_image(self, target_image_id, provider, target_identifier):
provider_image_id = self._provider_image_for_target_image_and_provider(target_image_id, provider)
if provider_image_id:
self._set_provider_image_attr(provider_image_id, 'target_identifier', target_identifier)
else:
metadata = dict(target_image=target_image_id, provider=provider, target_identifier=target_identifier)
return self.warehouse.create_provider_image(None, None, metadata)
def _load_template(self, image_id, build_id, template):
if not template:
if build_id:
template = self._template_for_build_id(build_id)
if not template and image_id:
template = self._template_for_image_id(image_id)
return Template(template)
def _commit_build(self, image_id, build_id):
parent_id = self._latest_build(image_id)
if parent_id:
self._set_build_parent(build_id, parent_id)
self._set_latest_build(image_id, build_id)
def _latest_build(self, image_id):
return self.warehouse.metadata_for_id_of_type(['latest_build'], image_id, 'image')['latest_build']
def _latest_unpushed(self, image_id):
return self.warehouse.metadata_for_id_of_type(['latest_unpushed'], image_id, 'image')['latest_unpushed']
def _set_latest_build(self, image_id, build_id):
self.warehouse.set_metadata_for_id_of_type({'latest_build' : build_id}, image_id, 'image')
def _set_build_parent(self, build_id, parent_id):
self.warehouse.set_metadata_for_id_of_type({'parent' : parent_id}, build_id, 'build')
def _targets_for_build_id(self, build_id):
targets = []
for target_image_id in self._target_images_for_build(build_id):
targets.append(self.warehouse.metadata_for_id_of_type(['target'], target_image_id, 'target_image')['target'])
return targets
def _targets_for_image_id(self, image_id):
build_id = self._latest_build(image_id)
if not build_id:
build_id = self._latest_unpushed(image_id)
return self._targets_for_build_id(build_id) if build_id else []
def _target_images_for_build(self, build_id):
return self.warehouse.query("target_image", "$build == \"%s\"" % (build_id,))
def _target_image_for_build_and_target(self, build_id, target):
results = self.warehouse.query("target_image", "$build == \"%s\" && $target == \"%s\"" % (build_id, target))
return results[0] if results else None
def _provider_image_for_target_image_and_provider(self, target_image_id, provider):
results = self.warehouse.query("provider_image", "$target_image == \"%s\" && $provider == \"%s\"" % (target_image_id, provider))
return results[0] if results else None
def _set_provider_image_attr(self, provider_image_id, attr, value):
self.warehouse.set_metadata_for_id_of_type({attr : value},
provider_image_id,
"provider_image")
def _template_for_target_image_id(self, target_image_id):
return self.warehouse.metadata_for_id_of_type(['template'], target_image_id, 'target_image')['template']
def _template_for_build_id(self, build_id):
target_image_ids = self._target_images_for_build(build_id)
return self._template_for_target_image_id(target_image_ids[0]) if target_image_ids else None
def _template_for_image_id(self, image_id):
build_id = self._latest_build(image_id)
if not build_id:
build_id = self._latest_unpushed(image_id)
return self._template_for_build_id(build_id) if build_id else None
