def setUp(self): logging.basicConfig( level=logging.NOTSET, format= '%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse( ApplicationConfiguration().configuration["warehouse"]) self.metadata = dict(key1="value1", key2="value2", key3="value3")
def setUp(self): logging.basicConfig( level=logging.NOTSET, format= '%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse( ApplicationConfiguration().configuration["warehouse"]) self.template_xml = "<template>This is a test template. There is not much to it.</template>"
def __init__(self, template=None, uuid=None, url=None, xml=None): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) self.identifier = None self.url = None self.xml = None path = None if(template): template_string = str(template) template_string_type = self.__template_string_type(template_string) if(template_string_type == "UUID"): uuid = template_string elif(template_string_type == "URL"): url = template_string elif(template_string_type == "XML"): xml = template_string elif(template_string_type == "PATH"): path = template_string if(uuid): uuid_string = uuid self.identifier, self.xml = self.__fetch_template_for_uuid(uuid_string) if((not self.identifier) and (not self.xml)): raise RuntimeError("Could not create a template with the uuid %s" % (uuid, )) elif(url): self.url = url self.identifier, self.xml = self.__fetch_template_with_url(url) elif(xml): self.xml = xml elif(path): template_file = open(path, "r") file_content = template_file.read() template_file.close() if(self.__string_is_xml_template(file_content)): self.xml = file_content else: raise ValueError("File %s does not contain properly formatted template xml:\n%s" % (path, self.__abbreviated_template(file_content))) else: raise ValueError("'template' must be a UUID, URL, XML string or XML document path...")
class Template(object): uuid_pattern = '([0-9a-f]{8})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{12})' identifier = props.prop("_identifier", "The identifier property.") url = props.prop("_url", "The url property.") xml = props.prop("_xml", "The xml property.") def __repr__(self): if(self.xml): return self.xml else: return super(Template, self).__repr__ def __init__(self, template=None, uuid=None, url=None, xml=None): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) self.identifier = None self.url = None self.xml = None path = None if(template): template_string = str(template) template_string_type = self.__template_string_type(template_string) if(template_string_type == "UUID"): uuid = template_string elif(template_string_type == "URL"): url = template_string elif(template_string_type == "XML"): xml = template_string elif(template_string_type == "PATH"): path = template_string if(uuid): uuid_string = uuid self.identifier, self.xml = self.__fetch_template_for_uuid(uuid_string) if((not self.identifier) and (not self.xml)): raise RuntimeError("Could not create a template with the uuid %s" % (uuid, )) elif(url): self.url = url self.identifier, self.xml = self.__fetch_template_with_url(url) elif(xml): self.xml = xml elif(path): template_file = open(path, "r") file_content = template_file.read() template_file.close() if(self.__string_is_xml_template(file_content)): self.xml = file_content else: raise ValueError("File %s does not contain properly formatted template xml:\n%s" % (path, self.__abbreviated_template(file_content))) else: raise ValueError("'template' must be a UUID, URL, XML string or XML document path...") def __template_string_type(self, template_string): regex = re.compile(Template.uuid_pattern) match = regex.search(template_string) if(template_string.lower().startswith("http")): return "URL" elif(("<template>" in template_string.lower()) and ("</template>" in template_string.lower())): return "XML" elif(match): return "UUID" elif(os.path.exists(template_string)): return "PATH" else: raise ValueError("'template_string' must be a UUID, URL, or XML document...\n--- TEMPLATE STRING ---\n%s\n-----------------" % (template_string, )) def __fetch_template_for_uuid(self, uuid_string): xml_string, metadata = self.warehouse.template_with_id(uuid_string) if(xml_string and self.__string_is_xml_template(xml_string)): return uuid_string, xml_string else: self.log.debug("Unable to fetch a valid template given template id %s:\n%s\nWill try fetching template id from a target image with this id..." % (uuid_string, self.__abbreviated_template(xml_string))) template_id, xml_string, metadata = self.warehouse.template_for_target_image_id(uuid_string) if(template_id and xml_string and self.__string_is_xml_template(xml_string)): return template_id, xml_string else: self.log.debug("Unable to fetch a valid template given a target image id %s:\n%s\n" % (uuid_string, self.__abbreviated_template(xml_string))) return None, None def __string_is_xml_template(self, text): return (("<template>" in text.lower()) and ("</template>" in text.lower())) def __fetch_template_with_url(self, url): template_id = None regex = re.compile(Template.uuid_pattern) match = regex.search(url) if (match): template_id = match.group() response_headers, response = httplib2.Http().request(url, "GET", headers={'content-type':'text/plain'}) if(response and self.__string_is_xml_template(response)): return template_id, response else: raise RuntimeError("Recieved status %s fetching a template from %s!\n--- Response Headers:\n%s\n--- Response:\n%s" % (response_headers["status"], url, response_headers, response)) def __abbreviated_template(self, template_string): lines = template_string.splitlines(True) if(len(lines) > 20): return "%s\n...\n...\n...\n%s" % ("".join(lines[0:10]), "".join(lines[-10:len(lines)])) else: return template_string
class testImageWarehouse(unittest.TestCase): def setUp(self): logging.basicConfig( level=logging.NOTSET, format= '%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse( ApplicationConfiguration().configuration["warehouse"]) self.metadata = dict(key1="value1", key2="value2", key3="value3") def tearDown(self): del self.warehouse del self.metadata def testImageWarehouseMethods(self): """Tests CRUD operations on target images, templates, icicles, and metadata on those objects...""" # TARGET IMAGE # store a target image target_image_id = str(uuid.uuid4()) file_content = "This is just to test storing a target image in warehouse. There is not much to see here." file_path = "/tmp/testImageWarehouse_testStoreAndFetchTargetImage.%s" % ( target_image_id, ) with open(file_path, 'w') as test_file: test_file.write(file_content) test_file.close() self.warehouse.store_target_image(target_image_id, file_path, metadata=self.metadata) # now fetch that target image target_image, metadata = self.warehouse.target_image_with_id( target_image_id, metadata_keys=self.metadata.keys()) # now make the assertions self.assertEqual(file_content, target_image) self.assertEqual(self.metadata, metadata) # TEMPLATE template_content = "<template>This is a test template. There is not much to see here.</template>" # store the template and let an id get assigned template_id = self.warehouse.store_template(template_content) self.assertIsNotNone(template_id) # store the template with a specified id template_id_known = str(uuid.uuid4()) template_id2 = self.warehouse.store_template(template_content, template_id_known) self.assertEqual(template_id_known, template_id2) # fetch the templates fetched_template_content, template_metadata1 = self.warehouse.template_with_id( template_id) self.assertEqual(template_content, fetched_template_content) fetched_template_content2, template_metadata2 = self.warehouse.template_with_id( template_id_known) self.assertEqual(template_content, fetched_template_content2) # set the template id for a target image and fetch it back self.warehouse.set_metadata_for_id_of_type(dict(template=template_id), target_image_id, "target_image") template_id3, fetched_template_content3, template_metadata3 = self.warehouse.template_for_target_image_id( target_image_id) self.assertEqual(template_id, template_id3) self.assertEqual(template_content, fetched_template_content3) # ICICLE icicle_content = "<icicle>This is a test icicle. There is not much to see here.</icicle>" # store the icicle and let an id get assigned icicle_id = self.warehouse.store_icicle(icicle_content) self.assertIsNotNone(icicle_id) # store the icicle with a specified id icicle_id_known = str(uuid.uuid4()) icicle_id2 = self.warehouse.store_icicle(icicle_content, icicle_id_known) self.assertEqual(icicle_id_known, icicle_id2) # fetch the icicles fetched_icicle_content, icicle_metadata1 = self.warehouse.icicle_with_id( icicle_id) self.assertEqual(icicle_content, fetched_icicle_content) fetched_icicle_content2, icicle_metadata2 = self.warehouse.icicle_with_id( icicle_id_known) self.assertEqual(icicle_content, fetched_icicle_content2) # set the icicle id for a target image and fetch it back self.warehouse.set_metadata_for_id_of_type(dict(icicle=icicle_id), target_image_id, "target_image") icicle_id3, fetched_icicle_content3, icicle_metadata3 = self.warehouse.icicle_for_target_image_id( target_image_id) self.assertEqual(icicle_id, icicle_id3) self.assertEqual(icicle_content, fetched_icicle_content3) self.assertTrue( self.warehouse.remove_target_image_with_id(target_image_id)) self.assertTrue(self.warehouse.remove_template_with_id(template_id)) self.assertTrue(self.warehouse.remove_template_with_id(template_id2)) self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id)) self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id2)) os.remove(file_path) def testImageAndBuildMethods(self): """Tests CRUD operations on images, builds and metadata on those objects...""" image_xml = '<image/>' image_id = self.warehouse.store_image(None, image_xml, self.metadata) self.assertIsNotNone(image_id) image_body, metadata = self.warehouse.object_with_id_of_type( image_id, 'image', self.metadata.keys()) self.assertEqual(image_xml, image_body) self.assertEqual(self.metadata, metadata) build_id = self.warehouse.store_build(None, self.metadata) build_body, metadata = self.warehouse.object_with_id_of_type( build_id, 'build', self.metadata.keys()) self.assertEqual('', build_body) self.assertEqual(self.metadata, metadata) ids = self.warehouse.query( 'build', '$object_type == "build" && $key1 == "value1"') self.assertIn(build_id, ids) def testBucketCreation(self): # self.assert_(self.warehouse.create_bucket_at_url("%s/unittests-create_bucket/%s" % (self.warehouse.url, str(uuid.uuid4())))) self.warehouse.create_bucket_at_url("%s/unittests-create_bucket" % (self.warehouse.url, )) self.assert_( self.warehouse.create_bucket_at_url( "%s/unittests-create_bucket/%s" % (self.warehouse.url, time.asctime().replace(' ', '-'))))
def _singleton_init(self): self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse'])
class BuildDispatcher(Singleton): def _singleton_init(self): self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse']) def import_image(self, image_id, build_id, target_identifier, image_desc, target, provider): image_id = self._ensure_image(image_id, image_desc) build_id = self._ensure_build(image_id, build_id) target_image_id = self._ensure_target_image(build_id, target) provider_image_id = self._ensure_provider_image(target_image_id, provider, target_identifier) self._commit_build(image_id, build_id) return (image_id, build_id, target_image_id, provider_image_id) def build_image_for_targets(self, image_id, build_id, template, targets, job_cls = BuildJob, *args, **kwargs): if image_id and not targets: targets = self._targets_for_image_id(image_id) template = self._load_template(image_id, build_id, template) image_id = self._ensure_image_with_template(image_id, template) build_id = self._ensure_build(image_id, build_id) watcher = BuildWatcher(image_id, build_id, len(targets), self.warehouse) jobs = [] for target in targets: job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.build_image(watcher) jobs.append(job) return jobs def push_image_to_providers(self, image_id, build_id, providers, credentials, job_cls = BuildJob, *args, **kwargs): if not build_id: build_id = self._latest_unpushed(image_id) watcher = PushWatcher(image_id, build_id, len(providers), self.warehouse) jobs = [] for provider in providers: target = self._map_provider_to_target(provider) target_image_id = self._target_image_for_build_and_target(build_id, target) template = self._template_for_target_image_id(target_image_id) job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.push_image(target_image_id, provider, credentials, watcher) jobs.append(job) return jobs def _xml_node(self, xml, xpath): nodes = libxml2.parseDoc(xml).xpathEval(xpath) if not nodes: return None return nodes[0].content def _ensure_image_with_template(self, image_id, template): name = self._xml_node(template.xml, '/template/name') if name: image_desc = '<image><name>%s</name></image>' % name else: image_desc = '</image>' return self._ensure_image(image_id, image_desc) def _ensure_image(self, image_id, image_desc): if image_id: return image_id else: return self.warehouse.store_image(None, image_desc) def _ensure_build(self, image_id, build_id): if build_id: return build_id return self.warehouse.store_build(None, dict(image = image_id)) def _ensure_target_image(self, build_id, target): target_image_id = self._target_image_for_build_and_target(build_id, target) if target_image_id: return target_image_id return self.warehouse.store_target_image(None, None, dict(build=build_id, target=target)) def _ensure_provider_image(self, target_image_id, provider, target_identifier): provider_image_id = self._provider_image_for_target_image_and_provider(target_image_id, provider) if provider_image_id: self._set_provider_image_attr(provider_image_id, 'target_identifier', target_identifier) else: metadata = dict(target_image=target_image_id, provider=provider, target_identifier=target_identifier) return self.warehouse.create_provider_image(None, None, metadata) def _load_template(self, image_id, build_id, template): if not template: if build_id: template = self._template_for_build_id(build_id) if not template and image_id: template = self._template_for_image_id(image_id) return Template(template) def _commit_build(self, image_id, build_id): parent_id = self._latest_build(image_id) if parent_id: self._set_build_parent(build_id, parent_id) self._set_latest_build(image_id, build_id) def _latest_build(self, image_id): return self.warehouse.metadata_for_id_of_type(['latest_build'], image_id, 'image')['latest_build'] def _latest_unpushed(self, image_id): return self.warehouse.metadata_for_id_of_type(['latest_unpushed'], image_id, 'image')['latest_unpushed'] def _set_latest_build(self, image_id, build_id): self.warehouse.set_metadata_for_id_of_type({'latest_build' : build_id}, image_id, 'image') def _set_build_parent(self, build_id, parent_id): self.warehouse.set_metadata_for_id_of_type({'parent' : parent_id}, build_id, 'build') def _targets_for_build_id(self, build_id): targets = [] for target_image_id in self._target_images_for_build(build_id): targets.append(self.warehouse.metadata_for_id_of_type(['target'], target_image_id, 'target_image')['target']) return targets def _targets_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._targets_for_build_id(build_id) if build_id else [] def _target_images_for_build(self, build_id): return self.warehouse.query("target_image", "$build == \"%s\"" % (build_id,)) def _target_image_for_build_and_target(self, build_id, target): results = self.warehouse.query("target_image", "$build == \"%s\" && $target == \"%s\"" % (build_id, target)) return results[0] if results else None def _provider_image_for_target_image_and_provider(self, target_image_id, provider): results = self.warehouse.query("provider_image", "$target_image == \"%s\" && $provider == \"%s\"" % (target_image_id, provider)) return results[0] if results else None def _set_provider_image_attr(provider_image_id, attr, value): self.warehouse.set_metadata_for_id_of_type({attr : value}, provider_image_id, "provider_image") def _template_for_target_image_id(self, target_image_id): return self.warehouse.metadata_for_id_of_type(['template'], target_image_id, 'target_image')['template'] def _template_for_build_id(self, build_id): target_image_ids = self._target_images_for_build(build_id) return self._template_for_target_image_id(target_image_ids[0]) if target_image_ids else None def _template_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._template_for_build_id(build_id) if build_id else None def _is_dynamic_provider(self, provider, filebase): provider_json = '/etc/%s.json' % (filebase) if not os.path.exists(provider_json): return False provider_sites = {} f = open(provider_json, 'r') try: provider_sites = json.loads(f.read()) finally: f.close() return provider in provider_sites # FIXME: this is a hack; conductor is the only one who really # knows this mapping, so perhaps it should provide it? # e.g. pass a provider => target dict into push_image # rather than just a list of providers. Perhaps just use # this heuristic for the command line? # # provider semantics, per target: # - ec2: region, one of ec2-us-east-1, ec2-us-west-1, ec2-ap-southeast-1, ec2-ap-northeast-1, ec2-eu-west-1 # - condorcloud: ignored # - rhevm: a key in /etc/rhevm.json and passed to op=register&site=provider # - vpshere: a key in /etc/vmware.json # - mock: any provider with 'mock' prefix # - rackspace: provider is rackspace # def _map_provider_to_target(self, provider): if provider.startswith('ec2-'): return 'ec2' elif provider == 'rackspace': return 'rackspace' elif self._is_dynamic_provider(provider, 'rhevm'): return 'rhevm' elif self._is_dynamic_provider(provider, 'vmware'): return 'vsphere' elif provider.startswith('mock'): return 'mock' else: return 'condorcloud' # condorcloud ignores provider
def setUp(self): logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) self.template_xml = "<template>This is a test template. There is not much to it.</template>"
class testTemplate(unittest.TestCase): def setUp(self): logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) self.template_xml = "<template>This is a test template. There is not much to it.</template>" def tearDown(self): del self.warehouse del self.template_xml def testTemplateFromUUID(self): template_id = self.warehouse.store_template(self.template_xml) template = Template(template_id) self.assertEqual(template_id, template.identifier) self.assertEqual(self.template_xml, template.xml) self.assertIsNone(template.url) def testTemplateFramImageID(self): template_id = self.warehouse.store_template(self.template_xml) template = Template(template_id) target = "mock" builder = MockBuilder(self.template_xml, target) builder.build_image() metadata = dict(template=template_id, target=target, icicle="None", target_parameters="None") self.warehouse.store_target_image(builder.new_image_id, builder.image, metadata=metadata) image_template = Template(builder.new_image_id) self.assertEqual(template_id, image_template.identifier) self.assertEqual(self.template_xml, image_template.xml) self.assertIsNone(template.url) def testTemplateFromXML(self): template = Template(self.template_xml) self.assertEqual(self.template_xml, template.xml) self.assertIsNone(template.identifier) self.assertIsNone(template.url) def testTemplateFromURL(self): template_id = self.warehouse.store_template(self.template_xml) template_url = "%s/%s/%s" % (self.warehouse.url, self.warehouse.template_bucket, template_id) template = Template(template_url) self.assertEqual(template_url, template.url) self.assertEqual(template_id, template.identifier) self.assertEqual(self.template_xml, template.xml) def testTemplateFromPath(self): (fd, template_path) = tempfile.mkstemp(prefix = "test_image_factory-") os.write(fd, self.template_xml) os.close(fd) template = Template(template_path) self.assertIsNone(template.url) self.assertIsNone(template.identifier) self.assertEqual(self.template_xml, template.xml) os.remove(template_path) def testTemplateStringRepresentation(self): template = Template(self.template_xml) self.assertEqual(self.template_xml, repr(template)) self.assertEqual(self.template_xml, str(template)) self.assertEqual(self.template_xml, "%r" % (template, )) self.assertEqual(self.template_xml, "%s" % (template, ))
def _singleton_init(self): self.warehouse = ImageWarehouse( ApplicationConfiguration().configuration['warehouse'])
class testTemplate(unittest.TestCase): def setUp(self): logging.basicConfig( level=logging.NOTSET, format= '%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse( ApplicationConfiguration().configuration["warehouse"]) self.template_xml = "<template>This is a test template. There is not much to it.</template>" def tearDown(self): del self.warehouse del self.template_xml def testTemplateFromUUID(self): template_id = self.warehouse.store_template(self.template_xml) template = Template(template_id) self.assertEqual(template_id, template.identifier) self.assertEqual(self.template_xml, template.xml) self.assertIsNone(template.url) def testTemplateFramImageID(self): template_id = self.warehouse.store_template(self.template_xml) template = Template(template_id) target = "mock" builder = MockBuilder(self.template_xml, target) builder.build_image() metadata = dict(template=template_id, target=target, icicle="None", target_parameters="None") self.warehouse.store_target_image(builder.new_image_id, builder.image, metadata=metadata) image_template = Template(builder.new_image_id) self.assertEqual(template_id, image_template.identifier) self.assertEqual(self.template_xml, image_template.xml) self.assertIsNone(template.url) def testTemplateFromXML(self): template = Template(self.template_xml) self.assertEqual(self.template_xml, template.xml) self.assertIsNone(template.identifier) self.assertIsNone(template.url) def testTemplateFromURL(self): template_id = self.warehouse.store_template(self.template_xml) template_url = "%s/%s/%s" % ( self.warehouse.url, self.warehouse.template_bucket, template_id) template = Template(template_url) self.assertEqual(template_url, template.url) self.assertEqual(template_id, template.identifier) self.assertEqual(self.template_xml, template.xml) def testTemplateFromPath(self): (fd, template_path) = tempfile.mkstemp(prefix="test_image_factory-") os.write(fd, self.template_xml) os.close(fd) template = Template(template_path) self.assertIsNone(template.url) self.assertIsNone(template.identifier) self.assertEqual(self.template_xml, template.xml) os.remove(template_path) def testTemplateStringRepresentation(self): template = Template(self.template_xml) self.assertEqual(self.template_xml, repr(template)) self.assertEqual(self.template_xml, str(template)) self.assertEqual(self.template_xml, "%r" % (template, )) self.assertEqual(self.template_xml, "%s" % (template, ))
class BuildDispatcher(Singleton): def _singleton_init(self): self.warehouse = ImageWarehouse( ApplicationConfiguration().configuration['warehouse']) def import_image(self, image_id, build_id, target_identifier, image_desc, target, provider): image_id = self._ensure_image(image_id, image_desc) build_id = self._ensure_build(image_id, build_id) target_image_id = self._ensure_target_image(build_id, target) provider_image_id = self._ensure_provider_image( target_image_id, provider, target_identifier) self._commit_build(image_id, build_id) return (image_id, build_id, target_image_id, provider_image_id) def build_image_for_targets(self, image_id, build_id, template, targets, job_cls=BuildJob, *args, **kwargs): if image_id and not targets: targets = self._targets_for_image_id(image_id) template = self._load_template(image_id, build_id, template) image_id = self._ensure_image_with_template(image_id, template) build_id = self._ensure_build(image_id, build_id) watcher = BuildWatcher(image_id, build_id, len(targets), self.warehouse) jobs = [] for target in targets: job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.build_image(watcher) jobs.append(job) return jobs def push_image_to_providers(self, image_id, build_id, providers, credentials, job_cls=BuildJob, *args, **kwargs): if not build_id: build_id = self._latest_unpushed(image_id) watcher = PushWatcher(image_id, build_id, len(providers), self.warehouse) jobs = [] for provider in providers: target = self._map_provider_to_target(provider) target_image_id = self._target_image_for_build_and_target( build_id, target) template = self._template_for_target_image_id(target_image_id) job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.push_image(target_image_id, provider, credentials, watcher) jobs.append(job) return jobs def _xml_node(self, xml, xpath): nodes = libxml2.parseDoc(xml).xpathEval(xpath) if not nodes: return None return nodes[0].content def _ensure_image_with_template(self, image_id, template): name = self._xml_node(template.xml, '/template/name') if name: image_desc = '<image><name>%s</name></image>' % name else: image_desc = '</image>' return self._ensure_image(image_id, image_desc) def _ensure_image(self, image_id, image_desc): if image_id: return image_id else: return self.warehouse.store_image(None, image_desc) def _ensure_build(self, image_id, build_id): if build_id: return build_id return self.warehouse.store_build(None, dict(image=image_id)) def _ensure_target_image(self, build_id, target): target_image_id = self._target_image_for_build_and_target( build_id, target) if target_image_id: return target_image_id return self.warehouse.store_target_image( None, None, dict(build=build_id, target=target)) def _ensure_provider_image(self, target_image_id, provider, target_identifier): provider_image_id = self._provider_image_for_target_image_and_provider( target_image_id, provider) if provider_image_id: self._set_provider_image_attr(provider_image_id, 'target_identifier', target_identifier) else: metadata = dict(target_image=target_image_id, provider=provider, target_identifier=target_identifier) return self.warehouse.create_provider_image(None, None, metadata) def _load_template(self, image_id, build_id, template): if not template: if build_id: template = self._template_for_build_id(build_id) if not template and image_id: template = self._template_for_image_id(image_id) return Template(template) def _commit_build(self, image_id, build_id): parent_id = self._latest_build(image_id) if parent_id: self._set_build_parent(build_id, parent_id) self._set_latest_build(image_id, build_id) def _latest_build(self, image_id): return self.warehouse.metadata_for_id_of_type(['latest_build'], image_id, 'image')['latest_build'] def _latest_unpushed(self, image_id): return self.warehouse.metadata_for_id_of_type( ['latest_unpushed'], image_id, 'image')['latest_unpushed'] def _set_latest_build(self, image_id, build_id): self.warehouse.set_metadata_for_id_of_type({'latest_build': build_id}, image_id, 'image') def _set_build_parent(self, build_id, parent_id): self.warehouse.set_metadata_for_id_of_type({'parent': parent_id}, build_id, 'build') def _targets_for_build_id(self, build_id): targets = [] for target_image_id in self._target_images_for_build(build_id): targets.append( self.warehouse.metadata_for_id_of_type( ['target'], target_image_id, 'target_image')['target']) return targets def _targets_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._targets_for_build_id(build_id) if build_id else [] def _target_images_for_build(self, build_id): return self.warehouse.query("target_image", "$build == \"%s\"" % (build_id, )) def _target_image_for_build_and_target(self, build_id, target): results = self.warehouse.query( "target_image", "$build == \"%s\" && $target == \"%s\"" % (build_id, target)) return results[0] if results else None def _provider_image_for_target_image_and_provider(self, target_image_id, provider): results = self.warehouse.query( "provider_image", "$target_image == \"%s\" && $provider == \"%s\"" % (target_image_id, provider)) return results[0] if results else None def _set_provider_image_attr(provider_image_id, attr, value): self.warehouse.set_metadata_for_id_of_type({attr: value}, provider_image_id, "provider_image") def _template_for_target_image_id(self, target_image_id): return self.warehouse.metadata_for_id_of_type( ['template'], target_image_id, 'target_image')['template'] def _template_for_build_id(self, build_id): target_image_ids = self._target_images_for_build(build_id) return self._template_for_target_image_id( target_image_ids[0]) if target_image_ids else None def _template_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._template_for_build_id(build_id) if build_id else None def _is_dynamic_provider(self, provider, filebase): provider_json = '/etc/%s.json' % (filebase) if not os.path.exists(provider_json): return False provider_sites = {} f = open(provider_json, 'r') try: provider_sites = json.loads(f.read()) finally: f.close() return provider in provider_sites # FIXME: this is a hack; conductor is the only one who really # knows this mapping, so perhaps it should provide it? # e.g. pass a provider => target dict into push_image # rather than just a list of providers. Perhaps just use # this heuristic for the command line? # # provider semantics, per target: # - ec2: region, one of ec2-us-east-1, ec2-us-west-1, ec2-ap-southeast-1, ec2-ap-northeast-1, ec2-eu-west-1 # - condorcloud: ignored # - rhevm: a key in /etc/rhevm.json and passed to op=register&site=provider # - vpshere: a key in /etc/vmware.json # - mock: any provider with 'mock' prefix # - rackspace: provider is rackspace # def _map_provider_to_target(self, provider): if provider.startswith('ec2-'): return 'ec2' elif provider == 'rackspace': return 'rackspace' elif self._is_dynamic_provider(provider, 'rhevm'): return 'rhevm' elif self._is_dynamic_provider(provider, 'vmware'): return 'vsphere' elif provider.startswith('mock'): return 'mock' else: return 'condorcloud' # condorcloud ignores provider
def setUp(self): logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) self.metadata = dict(key1="value1", key2="value2", key3="value3")
class testImageWarehouse(unittest.TestCase): def setUp(self): logging.basicConfig(level=logging.NOTSET, format='%(asctime)s %(levelname)s %(name)s pid(%(process)d) Message: %(message)s', filename='/tmp/imagefactory-unittests.log') self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) self.metadata = dict(key1="value1", key2="value2", key3="value3") def tearDown(self): del self.warehouse del self.metadata def testImageWarehouseMethods(self): """Tests CRUD operations on target images, templates, icicles, and metadata on those objects...""" # TARGET IMAGE # store a target image target_image_id = str(uuid.uuid4()) file_content = "This is just to test storing a target image in warehouse. There is not much to see here." file_path = "/tmp/testImageWarehouse_testStoreAndFetchTargetImage.%s" % (target_image_id, ) with open(file_path, 'w') as test_file: test_file.write(file_content) test_file.close() self.warehouse.store_target_image(target_image_id, file_path, metadata=self.metadata) # now fetch that target image target_image, metadata = self.warehouse.target_image_with_id(target_image_id, metadata_keys=self.metadata.keys()) # now make the assertions self.assertEqual(file_content, target_image) self.assertEqual(self.metadata, metadata) # TEMPLATE template_content = "<template>This is a test template. There is not much to see here.</template>" # store the template and let an id get assigned template_id = self.warehouse.store_template(template_content) self.assertIsNotNone(template_id) # store the template with a specified id template_id_known = str(uuid.uuid4()) template_id2 = self.warehouse.store_template(template_content, template_id_known) self.assertEqual(template_id_known, template_id2) # fetch the templates fetched_template_content, template_metadata1 = self.warehouse.template_with_id(template_id) self.assertEqual(template_content, fetched_template_content) fetched_template_content2, template_metadata2 = self.warehouse.template_with_id(template_id_known) self.assertEqual(template_content, fetched_template_content2) # set the template id for a target image and fetch it back self.warehouse.set_metadata_for_id_of_type(dict(template=template_id), target_image_id, "target_image") template_id3, fetched_template_content3, template_metadata3 = self.warehouse.template_for_target_image_id(target_image_id) self.assertEqual(template_id, template_id3) self.assertEqual(template_content, fetched_template_content3) # ICICLE icicle_content = "<icicle>This is a test icicle. There is not much to see here.</icicle>" # store the icicle and let an id get assigned icicle_id = self.warehouse.store_icicle(icicle_content) self.assertIsNotNone(icicle_id) # store the icicle with a specified id icicle_id_known = str(uuid.uuid4()) icicle_id2 = self.warehouse.store_icicle(icicle_content, icicle_id_known) self.assertEqual(icicle_id_known, icicle_id2) # fetch the icicles fetched_icicle_content, icicle_metadata1 = self.warehouse.icicle_with_id(icicle_id) self.assertEqual(icicle_content, fetched_icicle_content) fetched_icicle_content2, icicle_metadata2 = self.warehouse.icicle_with_id(icicle_id_known) self.assertEqual(icicle_content, fetched_icicle_content2) # set the icicle id for a target image and fetch it back self.warehouse.set_metadata_for_id_of_type(dict(icicle=icicle_id), target_image_id, "target_image") icicle_id3, fetched_icicle_content3, icicle_metadata3 = self.warehouse.icicle_for_target_image_id(target_image_id) self.assertEqual(icicle_id, icicle_id3) self.assertEqual(icicle_content, fetched_icicle_content3) self.assertTrue(self.warehouse.remove_target_image_with_id(target_image_id)) self.assertTrue(self.warehouse.remove_template_with_id(template_id)) self.assertTrue(self.warehouse.remove_template_with_id(template_id2)) self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id)) self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id2)) os.remove(file_path) def testImageAndBuildMethods(self): """Tests CRUD operations on images, builds and metadata on those objects...""" image_xml = '<image/>' image_id = self.warehouse.store_image(None, image_xml, self.metadata) self.assertIsNotNone(image_id) image_body, metadata = self.warehouse.object_with_id_of_type(image_id, 'image', self.metadata.keys()) self.assertEqual(image_xml, image_body) self.assertEqual(self.metadata, metadata) build_id = self.warehouse.store_build(None, self.metadata) build_body, metadata = self.warehouse.object_with_id_of_type(build_id, 'build', self.metadata.keys()) self.assertEqual('', build_body) self.assertEqual(self.metadata, metadata) ids = self.warehouse.query('build', '$object_type == "build" && $key1 == "value1"') self.assertIn(build_id, ids) def testBucketCreation(self): # self.assert_(self.warehouse.create_bucket_at_url("%s/unittests-create_bucket/%s" % (self.warehouse.url, str(uuid.uuid4())))) self.warehouse.create_bucket_at_url("%s/unittests-create_bucket" % (self.warehouse.url, )) self.assert_(self.warehouse.create_bucket_at_url("%s/unittests-create_bucket/%s" % (self.warehouse.url, time.asctime().replace(' ', '-'))))
def _singleton_init(self): super(ImageFactory, self)._singleton_init() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.qmf_object = Data(ImageFactory.qmf_schema) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"])
class ImageFactory(Singleton): # QMF schema for ImageFactory qmf_schema = Schema(SCHEMA_TYPE_DATA, "com.redhat.imagefactory", "ImageFactory") # method for building an image _build_image_method = SchemaMethod("image", desc="Build a new image for a given target cloud") _build_image_method.addArgument(SchemaProperty("template", SCHEMA_DATA_STRING, direction=DIR_IN, desc="string of xml, uuid, or url")) _build_image_method.addArgument(SchemaProperty("target", SCHEMA_DATA_STRING, direction=DIR_IN, desc="name of the cloud to target")) _build_image_method.addArgument(SchemaProperty("build_adaptor", SCHEMA_DATA_MAP, direction=DIR_OUT, desc="the QMF address of the build_adaptor instantiated")) qmf_schema.addMethod(_build_image_method) # method for building images _build_images_method = SchemaMethod("build_image", desc="Build an image for the given target clouds") _build_images_method.addArgument(SchemaProperty("image", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of an image previously built")) _build_images_method.addArgument(SchemaProperty("build", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of a previous build of the image")) _build_images_method.addArgument(SchemaProperty("template", SCHEMA_DATA_STRING, direction=DIR_IN, desc="string of xml, uuid, or url")) _build_images_method.addArgument(SchemaProperty("targets", SCHEMA_DATA_LIST, direction=DIR_IN, desc="names of the clouds to target")) _build_images_method.addArgument(SchemaProperty("build_adaptors", SCHEMA_DATA_LIST, direction=DIR_OUT, desc="the QMF addresses of the build_adaptors instantiated")) qmf_schema.addMethod(_build_images_method) # method for creating a provider_image from an image _push_image_method = SchemaMethod("provider_image", desc="Push an image to a provider.") _push_image_method.addArgument(SchemaProperty("image_id", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the uuid of an image previously built")) _push_image_method.addArgument(SchemaProperty("provider", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the name of the cloud provider, often a region")) _push_image_method.addArgument(SchemaProperty("credentials", SCHEMA_DATA_STRING, direction=DIR_IN, desc="an xml string representation of the credentials")) _push_image_method.addArgument(SchemaProperty("build_adaptor", SCHEMA_DATA_MAP, direction=DIR_OUT, desc="the QMF address of the build_adaptor instantiated")) qmf_schema.addMethod(_push_image_method) # method for pushing an image to multiple providers _push_images_method = SchemaMethod("push_image", desc="Push an image to multiple providers.") _push_images_method.addArgument(SchemaProperty("image", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of an image previously built")) _push_images_method.addArgument(SchemaProperty("build", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the UUID of a previous build of the image")) _push_images_method.addArgument(SchemaProperty("providers", SCHEMA_DATA_LIST, direction=DIR_IN, desc="the names of the cloud providers, often regions")) _push_images_method.addArgument(SchemaProperty("credentials", SCHEMA_DATA_STRING, direction=DIR_IN, desc="an xml string representation of the credentials")) _push_images_method.addArgument(SchemaProperty("build_adaptors", SCHEMA_DATA_LIST, direction=DIR_OUT, desc="the QMF addresses of the build_adaptors instantiated")) qmf_schema.addMethod(_push_images_method) # method for importing a provider image _import_image_method = SchemaMethod("import_image", desc="Import an image using a target specific image identifier") _import_image_method.addArgument(SchemaProperty("image", SCHEMA_DATA_STRING, direction=DIR_IN_OUT, desc="the UUID of an image previously built")) _import_image_method.addArgument(SchemaProperty("build", SCHEMA_DATA_STRING, direction=DIR_IN_OUT, desc="the UUID of a previous build of the image")) _import_image_method.addArgument(SchemaProperty("target_identifier", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the target specific image ID")) _import_image_method.addArgument(SchemaProperty("image_desc", SCHEMA_DATA_STRING, direction=DIR_IN, desc="an xml string description of the image")) _import_image_method.addArgument(SchemaProperty("target", SCHEMA_DATA_STRING, direction=DIR_IN, desc="name of the cloud to target")) _import_image_method.addArgument(SchemaProperty("provider", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the name of the cloud provider, often a region")) _import_image_method.addArgument(SchemaProperty("target_image", SCHEMA_DATA_STRING, direction=DIR_OUT, desc="the UUID of the target image object")) _import_image_method.addArgument(SchemaProperty("provider_image", SCHEMA_DATA_STRING, direction=DIR_OUT, desc="the UUID of the provider image object")) qmf_schema.addMethod(_import_image_method) # this method will return a representation of the object's finite state machine _states_method = SchemaMethod("instance_states", desc = "Returns a dictionary representing the finite state machine for instances.") _states_method.addArgument(SchemaProperty("class_name", SCHEMA_DATA_STRING, direction=DIR_IN, desc="the name of the class to query for instance states")) _states_method.addArgument(SchemaProperty("states", SCHEMA_DATA_STRING, direction=DIR_OUT, desc="string representation of a dictionary describing the workflow")) qmf_schema.addMethod(_states_method) @classmethod def object_states(cls): """Returns a dictionary representing the finite state machine for instances of this class.""" return {} qmf_object = props.prop("_qmf_object", "The qmf_object property.") agent = props.prop("_agent", "The property agent") def _singleton_init(self): super(ImageFactory, self)._singleton_init() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.qmf_object = Data(ImageFactory.qmf_schema) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration["warehouse"]) def __init__(self): pass def image(self,template,target): template_object = Template(template=template) build_adaptor = BuildAdaptor(template_object,target,agent=self.agent) build_adaptor.build_image() return build_adaptor def provider_image(self, image_id, provider, credentials): target_image_id = image_id image_metadata = self.warehouse.metadata_for_id_of_type(("template", "target"), target_image_id, "target_image") template_id = image_metadata["template"] target = image_metadata["target"] if (template_id and target): build_adaptor = BuildAdaptor(Template(uuid=template_id),target,agent=self.agent) build_adaptor.push_image(target_image_id, provider, credentials) return build_adaptor else: raise RuntimeError("Could not return build_adaptor!\nimage_metadata: %s\ntemplate_id: %s\ntemplate: %s\n" % (image_metadata, template_id, target)) def build_image(self, image, build, template, targets): return BuildDispatcher().build_image_for_targets(image, build, template, targets, BuildAdaptor, self.agent) def push_image(self, image, build, providers, credentials): return BuildDispatcher().push_image_to_providers(image, build, providers, credentials, BuildAdaptor, self.agent) def import_image(self, image, build, target_identifier, image_desc, target, provider): ids = BuildDispatcher().import_image(image, build, target_identifier, image_desc, target, provider) return {"image" : ids[0], "build" : ids[1], "target_image" : ids[2], "provider_image" : ids[3]} def instance_states(self, class_name): """Returns a dictionary representing the finite state machine for instances of the class specified.""" module_name = "imagefactory.qmfagent.%s" % (class_name, ) __import__(module_name) return dict(states=str(getattr(sys.modules[module_name], class_name).object_states()))