def list_builders(): collection = {"_type": "builders", "href": request.url} jobs = JobRegistry().jobs builders = [] for builder_id in jobs.keys(): job = jobs[builder_id] builders.append( { "completed": job.percent_complete, "status": job.status, "operation": job.operation, "target": job.target, "_type": "builder", "id": builder_id, "image_id": job.image_id, "build_id": job.build_id, "target_image_id": job.target_image_id, "provider": job.provider, "provider_account_identifier": job.provider_account_identifier, "href": "%s/%s" % (request.url, builder_id), } ) collection["builders"] = builders return collection
def list_builders(): collection = {'_type':'builders','href':request.url} jobs = JobRegistry().jobs builders = [] for builder_id in jobs.keys(): job = jobs[builder_id] builders.append({'completed':job.percent_complete, 'status':job.status, 'operation':job.operation, 'target':job.target, '_type':'builder', 'id':builder_id, 'href':'%s/%s' % (request.url, builder_id)}) collection['builders'] = builders return collection
def _singleton_init(self): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse']) self.job_registry = JobRegistry()
class BuildDispatcher(Singleton): def _singleton_init(self): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse']) self.job_registry = JobRegistry() def import_image(self, image_id, build_id, target_identifier, image_desc, target, provider): image_id = self._ensure_image(image_id, image_desc) build_id = self._ensure_build(image_id, build_id) target_image_id = self._ensure_target_image(build_id, target) provider_image_id = self._ensure_provider_image(target_image_id, provider, target_identifier) self._commit_build(image_id, build_id) return (image_id, build_id, target_image_id, provider_image_id) def build_image_for_targets(self, image_id, build_id, template, targets, job_cls = BuildJob, *args, **kwargs): if image_id and not targets: targets = self._targets_for_image_id(image_id) template = self._load_template(image_id, build_id, template) image_id = self._ensure_image_with_template(image_id, template) build_id = self._ensure_build(image_id, build_id) watcher = BuildWatcher(image_id, build_id, len(targets), self.warehouse) jobs = [] for target in targets: job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.build_image(watcher) jobs.append(job) self.job_registry.register(jobs) return jobs def push_image_to_providers(self, image_id, build_id, providers, credentials, job_cls = BuildJob, *args, **kwargs): if not build_id: build_id = self._latest_unpushed(image_id) watcher = PushWatcher(image_id, build_id, len(providers), self.warehouse) jobs = [] for provider in providers: target = self._map_provider_to_target(provider) target_image_id = self._target_image_for_build_and_target(build_id, target) template = self._template_for_target_image_id(target_image_id) job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.push_image(target_image_id, provider, credentials, watcher) jobs.append(job) self.job_registry.register(jobs) return jobs def get_dynamic_provider_data(self, provider): # Get provider details for RHEV-M or VSphere # First try to interpret this as an ad-hoc/dynamic provider def # If this fails, try to find it in one or the other of the config files # If this all fails return None # We use this in the builders as well so I have made it "public" try: xml_et = fromstring(provider) return xml_et.attrib except Exception as e: self.log.debug('Testing provider for XML: %s' % e) pass try: jload = json.loads(provider) return jload except ValueError as e: self.log.debug('Testing provider for JSON: %s' % e) pass rhevm_data = self._return_dynamic_provider_data(provider, "rhevm") if rhevm_data: rhevm_data['target'] = "rhevm" rhevm_data['name'] = provider return rhevm_data vsphere_data = self._return_dynamic_provider_data(provider, "vsphere") if vsphere_data: vsphere_data['target'] = "vsphere" vsphere_data['name'] = provider return vsphere_data # It is not there return None def _xml_node(self, xml, xpath): nodes = libxml2.parseDoc(xml).xpathEval(xpath) if not nodes: return None return nodes[0].content def _ensure_image_with_template(self, image_id, template, image_desc=None): if not (type(template) == Template): template = Template(template) if not image_desc: name = self._xml_node(template.xml, '/template/name') if name: image_desc = '<image><name>%s</name></image>' % name else: image_desc = '<image/>' if not template.identifier: template.identifier = self.warehouse.store_template(template.xml) return self._ensure_image(image_id, image_desc, template.identifier) def _ensure_image(self, image_id, image_desc, template_id=None): if image_id: return image_id elif(template_id): return self.warehouse.store_image(None, image_desc, dict(template=template_id)) else: return self.warehouse.store_image(None, image_desc) def _ensure_build(self, image_id, build_id): if build_id: return build_id return self.warehouse.store_build(None, dict(image = image_id)) def _ensure_target_image(self, build_id, target): target_image_id = self._target_image_for_build_and_target(build_id, target) if target_image_id: return target_image_id return self.warehouse.store_target_image(None, None, dict(build=build_id, target=target)) def _ensure_provider_image(self, target_image_id, provider, target_identifier): provider_image_id = self._provider_image_for_target_image_and_provider(target_image_id, provider) if provider_image_id: self._set_provider_image_attr(provider_image_id, 'target_identifier', target_identifier) else: metadata = dict(target_image=target_image_id, provider=provider, target_identifier=target_identifier) return self.warehouse.create_provider_image(None, None, metadata) def _load_template(self, image_id, build_id, template): if not template: if build_id: template = self._template_for_build_id(build_id) if not template and image_id: template = self._template_for_image_id(image_id) return Template(template) def _commit_build(self, image_id, build_id): parent_id = self._latest_build(image_id) if parent_id: self._set_build_parent(build_id, parent_id) self._set_latest_build(image_id, build_id) def _latest_build(self, image_id): return self.warehouse.metadata_for_id_of_type(['latest_build'], image_id, 'image')['latest_build'] def _latest_unpushed(self, image_id): return self.warehouse.metadata_for_id_of_type(['latest_unpushed'], image_id, 'image')['latest_unpushed'] def _set_latest_build(self, image_id, build_id): self.warehouse.set_metadata_for_id_of_type({'latest_build' : build_id}, image_id, 'image') def _set_build_parent(self, build_id, parent_id): self.warehouse.set_metadata_for_id_of_type({'parent' : parent_id}, build_id, 'build') def _targets_for_build_id(self, build_id): targets = [] for target_image_id in self._target_images_for_build(build_id): targets.append(self.warehouse.metadata_for_id_of_type(['target'], target_image_id, 'target_image')['target']) return targets def _targets_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._targets_for_build_id(build_id) if build_id else [] def _target_images_for_build(self, build_id): return self.warehouse.query("target_image", "$build == \"%s\"" % (build_id,)) def _target_image_for_build_and_target(self, build_id, target): results = self.warehouse.query("target_image", "$build == \"%s\" && $target == \"%s\"" % (build_id, target)) return results[0] if results else None def _provider_image_for_target_image_and_provider(self, target_image_id, provider): results = self.warehouse.query("provider_image", "$target_image == \"%s\" && $provider == \"%s\"" % (target_image_id, provider)) return results[0] if results else None def _set_provider_image_attr(self, provider_image_id, attr, value): self.warehouse.set_metadata_for_id_of_type({attr : value}, provider_image_id, "provider_image") def _template_for_target_image_id(self, target_image_id): return self.warehouse.metadata_for_id_of_type(['template'], target_image_id, 'target_image')['template'] def _template_for_build_id(self, build_id): target_image_ids = self._target_images_for_build(build_id) return self._template_for_target_image_id(target_image_ids[0]) if target_image_ids else None def _template_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._template_for_build_id(build_id) if build_id else None def _return_dynamic_provider_data(self, provider, filebase): provider_json = '/etc/imagefactory/%s.json' % (filebase) if not os.path.exists(provider_json): return False provider_sites = {} f = open(provider_json, 'r') try: provider_sites = json.loads(f.read()) finally: f.close() if provider in provider_sites: return provider_sites[provider] else: return None # FIXME: this is a hack; conductor is the only one who really # knows this mapping, so perhaps it should provide it? # e.g. pass a provider => target dict into push_image # rather than just a list of providers. Perhaps just use # this heuristic for the command line? # # provider semantics, per target: # - ec2: region, one of ec2-us-east-1, ec2-us-west-1, ec2-ap-southeast-1, ec2-ap-northeast-1, ec2-eu-west-1 # - condorcloud: ignored # - mock: any provider with 'mock' prefix # - rackspace: provider is rackspace # UPDATE - Sept 13, 2011 for dynamic providers # - vpshere: encoded in provider string or a key in /etc/vmware.json # - rhevm: encoded in provider string or a key in /etc/rhevm.json # def _map_provider_to_target(self, provider): # Check for dynamic providers first provider_data = self.get_dynamic_provider_data(provider) if provider_data: try: return provider_data['target'] except KeyError as e: self.log.debug('Provider data does not specify target!\n%s' % provider_data) raise Exception('Provider data does not specify target!\n%s' % provider_data) elif provider.startswith('ec2-'): return 'ec2' elif provider == 'rackspace': return 'rackspace' elif provider.startswith('mock'): return 'mock' else: self.log.warn('No matching provider found for %s, using "condorcloud" by default.' % (provider)) return 'condorcloud' # condorcloud ignores provider
class BuildDispatcher(Singleton): def _singleton_init(self): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.warehouse = ImageWarehouse(ApplicationConfiguration().configuration['warehouse']) self.job_registry = JobRegistry() self.builders = dict() ##### BEGIN new BuildDispatcher code for plugins and no warehouse ##### def builder_for_base_image(self, template, parameters=None): builder = Builder() builder.build_image_from_template(template) self.builders[builder.base_image.identifier] = builder return builder def builder_for_target_image(self, target, image_id=None, template=None, parameters=None): builder = Builder() builder.customize_image_for_target(target, image_id, template, parameters) self.builders[builder.target_image.identifier] = builder return builder def builder_for_provider_image(self, provider, credentials, target, image_id=None, template=None, parameters=None): builder = Builder() builder.create_image_on_provider(provider, credentials, target, image_id, template, parameters) self.builders[builder.provider_image.identifier] = builder return builder ##### END new BuildDispatcher code def import_image(self, image_id, build_id, target_identifier, image_desc, target, provider): image_id = self._ensure_image(image_id, image_desc) build_id = self._ensure_build(image_id, build_id) target_image_id = self._ensure_target_image(build_id, target) provider_image_id = self._ensure_provider_image(target_image_id, provider, target_identifier) self._commit_build(image_id, build_id) return (image_id, build_id, target_image_id, provider_image_id) def build_image_for_targets(self, image_id, build_id, template, targets, job_cls = BuildJob, *args, **kwargs): if image_id and not targets: targets = self._targets_for_image_id(image_id) template = self._load_template(image_id, build_id, template) image_id = self._ensure_image_with_template(image_id, template) build_id = self._ensure_build(image_id, build_id) watcher = BuildWatcher(image_id, build_id, len(targets), self.warehouse) jobs = [] for target in targets: job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.build_image(watcher) jobs.append(job) self.job_registry.register(jobs) return jobs def push_image_to_providers(self, image_id, build_id, providers, credentials, job_cls = BuildJob, *args, **kwargs): if not build_id: build_id = self._latest_unpushed(image_id) watcher = PushWatcher(image_id, build_id, len(providers), self.warehouse) jobs = [] for provider in providers: target = Provider.map_provider_to_target(provider) target_image_id = self._target_image_for_build_and_target(build_id, target) template = self._template_for_target_image_id(target_image_id) job = job_cls(template, target, image_id, build_id, *args, **kwargs) job.push_image(target_image_id, provider, credentials, watcher) jobs.append(job) self.job_registry.register(jobs) return jobs def _xml_node(self, xml, xpath): nodes = libxml2.parseDoc(xml).xpathEval(xpath) if not nodes: return None return nodes[0].content def _ensure_image_with_template(self, image_id, template, image_desc=None): if not (type(template) == Template): template = Template(template) if not image_desc: name = self._xml_node(template.xml, '/template/name') if name: image_desc = '<image><name>%s</name></image>' % name else: image_desc = '<image/>' if not template.identifier: template.identifier = self.warehouse.store_template(template.xml) return self._ensure_image(image_id, image_desc, template.identifier) def _ensure_image(self, image_id, image_desc, template_id=None): if image_id: return image_id elif(template_id): return self.warehouse.store_image(None, image_desc, dict(template=template_id)) else: return self.warehouse.store_image(None, image_desc) def _ensure_build(self, image_id, build_id): if build_id: return build_id return self.warehouse.store_build(None, dict(image = image_id)) def _ensure_target_image(self, build_id, target): target_image_id = self._target_image_for_build_and_target(build_id, target) if target_image_id: return target_image_id return self.warehouse.store_target_image(None, None, dict(build=build_id, target=target)) def _ensure_provider_image(self, target_image_id, provider, target_identifier): provider_image_id = self._provider_image_for_target_image_and_provider(target_image_id, provider) if provider_image_id: self._set_provider_image_attr(provider_image_id, 'target_identifier', target_identifier) else: metadata = dict(target_image=target_image_id, provider=provider, target_identifier=target_identifier) return self.warehouse.create_provider_image(None, None, metadata) def _load_template(self, image_id, build_id, template): if not template: if build_id: template = self._template_for_build_id(build_id) if not template and image_id: template = self._template_for_image_id(image_id) return Template(template) def _commit_build(self, image_id, build_id): parent_id = self._latest_build(image_id) if parent_id: self._set_build_parent(build_id, parent_id) self._set_latest_build(image_id, build_id) def _latest_build(self, image_id): return self.warehouse.metadata_for_id_of_type(['latest_build'], image_id, 'image')['latest_build'] def _latest_unpushed(self, image_id): return self.warehouse.metadata_for_id_of_type(['latest_unpushed'], image_id, 'image')['latest_unpushed'] def _set_latest_build(self, image_id, build_id): self.warehouse.set_metadata_for_id_of_type({'latest_build' : build_id}, image_id, 'image') def _set_build_parent(self, build_id, parent_id): self.warehouse.set_metadata_for_id_of_type({'parent' : parent_id}, build_id, 'build') def _targets_for_build_id(self, build_id): targets = [] for target_image_id in self._target_images_for_build(build_id): targets.append(self.warehouse.metadata_for_id_of_type(['target'], target_image_id, 'target_image')['target']) return targets def _targets_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._targets_for_build_id(build_id) if build_id else [] def _target_images_for_build(self, build_id): return self.warehouse.query("target_image", "$build == \"%s\"" % (build_id,)) def _target_image_for_build_and_target(self, build_id, target): results = self.warehouse.query("target_image", "$build == \"%s\" && $target == \"%s\"" % (build_id, target)) return results[0] if results else None def _provider_image_for_target_image_and_provider(self, target_image_id, provider): results = self.warehouse.query("provider_image", "$target_image == \"%s\" && $provider == \"%s\"" % (target_image_id, provider)) return results[0] if results else None def _set_provider_image_attr(self, provider_image_id, attr, value): self.warehouse.set_metadata_for_id_of_type({attr : value}, provider_image_id, "provider_image") def _template_for_target_image_id(self, target_image_id): return self.warehouse.metadata_for_id_of_type(['template'], target_image_id, 'target_image')['template'] def _template_for_build_id(self, build_id): target_image_ids = self._target_images_for_build(build_id) return self._template_for_target_image_id(target_image_ids[0]) if target_image_ids else None def _template_for_image_id(self, image_id): build_id = self._latest_build(image_id) if not build_id: build_id = self._latest_unpushed(image_id) return self._template_for_build_id(build_id) if build_id else None