def create_docker_vm(self, group_id, provider_id, version, date, pull_url): group = Group.objects.get(id=group_id) provider = Provider.objects.get(id=provider_id, working=True, disabled=False) with transaction.atomic(): if provider.remaining_configuring_slots < 1: self.retry(args=(group_id, provider_id, version, date, pull_url), countdown=60, max_retries=60) new_name = docker_vm_name(version, date) new_template = Template(template_group=group, provider=provider, container='cfme', name=new_name, original_name=provider.container_base_template, version=version, date=date, ready=False, exists=False, usable=True, preconfigured=True, template_type=Template.DOCKER_VM) new_template.save() workflow = chain( prepare_template_deploy.si(new_template.id), configure_docker_template.si(new_template.id, pull_url), prepare_template_seal.si(new_template.id), prepare_template_poweroff.si(new_template.id), prepare_template_finish.si(new_template.id), ) workflow.link_error(prepare_template_delete_on_error.si(new_template.id)) workflow()
def poke_trackerbot(self): """This beat-scheduled task periodically polls the trackerbot if there are any new templates. """ template_usability = [] # Extract data from trackerbot tbapi = trackerbot() objects = depaginate( tbapi, tbapi.providertemplate().get(limit=TRACKERBOT_PAGINATE))["objects"] per_group = {} for obj in objects: if obj["template"]["group"]["name"] == 'unknown': continue if obj["template"]["group"]["name"] not in per_group: per_group[obj["template"]["group"]["name"]] = [] per_group[obj["template"]["group"]["name"]].append(obj) # Sort them using the build date for group in list(per_group.keys()): per_group[group] = sorted(per_group[group], reverse=True, key=lambda o: o["template"]["datestamp"]) objects = [] # And interleave the the groups while any(per_group.values()): for key in list(per_group.keys()): if per_group[key]: objects.append(per_group[key].pop(0)) for template in objects: if template["provider"]["key"] not in list( conf.cfme_data.management_systems.keys()): # If we don't use that provider in yamls, set the template as not usable # 1) It will prevent adding this template if not added # 2) It'll mark the template as unusable if it already exists template["usable"] = False template_usability.append( (template["provider"]["key"], template["template"]["name"], template["usable"])) if not template["usable"]: continue group, create = Group.objects.get_or_create( id=template["template"]["group"]["name"]) # Check if the template is already obsolete if group.template_obsolete_days is not None: build_date = parsetime.from_iso_date( template["template"]["datestamp"]) if build_date <= (parsetime.today() - timedelta(days=group.template_obsolete_days)): # It is already obsolete, so ignore it continue if conf.cfme_data.management_systems.get( template["provider"]["key"], {}).get( "use_for_sprout", False ): # only create provider in db if it is marked to use for sprout provider, create = Provider.objects.get_or_create( id=template["provider"]["key"]) else: continue if not provider.is_working: continue if "sprout" not in provider.provider_data: continue if not provider.provider_type: provider.provider_type = provider.provider_data.get('type') provider.save(update_fields=['provider_type']) template_name = template["template"]["name"] ga_released = template['template']['ga_released'] custom_data = template['template'].get('custom_data', "{}") processed_custom_data = yaml.safe_load(custom_data) template_info = TemplateName.parse_template(template_name) if not template_info.datestamp: # Not a CFME/MIQ template, ignore it. continue # Original one original_template = None try: original_template = Template.objects.get( provider=provider, template_group=group, original_name=template_name, name=template_name, preconfigured=False) if original_template.ga_released != ga_released: original_template.ga_released = ga_released original_template.save(update_fields=['ga_released']) if provider.provider_type == 'openshift': if original_template.custom_data != custom_data: original_template.custom_data = processed_custom_data original_template.template_type = Template.OPENSHIFT_POD original_template.container = 'cloudforms-0' original_template.save(update_fields=[ 'custom_data', 'container', 'template_type' ]) except ObjectDoesNotExist: if template_name in provider.templates: if template_info.datestamp is None: self.logger.warning( "Ignoring template {} because it does not have a date!" .format(template_name)) continue template_version = template_info.version if template_version is None: # Make up a faux version # First 3 fields of version get parsed as a zstream # therefore ... makes it a "nil" stream template_version = "...{}".format( template_info.datestamp.strftime("%Y%m%d")) # openshift has two templates bound to one build version # sometimes sprout tries using second template -extdb that's wrong # so, such template has to be marked as not usable inside sprout usable = not template_name.endswith('-extdb') with transaction.atomic(): tpl = Template(provider=provider, template_group=group, original_name=template_name, name=template_name, preconfigured=False, date=template_info.datestamp, ready=True, exists=True, usable=usable, version=template_version) tpl.save() if provider.provider_type == 'openshift': tpl.custom_data = processed_custom_data tpl.container = 'cloudforms-0' tpl.template_type = Template.OPENSHIFT_POD tpl.save(update_fields=[ 'container', 'template_type', 'custom_data' ]) original_template = tpl self.logger.info("Created a new template #{}".format( tpl.id)) # If the provider is set to not preconfigure templates, do not bother even doing it. if provider.num_simultaneous_configuring > 0: # Preconfigured one try: # openshift providers don't have preconfigured templates. # so regular template should be used if provider.provider_type != 'openshift': preconfigured_template = Template.objects.get( provider=provider, template_group=group, original_name=template_name, preconfigured=True) else: preconfigured_template = Template.objects.get( provider=provider, template_group=group, name=template_name, preconfigured=True) preconfigured_template.custom_data = processed_custom_data preconfigured_template.container = 'cloudforms-0' preconfigured_template.template_type = Template.OPENSHIFT_POD preconfigured_template.save(update_fields=[ 'container', 'template_type', 'custom_data' ]) if preconfigured_template.ga_released != ga_released: preconfigured_template.ga_released = ga_released preconfigured_template.save(update_fields=['ga_released']) except ObjectDoesNotExist: if template_name in provider.templates and provider.provider_type != 'openshift': original_id = original_template.id if original_template is not None else None create_appliance_template.delay( provider.id, group.id, template_name, source_template_id=original_id) # If any of the templates becomes unusable, let sprout know about it # Similarly if some of them becomes usable ... for provider_id, template_name, usability in template_usability: if conf.cfme_data.management_systems.get(provider_id, {}).get( "use_for_sprout", False ): # only create provider in db if it is marked to use for sprout provider, create = Provider.objects.get_or_create(id=provider_id) else: continue if not provider.working or provider.disabled: continue with transaction.atomic(): for template in Template.objects.filter( provider=provider, original_name=template_name): template.usable = usability template.save(update_fields=['usable']) # Kill all shepherd appliances if they were accidentally spun up if not usability: for appliance in Appliance.objects.filter( template=template, marked_for_deletion=False, appliance_pool=None): self.logger.info('Killing an appliance {}/{} ' 'because its template was marked ' 'as unusable'.format( appliance.id, appliance.name)) Appliance.kill(appliance)
def create_appliance_template(self, provider_id, group_id, template_name, source_template_id=None): """This task creates a template from a fresh CFME template. In case of fatal error during the operation, the template object is deleted to make sure the operation will be retried next time when poke_trackerbot runs.""" provider = Provider.objects.get(id=provider_id, working=True, disabled=False) provider.cleanup() # Precaution group = Group.objects.get(id=group_id) with transaction.atomic(): # Limit the number of concurrent template configurations if provider.remaining_configuring_slots == 0: return False # It will be kicked again when trackerbot gets poked try: Template.objects.get(template_group=group, provider=provider, original_name=template_name, preconfigured=True) return False except ObjectDoesNotExist: pass # Fire off the template preparation template_info = TemplateName.parse_template(template_name) template_date_fmt = template_info.datestamp.strftime("%Y%m%d") if not template_info.datestamp: return # Make up a faux version # First 3 fields of version get parsed as a zstream # therefore ... makes it a "nil" stream template_version = template_info.version or "...{}".format( template_date_fmt) new_template_name = settings.TEMPLATE_FORMAT.format( group=group.id, date=template_date_fmt, rnd=fauxfactory.gen_alphanumeric(8)) if provider.template_name_length is not None: allowed_length = provider.template_name_length # There is some limit if len(new_template_name) > allowed_length: # Cut it down randoms_length = len(new_template_name.rsplit("_", 1)[-1]) minimum_length = (len(new_template_name) - randoms_length) + 1 # One random must be if minimum_length <= allowed_length: # Just cut it new_template_name = new_template_name[:allowed_length] else: # Another solution new_template_name = settings.TEMPLATE_FORMAT.format( group=group.id[:2], date=template_date_fmt, # Use only first 2 of grp rnd=fauxfactory.gen_alphanumeric( 2)) # And just 2 chars random # TODO: If anything larger comes, do fix that! if source_template_id is not None: try: source_template = Template.objects.get(id=source_template_id) except ObjectDoesNotExist: source_template = None else: source_template = None template = Template(provider=provider, template_group=group, name=new_template_name, date=template_info.datestamp, version=template_version, original_name=template_name, parent_template=source_template, exists=False) template.save() workflow = chain( prepare_template_deploy.si(template.id), prepare_template_verify_version.si(template.id), prepare_template_configure.si(template.id), prepare_template_seal.si(template.id), prepare_template_poweroff.si(template.id), prepare_template_finish.si(template.id), ) workflow.link_error(prepare_template_delete_on_error.si(template.id)) workflow()