def kill(self, request, appliances): for appliance in appliances: Appliance.kill(appliance) self.message_user(request, "Initiated kill of {}.".format(appliance.name)) self.logger.info( "User {}/{} requested kill of appliance {}".format( request.user.pk, request.user.username, appliance.id))
def clone_template(self, template_id): self.logger.info("Cloning template {}".format(template_id)) template = Template.objects.get(id=template_id) new_appliance_name = gen_appliance_name(template_id) appliance = Appliance(template=template, name=new_appliance_name) appliance.save() clone_template_to_appliance.delay(appliance.id)
def kill_appliance(request, appliance_id): if not request.user.is_authenticated(): return go_home(request) try: appliance = Appliance.objects.get(id=appliance_id) except ObjectDoesNotExist: messages.error(request, 'Appliance with ID {} does not exist!.'.format(appliance_id)) return go_back_or_home(request) if not can_operate_appliance_or_pool(appliance, request.user): messages.error(request, 'This appliance belongs either to some other user or nobody.') return go_back_or_home(request) Appliance.kill(appliance) messages.success(request, 'Kill initiated.') return go_back_or_home(request)
def process_delayed_provision_tasks(self): """This picks up the provisioning tasks that were delayed due to ocncurrency limit of provision. Goes one task by one and when some of them can be provisioned, it starts the provisioning and then deletes the task. """ for task in DelayedProvisionTask.objects.order_by("id"): if task.pool.not_needed_anymore: task.delete() continue # Try retrieve from shepherd appliances_given = Appliance.give_to_pool(task.pool, 1) if appliances_given == 0: # No free appliance in shepherd, so do it on our own tpls = task.pool.possible_provisioning_templates if task.provider_to_avoid is not None: filtered_tpls = [ tpl for tpl in tpls if tpl.provider != task.provider_to_avoid ] if filtered_tpls: # There are other providers to provision on, so try one of them tpls = filtered_tpls # If there is no other provider to provision on, we will use the original list. # This will cause additional rejects until the provider quota is met if tpls: clone_template_to_pool(tpls[0].id, task.pool.id, task.lease_time) task.delete() else: # Try freeing up some space in provider for provider in task.pool.possible_providers: appliances = provider.free_shepherd_appliances.exclude( **task.pool.appliance_filter_params) if appliances: appl = random.choice(appliances) self.logger.info('Freeing some space in provider by ' 'killing appliance {}/{}'.format( appl.id, appl.name)) Appliance.kill(appl) break # Just one else: # There was a free appliance in shepherd, so we took it and we don't need this task more task.delete()
def destroy_appliance(user, appliance): """Destroy the appliance. If the kill task was called, id is returned, otherwise None You can specify appliance by IP address, id or name. """ appliance = get_appliance(appliance, user) try: return Appliance.kill(appliance).task_id except AttributeError: # None was returned return None
def delete_nonexistent_appliances(self): """Goes through orphaned appliances' objects and deletes them from the database.""" expiration_time = (timezone.now() - timedelta(**settings.ORPHANED_APPLIANCE_GRACE_TIME)) for appliance in Appliance.objects.filter(ready=True).all(): if appliance.name in redis.renaming_appliances: continue if appliance.power_state == Appliance.Power.ORPHANED: if appliance.power_state_changed > expiration_time: # Ignore it for now self.logger.info("orphaned appliance {} power " "state has been changed".format(appliance.id)) continue self.logger.info( "I will delete orphaned " "appliance {}/{} on provider {}".format(appliance.id, appliance.name, appliance.provider.id)) try: appliance.delete() except ObjectDoesNotExist as e: if "AppliancePool" in str(e): # Someone managed to delete the appliance pool before appliance.appliance_pool = None appliance.save(update_fields=['appliance_pool']) appliance.delete() else: raise # No diaper pattern here! # If something happened to the appliance provisioning process, just delete it to remove # the garbage. It will be respinned again by shepherd. # Grace time is specified in BROKEN_APPLIANCE_GRACE_TIME expiration_time = (timezone.now() - timedelta(**settings.BROKEN_APPLIANCE_GRACE_TIME)) for appliance in Appliance.objects.filter(ready=False, marked_for_deletion=False).all(): if appliance.status_changed < expiration_time: self.logger.info("Killing broken appliance {}/{} " "on provider {}".format(appliance.id, appliance.name, appliance.provider.id)) Appliance.kill(appliance) # Use kill because the appliance may still exist # And now - if something happened during appliance deletion, call kill again for appliance in Appliance.objects.filter( marked_for_deletion=True, status_changed__lt=expiration_time).all(): self.logger.info( "Trying to kill unkilled appliance {}/{} " "on provider {}".format(appliance.id, appliance.name, appliance.provider.id)) Appliance.kill(appliance, force_delete=True)
def destroy_appliance(user, appliance): """Destroy the appliance. If the kill task was called, id is returned, otherwise None You can specify appliance by IP address, id or name. """ appliance = get_appliance(appliance) if appliance.owner is None: if not user.is_staff: raise Exception("Only staff can operate with nonowned appliances") elif appliance.owner != user: raise Exception("This appliance belongs to a different user!") try: return Appliance.kill(appliance).task_id except AttributeError: # None was returned return None
def command(csv_file): csv_reader = csv.reader(csv_file) appliances = [ Appliance( product_line=ProductLine.objects.get_or_create( name=product_line)[0], serial_number=serial_number, model_number=model_number, ) for product_line, model_number, serial_number in csv_reader ] print("Here are the appliances extracted:") pprint(appliances) if click.confirm("Proceed?"): Appliance.objects.bulk_create(appliances)
def apply_lease_times_after_pool_fulfilled(self, appliance_pool_id, time_minutes): try: pool = AppliancePool.objects.get(id=appliance_pool_id) except ObjectDoesNotExist as e: self.logger.error("It seems such appliance pool %s doesn't exist: %s", appliance_pool_id, str(e)) return False if pool.fulfilled: pool.logger.info("Applying lease time and renaming appliances") for appliance in pool.appliances: apply_lease_times.delay(appliance.id, time_minutes) with transaction.atomic(): pool.finished = True pool.save(update_fields=['finished']) pool.logger.info( "Pool {} setup is finished".format(appliance_pool_id)) else: # Look whether we can swap any provisioning appliance with some in shepherd pool.logger.info("Pool isn't fulfilled yet") unfinished = list( Appliance.objects.filter(appliance_pool=pool, ready=False, marked_for_deletion=False).all()) random.shuffle(unfinished) if len(unfinished) > 0: pool.logger.info('There are %s unfinished appliances', len(unfinished)) n = Appliance.give_to_pool(pool, len(unfinished)) with transaction.atomic(): for _ in range(n): appl = unfinished.pop() appl.appliance_pool = None appl.save(update_fields=['appliance_pool']) try: pool.logger.info("Retrying to apply lease again") self.retry(args=(appliance_pool_id, time_minutes), countdown=30, max_retries=120) except MaxRetriesExceededError: # Bad luck, pool fulfillment failed. So destroy it. pool.logger.error( "Waiting for fulfillment failed. Initiating the destruction process." ) pool.kill()
def request_appliance_pool(self, appliance_pool_id, time_minutes): """This task gives maximum possible amount of spinned-up appliances to the specified pool and then if there is need to spin up another appliances, it spins them up via clone_template_to_pool task.""" self.logger.info("Appliance pool {} requested for {} minutes.".format( appliance_pool_id, time_minutes)) pool = AppliancePool.objects.get(id=appliance_pool_id) n = Appliance.give_to_pool(pool) for i in range(pool.total_count - n): tpls = pool.possible_provisioning_templates if tpls: template_id = tpls[0].id clone_template_to_pool(template_id, pool.id, time_minutes) else: with transaction.atomic(): task = DelayedProvisionTask(pool=pool, lease_time=time_minutes) task.save() apply_lease_times_after_pool_fulfilled.delay(appliance_pool_id, time_minutes)
def clone_template_to_pool(template_id, appliance_pool_id, time_minutes): template = Template.objects.get(id=template_id) with transaction.atomic(): pool = AppliancePool.objects.get(id=appliance_pool_id) if pool.not_needed_anymore: return new_appliance_name = gen_appliance_name(template_id, username=pool.owner.username) appliance = Appliance(template=template, name=new_appliance_name, appliance_pool=pool) appliance.save() appliance.set_lease_time() # Set pool to these params to keep the appliances with same versions/dates pool.version = template.version pool.date = template.date pool.save(update_fields=['version', 'date']) clone_template_to_appliance.delay(appliance.id, time_minutes, pool.yum_update)
def appliance_action(request, appliance_id, action, x=None): try: appliance = Appliance.objects.get(id=appliance_id) except ObjectDoesNotExist: messages.warning( request, 'Appliance with ID {} does not exist!.'.format(appliance_id)) return go_back_or_home(request) if not can_operate_appliance_or_pool(appliance, request.user): messages.warning( request, 'This appliance belongs either to some other user or nobody.') return go_back_or_home(request) if action == "start": if appliance.power_state != Appliance.Power.ON: chain(appliance_power_on.si(appliance.id), (wait_appliance_ready if appliance.preconfigured else mark_appliance_ready).si(appliance.id))() messages.success(request, 'Initiated launch of appliance.') return go_back_or_home(request) else: messages.info(request, 'Appliance was already powered on.') return go_back_or_home(request) elif action == "reboot": if appliance.power_state == Appliance.Power.ON: chain(appliance_reboot.si(appliance.id), mark_appliance_ready.si(appliance.id))() messages.success(request, 'Initiated reboot of appliance.') return go_back_or_home(request) else: messages.warning(request, 'Only powered on appliances can be rebooted') return go_back_or_home(request) elif action == "stop": if appliance.power_state != Appliance.Power.OFF: appliance_power_off.delay(appliance.id) messages.success(request, 'Initiated stop of appliance.') return go_back_or_home(request) else: messages.info(request, 'Appliance was already powered off.') return go_back_or_home(request) elif action == "suspend": if appliance.power_state != Appliance.Power.SUSPENDED: appliance_suspend.delay(appliance.id) messages.success(request, 'Initiated suspend of appliance.') return go_back_or_home(request) else: messages.info(request, 'Appliance was already suspended.') return go_back_or_home(request) elif action == "kill": Appliance.kill(appliance) messages.success(request, 'Kill initiated.') return go_back_or_home(request) elif action == "dont_expire": if not request.user.is_superuser: messages.warning( request, 'Disabling expiration time is allowed only for superusers.') return go_back_or_home(request) with transaction.atomic(): appliance.leased_until = None appliance.save() messages.success(request, 'Lease disabled successfully. Be careful.') return go_back_or_home(request) elif action == "set_lease": if not can_operate_appliance_or_pool(appliance, request.user): messages.warning( request, 'This appliance belongs either to some other user or nobody.') return go_back_or_home(request) appliance.prolong_lease(time=int(x)) messages.success(request, 'Lease prolonged successfully.') return go_back_or_home(request) else: messages.warning(request, "Unknown action '{}'".format(action))
def appliance_action(request, appliance_id, action, x=None): try: appliance = Appliance.objects.get(id=appliance_id) except ObjectDoesNotExist: messages.warning(request, 'Appliance with ID {} does not exist!.'.format(appliance_id)) return go_back_or_home(request) if not can_operate_appliance_or_pool(appliance, request.user): messages.warning(request, 'This appliance belongs either to some other user or nobody.') return go_back_or_home(request) if action == "start": if appliance.power_state != Appliance.Power.ON: chain( appliance_power_on.si(appliance.id), (wait_appliance_ready if appliance.preconfigured else mark_appliance_ready).si( appliance.id))() messages.success(request, 'Initiated launch of appliance.') return go_back_or_home(request) else: messages.info(request, 'Appliance was already powered on.') return go_back_or_home(request) elif action == "reboot": if appliance.power_state == Appliance.Power.ON: chain( appliance_reboot.si(appliance.id), mark_appliance_ready.si(appliance.id))() messages.success(request, 'Initiated reboot of appliance.') return go_back_or_home(request) else: messages.warning(request, 'Only powered on appliances can be rebooted') return go_back_or_home(request) elif action == "stop": if appliance.power_state != Appliance.Power.OFF: appliance_power_off.delay(appliance.id) messages.success(request, 'Initiated stop of appliance.') return go_back_or_home(request) else: messages.info(request, 'Appliance was already powered off.') return go_back_or_home(request) elif action == "suspend": if appliance.power_state != Appliance.Power.SUSPENDED: appliance_suspend.delay(appliance.id) messages.success(request, 'Initiated suspend of appliance.') return go_back_or_home(request) else: messages.info(request, 'Appliance was already suspended.') return go_back_or_home(request) elif action == "kill": Appliance.kill(appliance) messages.success(request, 'Kill initiated.') return go_back_or_home(request) elif action == "dont_expire": if not request.user.is_superuser: messages.warning(request, 'Disabling expiration time is allowed only for superusers.') return go_back_or_home(request) with transaction.atomic(): appliance.leased_until = None appliance.save() messages.success(request, 'Lease disabled successfully. Be careful.') return go_back_or_home(request) elif action == "set_lease": if not can_operate_appliance_or_pool(appliance, request.user): messages.warning(request, 'This appliance belongs either to some other user or nobody.') return go_back_or_home(request) appliance.prolong_lease(time=int(x)) messages.success(request, 'Lease prolonged successfully.') return go_back_or_home(request) else: messages.warning(request, "Unknown action '{}'".format(action))
def newHw(request): if request.POST: txt_hwid = request.POST.get('txt_hwid').strip() txt_psw = request.POST.get('txt_sn').strip() txt_title = request.POST.get('txt_title').strip() hwtype_radio = request.POST.get('hwtype') import re reg = re.compile(r'^[\w\s]+$') if reg.search(txt_hwid) is None: messages.info( request, 'Please use only alphanumeric symbols and whitespaces for hwid.' ) return render(request, 'hardware/newhw.html') if reg.search(txt_psw) is None: messages.info( request, 'Please use only alphanumeric symbols and whitespaces for serial number.' ) return render(request, 'hardware/newhw.html') if reg.search(txt_title) is None: messages.info( request, 'Please use only alphanumeric symbols and whitespaces for title.' ) return render(request, 'hardware/newhw.html') from django.db import IntegrityError if hwtype_radio is not None: if hwtype_radio == '0': from appliances.models import Appliance, ApplianceChore try: a = Appliance() a.hwid = txt_hwid a.password = txt_psw a.name = txt_title a.generatePad() a.save() ac = ApplianceChore() ac.appliance = a ac.name = a.name + " Chore" ac.save() except IntegrityError: messages.error(request, 'Appliance with this id already exists.') return render(request, 'hardware/newhw.html') else: messages.info(request, a.getName() + ' successfully registered.') return redirect('hardware:hwlist') elif hwtype_radio == '1': from dishes.models import ItemTracker try: h = ItemTracker() h.hwid = txt_hwid h.password = txt_psw h.name = txt_title h.generatePad() h.save() except IntegrityError: messages.error( request, 'Item tracker with this id already exists.') return render(request, 'hardware/newhw.html') else: messages.info(request, h.getName() + ' successfully registered.') return redirect('hardware:hwlist') else: messages.info(request, 'Please select hardware type') return render(request, 'hardware/newhw.html') return render(request, 'hardware/newhw.html')
def newHw(request): if request.POST: txt_hwid = request.POST.get('txt_hwid').strip() txt_psw = request.POST.get('txt_sn').strip() txt_title = request.POST.get('txt_title').strip() hwtype_radio = request.POST.get('hwtype') import re reg = re.compile(r'^[\w\s]+$') if reg.search(txt_hwid) is None: messages.info(request, 'Please use only alphanumeric symbols and whitespaces for hwid.') return render(request, 'hardware/newhw.html') if reg.search(txt_psw) is None: messages.info(request, 'Please use only alphanumeric symbols and whitespaces for serial number.') return render(request, 'hardware/newhw.html') if reg.search(txt_title) is None: messages.info(request, 'Please use only alphanumeric symbols and whitespaces for title.') return render(request, 'hardware/newhw.html') from django.db import IntegrityError if hwtype_radio is not None: if hwtype_radio == '0': from appliances.models import Appliance, ApplianceChore try: a = Appliance() a.hwid = txt_hwid a.password = txt_psw a.name = txt_title a.generatePad() a.save() ac = ApplianceChore() ac.appliance = a ac.name = a.name + " Chore" ac.save() except IntegrityError: messages.error(request, 'Appliance with this id already exists.') return render(request, 'hardware/newhw.html') else: messages.info(request, a.getName() + ' successfully registered.') return redirect('hardware:hwlist') elif hwtype_radio == '1': from dishes.models import ItemTracker try: h = ItemTracker() h.hwid = txt_hwid h.password = txt_psw h.name = txt_title h.generatePad() h.save() except IntegrityError: messages.error(request, 'Item tracker with this id already exists.') return render(request, 'hardware/newhw.html') else: messages.info(request, h.getName() + ' successfully registered.') return redirect('hardware:hwlist') else: messages.info(request, 'Please select hardware type') return render(request, 'hardware/newhw.html') return render(request, 'hardware/newhw.html')
def synchronize_untracked_vms_in_provider(self, provider_id): """'re'-synchronizes any vms that might be lost during outages.""" provider = Provider.objects.get(id=provider_id, working=True, disabled=False) provider_api = provider.api if not hasattr(provider_api, 'list_vms'): # This provider does not have VMs return for vm in sorted(provider_api.list_vms(), key=lambda pvm: getattr(pvm, 'name', pvm)): if ( Appliance.objects.filter(name=getattr(vm, 'name', vm), template__provider=provider).count() != 0 ): continue # We have an untracked VM. Let's investigate try: appliance_id = vm.get_meta_value('sprout_id') except KeyError: continue except (AttributeError, NotImplementedError): # Do not bother if not implemented in the VM object's API return # just check it again ... if Appliance.objects.filter(id=appliance_id).count() == 1: # For some reason it is already in continue # Now it appears that this is a VM that was in Sprout construct = {'id': appliance_id} # Retrieve appliance data try: self.logger.info('Trying to reconstruct appliance %d/%s', appliance_id, vm.name) construct['name'] = vm.name template_id = vm.get_meta_value('sprout_source_template_id') # Templates are not deleted from the DB so this should be OK. construct['template'] = Template.objects.get(id=template_id) construct['name'] = vm.name construct['ready'] = vm.get_meta_value('sprout_ready') construct['description'] = vm.get_meta_value('sprout_description') construct['lun_disk_connected'] = vm.get_meta_value('sprout_lun_disk_connected') construct['swap'] = vm.get_meta_value('sprout_swap') construct['ssh_failed'] = vm.get_meta_value('sprout_ssh_failed') # Time fields construct['datetime_leased'] = parsedate(vm.get_meta_value('sprout_datetime_leased')) construct['leased_until'] = parsedate(vm.get_meta_value('sprout_leased_until')) construct['status_changed'] = parsedate(vm.get_meta_value('sprout_status_changed')) construct['created_on'] = parsedate(vm.get_meta_value('sprout_created_on')) construct['modified_on'] = parsedate(vm.get_meta_value('sprout_modified_on')) except KeyError as e: self.logger.error('Failed to reconstruct %d/%s', appliance_id, vm.name) self.logger.exception(e) continue # Retrieve pool data if applicable try: pool_id = vm.get_meta_value('sprout_pool_id') pool_construct = dict(id=pool_id) pool_construct['total_count'] = vm.get_meta_value('sprout_pool_total_count') group_id = vm.get_meta_value('sprout_pool_group') pool_construct['group'] = Group.objects.get(id=group_id) try: construct_provider_id = vm.get_meta_value('sprout_pool_provider') pool_construct['provider'] = Provider.objects.get(id=construct_provider_id) except (KeyError, ObjectDoesNotExist): # optional pool_construct['provider'] = None pool_construct['version'] = vm.get_meta_value('sprout_pool_version') pool_construct['date'] = parsedate(vm.get_meta_value('sprout_pool_appliance_date')) owner_id = vm.get_meta_value('sprout_pool_owner_id') try: owner = User.objects.get(id=owner_id) except ObjectDoesNotExist: owner_username = vm.get_meta_value('sprout_pool_owner_username') owner = User(id=owner_id, username=owner_username) owner.save() pool_construct['owner'] = owner pool_construct['preconfigured'] = vm.get_meta_value('sprout_pool_preconfigured') pool_construct['description'] = vm.get_meta_value('sprout_pool_description') pool_construct['not_needed_anymore'] = vm.get_meta_value( 'sprout_pool_not_needed_anymore') pool_construct['finished'] = vm.get_meta_value('sprout_pool_finished') pool_construct['yum_update'] = vm.get_meta_value('sprout_pool_yum_update') try: construct['appliance_pool'] = AppliancePool.objects.get(id=pool_id) except ObjectDoesNotExist: pool = AppliancePool(**pool_construct) pool.save() construct['appliance_pool'] = pool except KeyError: pass appliance = Appliance(**construct) appliance.save() # And now, refresh! refresh_appliances_provider.delay(provider.id)
def poke_trackerbot(self): """This beat-scheduled task periodically polls the trackerbot if there are any new templates. """ template_usability = [] # Extract data from trackerbot tbapi = trackerbot() objects = depaginate( tbapi, tbapi.providertemplate().get(limit=TRACKERBOT_PAGINATE))["objects"] per_group = {} for obj in objects: if obj["template"]["group"]["name"] == 'unknown': continue if obj["template"]["group"]["name"] not in per_group: per_group[obj["template"]["group"]["name"]] = [] per_group[obj["template"]["group"]["name"]].append(obj) # Sort them using the build date for group in list(per_group.keys()): per_group[group] = sorted(per_group[group], reverse=True, key=lambda o: o["template"]["datestamp"]) objects = [] # And interleave the the groups while any(per_group.values()): for key in list(per_group.keys()): if per_group[key]: objects.append(per_group[key].pop(0)) for template in objects: if template["provider"]["key"] not in list( conf.cfme_data.management_systems.keys()): # If we don't use that provider in yamls, set the template as not usable # 1) It will prevent adding this template if not added # 2) It'll mark the template as unusable if it already exists template["usable"] = False template_usability.append( (template["provider"]["key"], template["template"]["name"], template["usable"])) if not template["usable"]: continue group, create = Group.objects.get_or_create( id=template["template"]["group"]["name"]) # Check if the template is already obsolete if group.template_obsolete_days is not None: build_date = parsetime.from_iso_date( template["template"]["datestamp"]) if build_date <= (parsetime.today() - timedelta(days=group.template_obsolete_days)): # It is already obsolete, so ignore it continue if conf.cfme_data.management_systems.get( template["provider"]["key"], {}).get( "use_for_sprout", False ): # only create provider in db if it is marked to use for sprout provider, create = Provider.objects.get_or_create( id=template["provider"]["key"]) else: continue if not provider.is_working: continue if "sprout" not in provider.provider_data: continue if not provider.provider_type: provider.provider_type = provider.provider_data.get('type') provider.save(update_fields=['provider_type']) template_name = template["template"]["name"] ga_released = template['template']['ga_released'] custom_data = template['template'].get('custom_data', "{}") processed_custom_data = yaml.safe_load(custom_data) template_info = TemplateName.parse_template(template_name) if not template_info.datestamp: # Not a CFME/MIQ template, ignore it. continue # Original one original_template = None try: original_template = Template.objects.get( provider=provider, template_group=group, original_name=template_name, name=template_name, preconfigured=False) if original_template.ga_released != ga_released: original_template.ga_released = ga_released original_template.save(update_fields=['ga_released']) if provider.provider_type == 'openshift': if original_template.custom_data != custom_data: original_template.custom_data = processed_custom_data original_template.template_type = Template.OPENSHIFT_POD original_template.container = 'cloudforms-0' original_template.save(update_fields=[ 'custom_data', 'container', 'template_type' ]) except ObjectDoesNotExist: if template_name in provider.templates: if template_info.datestamp is None: self.logger.warning( "Ignoring template {} because it does not have a date!" .format(template_name)) continue template_version = template_info.version if template_version is None: # Make up a faux version # First 3 fields of version get parsed as a zstream # therefore ... makes it a "nil" stream template_version = "...{}".format( template_info.datestamp.strftime("%Y%m%d")) # openshift has two templates bound to one build version # sometimes sprout tries using second template -extdb that's wrong # so, such template has to be marked as not usable inside sprout usable = not template_name.endswith('-extdb') with transaction.atomic(): tpl = Template(provider=provider, template_group=group, original_name=template_name, name=template_name, preconfigured=False, date=template_info.datestamp, ready=True, exists=True, usable=usable, version=template_version) tpl.save() if provider.provider_type == 'openshift': tpl.custom_data = processed_custom_data tpl.container = 'cloudforms-0' tpl.template_type = Template.OPENSHIFT_POD tpl.save(update_fields=[ 'container', 'template_type', 'custom_data' ]) original_template = tpl self.logger.info("Created a new template #{}".format( tpl.id)) # If the provider is set to not preconfigure templates, do not bother even doing it. if provider.num_simultaneous_configuring > 0: # Preconfigured one try: # openshift providers don't have preconfigured templates. # so regular template should be used if provider.provider_type != 'openshift': preconfigured_template = Template.objects.get( provider=provider, template_group=group, original_name=template_name, preconfigured=True) else: preconfigured_template = Template.objects.get( provider=provider, template_group=group, name=template_name, preconfigured=True) preconfigured_template.custom_data = processed_custom_data preconfigured_template.container = 'cloudforms-0' preconfigured_template.template_type = Template.OPENSHIFT_POD preconfigured_template.save(update_fields=[ 'container', 'template_type', 'custom_data' ]) if preconfigured_template.ga_released != ga_released: preconfigured_template.ga_released = ga_released preconfigured_template.save(update_fields=['ga_released']) except ObjectDoesNotExist: if template_name in provider.templates and provider.provider_type != 'openshift': original_id = original_template.id if original_template is not None else None create_appliance_template.delay( provider.id, group.id, template_name, source_template_id=original_id) # If any of the templates becomes unusable, let sprout know about it # Similarly if some of them becomes usable ... for provider_id, template_name, usability in template_usability: if conf.cfme_data.management_systems.get(provider_id, {}).get( "use_for_sprout", False ): # only create provider in db if it is marked to use for sprout provider, create = Provider.objects.get_or_create(id=provider_id) else: continue if not provider.working or provider.disabled: continue with transaction.atomic(): for template in Template.objects.filter( provider=provider, original_name=template_name): template.usable = usability template.save(update_fields=['usable']) # Kill all shepherd appliances if they were accidentally spun up if not usability: for appliance in Appliance.objects.filter( template=template, marked_for_deletion=False, appliance_pool=None): self.logger.info('Killing an appliance {}/{} ' 'because its template was marked ' 'as unusable'.format( appliance.id, appliance.name)) Appliance.kill(appliance)
def generic_shepherd(self, preconfigured): """This task takes care of having the required templates spinned into required number of appliances. For each template group, it keeps the last template's appliances spinned up in required quantity. If new template comes out of the door, it automatically kills the older running template's appliances and spins up new ones. Sorts the groups by the fulfillment.""" for gs in sorted( GroupShepherd.objects.all(), key=lambda g: g.get_fulfillment_percentage(preconfigured)): prov_filter = {'provider__user_groups': gs.user_group} group_versions = Template.get_versions( template_group=gs.template_group, ready=True, usable=True, preconfigured=preconfigured, **prov_filter) group_dates = Template.get_dates(template_group=gs.template_group, ready=True, usable=True, preconfigured=preconfigured, **prov_filter) if group_versions: # Downstream - by version (downstream releases) version = group_versions[0] # Find the latest date (one version can have new build) dates = Template.get_dates(template_group=gs.template_group, ready=True, usable=True, version=group_versions[0], preconfigured=preconfigured, **prov_filter) if not dates: # No template yet? continue date = dates[0] filter_keep = {"version": version, "date": date} filters_kill = [] for kill_date in dates[1:]: filters_kill.append({"version": version, "date": kill_date}) for kill_version in group_versions[1:]: filters_kill.append({"version": kill_version}) elif group_dates: # Upstream - by date (upstream nightlies) filter_keep = {"date": group_dates[0]} filters_kill = [{"date": v} for v in group_dates[1:]] else: continue # Ignore this group, no templates detected yet filter_keep.update(prov_filter) for filt in filters_kill: filt.update(prov_filter) # Keeping current appliances # Retrieve list of all templates for given group # I know joins might be a bit better solution but I'll leave that for later. possible_templates = list( Template.objects.filter(usable=True, ready=True, template_group=gs.template_group, preconfigured=preconfigured, **filter_keep).all()) # If it can be deployed, it must exist possible_templates_for_provision = [ tpl for tpl in possible_templates if tpl.exists ] appliances = [] for template in possible_templates: appliances.extend( Appliance.objects.filter(template=template, appliance_pool=None, marked_for_deletion=False)) # If we then want to delete some templates, better kill the eldest. status_changed # says which one was provisioned when, because nothing else then touches that field. appliances.sort(key=lambda appliance: appliance.status_changed) pool_size = gs.template_pool_size if preconfigured else gs.unconfigured_template_pool_size if len(appliances) < pool_size and possible_templates_for_provision: # There must be some templates in order to run the provisioning # Provision ONE appliance at time for each group, that way it is possible to maintain # reasonable balancing with transaction.atomic(): # Now look for templates that are on non-busy providers tpl_free = [ t for t in possible_templates_for_provision if not t.provider.disabled and t.provider.free ] if tpl_free: chosen_template = sorted( tpl_free, key=lambda t: t.provider.appliance_load)[0] new_appliance_name = gen_appliance_name(chosen_template.id) appliance = Appliance(template=chosen_template, name=new_appliance_name) appliance.save() self.logger.info("Adding an appliance to shepherd: %s/%s", appliance.id, appliance.name) clone_template_to_appliance.delay(appliance.id, None) elif len(appliances) > pool_size: # Too many appliances, kill the surplus # Only kill those that are visible only for one group. This is necessary so the groups # don't "fight" for appliance in appliances[:len(appliances) - pool_size]: if appliance.is_visible_only_in_group(gs.user_group): self.logger.info( "Killing an extra appliance {}/{} in shepherd".format( appliance.id, appliance.name)) Appliance.kill(appliance) # Killing old appliances for filter_kill in filters_kill: for template in Template.objects.filter( ready=True, usable=True, template_group=gs.template_group, preconfigured=preconfigured, **filter_kill): for a in Appliance.objects.filter(template=template, appliance_pool=None, marked_for_deletion=False): self.logger.info( "Killing appliance {}/{} in shepherd because it is obsolete now" .format(a.id, a.name)) Appliance.kill(a)