def kill(self, request, appliances):
for appliance in appliances:
Appliance.kill(appliance)
self.message_user(request, "Initiated kill of {}.".format(appliance.name))
self.logger.info(
"User {}/{} requested kill of appliance {}".format(
request.user.pk, request.user.username, appliance.id))
def kill(self, request, appliances):
for appliance in appliances:
Appliance.kill(appliance)
self.message_user(request, "Initiated kill of {}.".format(appliance.name))
self.logger.info(
"User {}/{} requested kill of appliance {}".format(
request.user.pk, request.user.username, appliance.id))
def clone_template(self, template_id):
self.logger.info("Cloning template {}".format(template_id))
template = Template.objects.get(id=template_id)
new_appliance_name = gen_appliance_name(template_id)
appliance = Appliance(template=template, name=new_appliance_name)
appliance.save()
clone_template_to_appliance.delay(appliance.id)
def kill_appliance(request, appliance_id):
if not request.user.is_authenticated():
return go_home(request)
try:
appliance = Appliance.objects.get(id=appliance_id)
except ObjectDoesNotExist:
messages.error(request, 'Appliance with ID {} does not exist!.'.format(appliance_id))
return go_back_or_home(request)
if not can_operate_appliance_or_pool(appliance, request.user):
messages.error(request, 'This appliance belongs either to some other user or nobody.')
return go_back_or_home(request)
Appliance.kill(appliance)
messages.success(request, 'Kill initiated.')
return go_back_or_home(request)
def process_delayed_provision_tasks(self):
"""This picks up the provisioning tasks that were delayed due to ocncurrency limit of provision.
Goes one task by one and when some of them can be provisioned, it starts the provisioning and
then deletes the task.
"""
for task in DelayedProvisionTask.objects.order_by("id"):
if task.pool.not_needed_anymore:
task.delete()
continue
# Try retrieve from shepherd
appliances_given = Appliance.give_to_pool(task.pool, 1)
if appliances_given == 0:
# No free appliance in shepherd, so do it on our own
tpls = task.pool.possible_provisioning_templates
if task.provider_to_avoid is not None:
filtered_tpls = [
tpl for tpl in tpls
if tpl.provider != task.provider_to_avoid
]
if filtered_tpls:
# There are other providers to provision on, so try one of them
tpls = filtered_tpls
# If there is no other provider to provision on, we will use the original list.
# This will cause additional rejects until the provider quota is met
if tpls:
clone_template_to_pool(tpls[0].id, task.pool.id,
task.lease_time)
task.delete()
else:
# Try freeing up some space in provider
for provider in task.pool.possible_providers:
appliances = provider.free_shepherd_appliances.exclude(
**task.pool.appliance_filter_params)
if appliances:
appl = random.choice(appliances)
self.logger.info('Freeing some space in provider by '
'killing appliance {}/{}'.format(
appl.id, appl.name))
Appliance.kill(appl)
break # Just one
else:
# There was a free appliance in shepherd, so we took it and we don't need this task more
task.delete()
def destroy_appliance(user, appliance):
"""Destroy the appliance. If the kill task was called, id is returned, otherwise None
You can specify appliance by IP address, id or name.
"""
appliance = get_appliance(appliance, user)
try:
return Appliance.kill(appliance).task_id
except AttributeError: # None was returned
return None
def destroy_appliance(user, appliance):
"""Destroy the appliance. If the kill task was called, id is returned, otherwise None
You can specify appliance by IP address, id or name.
"""
appliance = get_appliance(appliance, user)
try:
return Appliance.kill(appliance).task_id
except AttributeError: # None was returned
return None
def delete_nonexistent_appliances(self):
"""Goes through orphaned appliances' objects and deletes them from the database."""
expiration_time = (timezone.now() - timedelta(**settings.ORPHANED_APPLIANCE_GRACE_TIME))
for appliance in Appliance.objects.filter(ready=True).all():
if appliance.name in redis.renaming_appliances:
continue
if appliance.power_state == Appliance.Power.ORPHANED:
if appliance.power_state_changed > expiration_time:
# Ignore it for now
self.logger.info("orphaned appliance {} power "
"state has been changed".format(appliance.id))
continue
self.logger.info(
"I will delete orphaned "
"appliance {}/{} on provider {}".format(appliance.id, appliance.name,
appliance.provider.id))
try:
appliance.delete()
except ObjectDoesNotExist as e:
if "AppliancePool" in str(e):
# Someone managed to delete the appliance pool before
appliance.appliance_pool = None
appliance.save(update_fields=['appliance_pool'])
appliance.delete()
else:
raise # No diaper pattern here!
# If something happened to the appliance provisioning process, just delete it to remove
# the garbage. It will be respinned again by shepherd.
# Grace time is specified in BROKEN_APPLIANCE_GRACE_TIME
expiration_time = (timezone.now() - timedelta(**settings.BROKEN_APPLIANCE_GRACE_TIME))
for appliance in Appliance.objects.filter(ready=False, marked_for_deletion=False).all():
if appliance.status_changed < expiration_time:
self.logger.info("Killing broken appliance {}/{} "
"on provider {}".format(appliance.id, appliance.name,
appliance.provider.id))
Appliance.kill(appliance) # Use kill because the appliance may still exist
# And now - if something happened during appliance deletion, call kill again
for appliance in Appliance.objects.filter(
marked_for_deletion=True, status_changed__lt=expiration_time).all():
self.logger.info(
"Trying to kill unkilled appliance {}/{} "
"on provider {}".format(appliance.id, appliance.name, appliance.provider.id))
Appliance.kill(appliance, force_delete=True)
def destroy_appliance(user, appliance):
"""Destroy the appliance. If the kill task was called, id is returned, otherwise None
You can specify appliance by IP address, id or name.
"""
appliance = get_appliance(appliance)
if appliance.owner is None:
if not user.is_staff:
raise Exception("Only staff can operate with nonowned appliances")
elif appliance.owner != user:
raise Exception("This appliance belongs to a different user!")
try:
return Appliance.kill(appliance).task_id
except AttributeError: # None was returned
return None
def command(csv_file):
csv_reader = csv.reader(csv_file)
appliances = [
Appliance(
product_line=ProductLine.objects.get_or_create(
name=product_line)[0],
serial_number=serial_number,
model_number=model_number,
) for product_line, model_number, serial_number in csv_reader
]
print("Here are the appliances extracted:")
pprint(appliances)
if click.confirm("Proceed?"):
Appliance.objects.bulk_create(appliances)
def apply_lease_times_after_pool_fulfilled(self, appliance_pool_id,
time_minutes):
try:
pool = AppliancePool.objects.get(id=appliance_pool_id)
except ObjectDoesNotExist as e:
self.logger.error("It seems such appliance pool %s doesn't exist: %s",
appliance_pool_id, str(e))
return False
if pool.fulfilled:
pool.logger.info("Applying lease time and renaming appliances")
for appliance in pool.appliances:
apply_lease_times.delay(appliance.id, time_minutes)
with transaction.atomic():
pool.finished = True
pool.save(update_fields=['finished'])
pool.logger.info(
"Pool {} setup is finished".format(appliance_pool_id))
else:
# Look whether we can swap any provisioning appliance with some in shepherd
pool.logger.info("Pool isn't fulfilled yet")
unfinished = list(
Appliance.objects.filter(appliance_pool=pool,
ready=False,
marked_for_deletion=False).all())
random.shuffle(unfinished)
if len(unfinished) > 0:
pool.logger.info('There are %s unfinished appliances',
len(unfinished))
n = Appliance.give_to_pool(pool, len(unfinished))
with transaction.atomic():
for _ in range(n):
appl = unfinished.pop()
appl.appliance_pool = None
appl.save(update_fields=['appliance_pool'])
try:
pool.logger.info("Retrying to apply lease again")
self.retry(args=(appliance_pool_id, time_minutes),
countdown=30,
max_retries=120)
except MaxRetriesExceededError: # Bad luck, pool fulfillment failed. So destroy it.
pool.logger.error(
"Waiting for fulfillment failed. Initiating the destruction process."
)
pool.kill()
def request_appliance_pool(self, appliance_pool_id, time_minutes):
"""This task gives maximum possible amount of spinned-up appliances to the specified pool and
then if there is need to spin up another appliances, it spins them up via clone_template_to_pool
task."""
self.logger.info("Appliance pool {} requested for {} minutes.".format(
appliance_pool_id, time_minutes))
pool = AppliancePool.objects.get(id=appliance_pool_id)
n = Appliance.give_to_pool(pool)
for i in range(pool.total_count - n):
tpls = pool.possible_provisioning_templates
if tpls:
template_id = tpls[0].id
clone_template_to_pool(template_id, pool.id, time_minutes)
else:
with transaction.atomic():
task = DelayedProvisionTask(pool=pool, lease_time=time_minutes)
task.save()
apply_lease_times_after_pool_fulfilled.delay(appliance_pool_id,
time_minutes)
def clone_template_to_pool(template_id, appliance_pool_id, time_minutes):
template = Template.objects.get(id=template_id)
with transaction.atomic():
pool = AppliancePool.objects.get(id=appliance_pool_id)
if pool.not_needed_anymore:
return
new_appliance_name = gen_appliance_name(template_id,
username=pool.owner.username)
appliance = Appliance(template=template,
name=new_appliance_name,
appliance_pool=pool)
appliance.save()
appliance.set_lease_time()
# Set pool to these params to keep the appliances with same versions/dates
pool.version = template.version
pool.date = template.date
pool.save(update_fields=['version', 'date'])
clone_template_to_appliance.delay(appliance.id, time_minutes,
pool.yum_update)
def appliance_action(request, appliance_id, action, x=None):
try:
appliance = Appliance.objects.get(id=appliance_id)
except ObjectDoesNotExist:
messages.warning(
request,
'Appliance with ID {} does not exist!.'.format(appliance_id))
return go_back_or_home(request)
if not can_operate_appliance_or_pool(appliance, request.user):
messages.warning(
request,
'This appliance belongs either to some other user or nobody.')
return go_back_or_home(request)
if action == "start":
if appliance.power_state != Appliance.Power.ON:
chain(appliance_power_on.si(appliance.id),
(wait_appliance_ready if appliance.preconfigured else
mark_appliance_ready).si(appliance.id))()
messages.success(request, 'Initiated launch of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already powered on.')
return go_back_or_home(request)
elif action == "reboot":
if appliance.power_state == Appliance.Power.ON:
chain(appliance_reboot.si(appliance.id),
mark_appliance_ready.si(appliance.id))()
messages.success(request, 'Initiated reboot of appliance.')
return go_back_or_home(request)
else:
messages.warning(request,
'Only powered on appliances can be rebooted')
return go_back_or_home(request)
elif action == "stop":
if appliance.power_state != Appliance.Power.OFF:
appliance_power_off.delay(appliance.id)
messages.success(request, 'Initiated stop of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already powered off.')
return go_back_or_home(request)
elif action == "suspend":
if appliance.power_state != Appliance.Power.SUSPENDED:
appliance_suspend.delay(appliance.id)
messages.success(request, 'Initiated suspend of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already suspended.')
return go_back_or_home(request)
elif action == "kill":
Appliance.kill(appliance)
messages.success(request, 'Kill initiated.')
return go_back_or_home(request)
elif action == "dont_expire":
if not request.user.is_superuser:
messages.warning(
request,
'Disabling expiration time is allowed only for superusers.')
return go_back_or_home(request)
with transaction.atomic():
appliance.leased_until = None
appliance.save()
messages.success(request, 'Lease disabled successfully. Be careful.')
return go_back_or_home(request)
elif action == "set_lease":
if not can_operate_appliance_or_pool(appliance, request.user):
messages.warning(
request,
'This appliance belongs either to some other user or nobody.')
return go_back_or_home(request)
appliance.prolong_lease(time=int(x))
messages.success(request, 'Lease prolonged successfully.')
return go_back_or_home(request)
else:
messages.warning(request, "Unknown action '{}'".format(action))
def appliance_action(request, appliance_id, action, x=None):
try:
appliance = Appliance.objects.get(id=appliance_id)
except ObjectDoesNotExist:
messages.warning(request, 'Appliance with ID {} does not exist!.'.format(appliance_id))
return go_back_or_home(request)
if not can_operate_appliance_or_pool(appliance, request.user):
messages.warning(request, 'This appliance belongs either to some other user or nobody.')
return go_back_or_home(request)
if action == "start":
if appliance.power_state != Appliance.Power.ON:
chain(
appliance_power_on.si(appliance.id),
(wait_appliance_ready if appliance.preconfigured else mark_appliance_ready).si(
appliance.id))()
messages.success(request, 'Initiated launch of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already powered on.')
return go_back_or_home(request)
elif action == "reboot":
if appliance.power_state == Appliance.Power.ON:
chain(
appliance_reboot.si(appliance.id),
mark_appliance_ready.si(appliance.id))()
messages.success(request, 'Initiated reboot of appliance.')
return go_back_or_home(request)
else:
messages.warning(request, 'Only powered on appliances can be rebooted')
return go_back_or_home(request)
elif action == "stop":
if appliance.power_state != Appliance.Power.OFF:
appliance_power_off.delay(appliance.id)
messages.success(request, 'Initiated stop of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already powered off.')
return go_back_or_home(request)
elif action == "suspend":
if appliance.power_state != Appliance.Power.SUSPENDED:
appliance_suspend.delay(appliance.id)
messages.success(request, 'Initiated suspend of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already suspended.')
return go_back_or_home(request)
elif action == "kill":
Appliance.kill(appliance)
messages.success(request, 'Kill initiated.')
return go_back_or_home(request)
elif action == "dont_expire":
if not request.user.is_superuser:
messages.warning(request, 'Disabling expiration time is allowed only for superusers.')
return go_back_or_home(request)
with transaction.atomic():
appliance.leased_until = None
appliance.save()
messages.success(request, 'Lease disabled successfully. Be careful.')
return go_back_or_home(request)
elif action == "set_lease":
if not can_operate_appliance_or_pool(appliance, request.user):
messages.warning(request, 'This appliance belongs either to some other user or nobody.')
return go_back_or_home(request)
appliance.prolong_lease(time=int(x))
messages.success(request, 'Lease prolonged successfully.')
return go_back_or_home(request)
else:
messages.warning(request, "Unknown action '{}'".format(action))
def newHw(request):
if request.POST:
txt_hwid = request.POST.get('txt_hwid').strip()
txt_psw = request.POST.get('txt_sn').strip()
txt_title = request.POST.get('txt_title').strip()
hwtype_radio = request.POST.get('hwtype')
import re
reg = re.compile(r'^[\w\s]+$')
if reg.search(txt_hwid) is None:
messages.info(
request,
'Please use only alphanumeric symbols and whitespaces for hwid.'
)
return render(request, 'hardware/newhw.html')
if reg.search(txt_psw) is None:
messages.info(
request,
'Please use only alphanumeric symbols and whitespaces for serial number.'
)
return render(request, 'hardware/newhw.html')
if reg.search(txt_title) is None:
messages.info(
request,
'Please use only alphanumeric symbols and whitespaces for title.'
)
return render(request, 'hardware/newhw.html')
from django.db import IntegrityError
if hwtype_radio is not None:
if hwtype_radio == '0':
from appliances.models import Appliance, ApplianceChore
try:
a = Appliance()
a.hwid = txt_hwid
a.password = txt_psw
a.name = txt_title
a.generatePad()
a.save()
ac = ApplianceChore()
ac.appliance = a
ac.name = a.name + " Chore"
ac.save()
except IntegrityError:
messages.error(request,
'Appliance with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request,
a.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
elif hwtype_radio == '1':
from dishes.models import ItemTracker
try:
h = ItemTracker()
h.hwid = txt_hwid
h.password = txt_psw
h.name = txt_title
h.generatePad()
h.save()
except IntegrityError:
messages.error(
request, 'Item tracker with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request,
h.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
else:
messages.info(request, 'Please select hardware type')
return render(request, 'hardware/newhw.html')
return render(request, 'hardware/newhw.html')
def newHw(request):
if request.POST:
txt_hwid = request.POST.get('txt_hwid').strip()
txt_psw = request.POST.get('txt_sn').strip()
txt_title = request.POST.get('txt_title').strip()
hwtype_radio = request.POST.get('hwtype')
import re
reg = re.compile(r'^[\w\s]+$')
if reg.search(txt_hwid) is None:
messages.info(request, 'Please use only alphanumeric symbols and whitespaces for hwid.')
return render(request, 'hardware/newhw.html')
if reg.search(txt_psw) is None:
messages.info(request, 'Please use only alphanumeric symbols and whitespaces for serial number.')
return render(request, 'hardware/newhw.html')
if reg.search(txt_title) is None:
messages.info(request, 'Please use only alphanumeric symbols and whitespaces for title.')
return render(request, 'hardware/newhw.html')
from django.db import IntegrityError
if hwtype_radio is not None:
if hwtype_radio == '0':
from appliances.models import Appliance, ApplianceChore
try:
a = Appliance()
a.hwid = txt_hwid
a.password = txt_psw
a.name = txt_title
a.generatePad()
a.save()
ac = ApplianceChore()
ac.appliance = a
ac.name = a.name + " Chore"
ac.save()
except IntegrityError:
messages.error(request, 'Appliance with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request, a.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
elif hwtype_radio == '1':
from dishes.models import ItemTracker
try:
h = ItemTracker()
h.hwid = txt_hwid
h.password = txt_psw
h.name = txt_title
h.generatePad()
h.save()
except IntegrityError:
messages.error(request, 'Item tracker with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request, h.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
else:
messages.info(request, 'Please select hardware type')
return render(request, 'hardware/newhw.html')
return render(request, 'hardware/newhw.html')
def synchronize_untracked_vms_in_provider(self, provider_id):
"""'re'-synchronizes any vms that might be lost during outages."""
provider = Provider.objects.get(id=provider_id, working=True, disabled=False)
provider_api = provider.api
if not hasattr(provider_api, 'list_vms'):
# This provider does not have VMs
return
for vm in sorted(provider_api.list_vms(), key=lambda pvm: getattr(pvm, 'name', pvm)):
if (
Appliance.objects.filter(name=getattr(vm, 'name', vm),
template__provider=provider).count() != 0
):
continue
# We have an untracked VM. Let's investigate
try:
appliance_id = vm.get_meta_value('sprout_id')
except KeyError:
continue
except (AttributeError, NotImplementedError):
# Do not bother if not implemented in the VM object's API
return
# just check it again ...
if Appliance.objects.filter(id=appliance_id).count() == 1:
# For some reason it is already in
continue
# Now it appears that this is a VM that was in Sprout
construct = {'id': appliance_id}
# Retrieve appliance data
try:
self.logger.info('Trying to reconstruct appliance %d/%s', appliance_id, vm.name)
construct['name'] = vm.name
template_id = vm.get_meta_value('sprout_source_template_id')
# Templates are not deleted from the DB so this should be OK.
construct['template'] = Template.objects.get(id=template_id)
construct['name'] = vm.name
construct['ready'] = vm.get_meta_value('sprout_ready')
construct['description'] = vm.get_meta_value('sprout_description')
construct['lun_disk_connected'] = vm.get_meta_value('sprout_lun_disk_connected')
construct['swap'] = vm.get_meta_value('sprout_swap')
construct['ssh_failed'] = vm.get_meta_value('sprout_ssh_failed')
# Time fields
construct['datetime_leased'] = parsedate(vm.get_meta_value('sprout_datetime_leased'))
construct['leased_until'] = parsedate(vm.get_meta_value('sprout_leased_until'))
construct['status_changed'] = parsedate(vm.get_meta_value('sprout_status_changed'))
construct['created_on'] = parsedate(vm.get_meta_value('sprout_created_on'))
construct['modified_on'] = parsedate(vm.get_meta_value('sprout_modified_on'))
except KeyError as e:
self.logger.error('Failed to reconstruct %d/%s', appliance_id, vm.name)
self.logger.exception(e)
continue
# Retrieve pool data if applicable
try:
pool_id = vm.get_meta_value('sprout_pool_id')
pool_construct = dict(id=pool_id)
pool_construct['total_count'] = vm.get_meta_value('sprout_pool_total_count')
group_id = vm.get_meta_value('sprout_pool_group')
pool_construct['group'] = Group.objects.get(id=group_id)
try:
construct_provider_id = vm.get_meta_value('sprout_pool_provider')
pool_construct['provider'] = Provider.objects.get(id=construct_provider_id)
except (KeyError, ObjectDoesNotExist):
# optional
pool_construct['provider'] = None
pool_construct['version'] = vm.get_meta_value('sprout_pool_version')
pool_construct['date'] = parsedate(vm.get_meta_value('sprout_pool_appliance_date'))
owner_id = vm.get_meta_value('sprout_pool_owner_id')
try:
owner = User.objects.get(id=owner_id)
except ObjectDoesNotExist:
owner_username = vm.get_meta_value('sprout_pool_owner_username')
owner = User(id=owner_id, username=owner_username)
owner.save()
pool_construct['owner'] = owner
pool_construct['preconfigured'] = vm.get_meta_value('sprout_pool_preconfigured')
pool_construct['description'] = vm.get_meta_value('sprout_pool_description')
pool_construct['not_needed_anymore'] = vm.get_meta_value(
'sprout_pool_not_needed_anymore')
pool_construct['finished'] = vm.get_meta_value('sprout_pool_finished')
pool_construct['yum_update'] = vm.get_meta_value('sprout_pool_yum_update')
try:
construct['appliance_pool'] = AppliancePool.objects.get(id=pool_id)
except ObjectDoesNotExist:
pool = AppliancePool(**pool_construct)
pool.save()
construct['appliance_pool'] = pool
except KeyError:
pass
appliance = Appliance(**construct)
appliance.save()
# And now, refresh!
refresh_appliances_provider.delay(provider.id)
def poke_trackerbot(self):
"""This beat-scheduled task periodically polls the trackerbot if there are any new templates.
"""
template_usability = []
# Extract data from trackerbot
tbapi = trackerbot()
objects = depaginate(
tbapi,
tbapi.providertemplate().get(limit=TRACKERBOT_PAGINATE))["objects"]
per_group = {}
for obj in objects:
if obj["template"]["group"]["name"] == 'unknown':
continue
if obj["template"]["group"]["name"] not in per_group:
per_group[obj["template"]["group"]["name"]] = []
per_group[obj["template"]["group"]["name"]].append(obj)
# Sort them using the build date
for group in list(per_group.keys()):
per_group[group] = sorted(per_group[group],
reverse=True,
key=lambda o: o["template"]["datestamp"])
objects = []
# And interleave the the groups
while any(per_group.values()):
for key in list(per_group.keys()):
if per_group[key]:
objects.append(per_group[key].pop(0))
for template in objects:
if template["provider"]["key"] not in list(
conf.cfme_data.management_systems.keys()):
# If we don't use that provider in yamls, set the template as not usable
# 1) It will prevent adding this template if not added
# 2) It'll mark the template as unusable if it already exists
template["usable"] = False
template_usability.append(
(template["provider"]["key"], template["template"]["name"],
template["usable"]))
if not template["usable"]:
continue
group, create = Group.objects.get_or_create(
id=template["template"]["group"]["name"])
# Check if the template is already obsolete
if group.template_obsolete_days is not None:
build_date = parsetime.from_iso_date(
template["template"]["datestamp"])
if build_date <= (parsetime.today() -
timedelta(days=group.template_obsolete_days)):
# It is already obsolete, so ignore it
continue
if conf.cfme_data.management_systems.get(
template["provider"]["key"],
{}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(
id=template["provider"]["key"])
else:
continue
if not provider.is_working:
continue
if "sprout" not in provider.provider_data:
continue
if not provider.provider_type:
provider.provider_type = provider.provider_data.get('type')
provider.save(update_fields=['provider_type'])
template_name = template["template"]["name"]
ga_released = template['template']['ga_released']
custom_data = template['template'].get('custom_data', "{}")
processed_custom_data = yaml.safe_load(custom_data)
template_info = TemplateName.parse_template(template_name)
if not template_info.datestamp:
# Not a CFME/MIQ template, ignore it.
continue
# Original one
original_template = None
try:
original_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False)
if original_template.ga_released != ga_released:
original_template.ga_released = ga_released
original_template.save(update_fields=['ga_released'])
if provider.provider_type == 'openshift':
if original_template.custom_data != custom_data:
original_template.custom_data = processed_custom_data
original_template.template_type = Template.OPENSHIFT_POD
original_template.container = 'cloudforms-0'
original_template.save(update_fields=[
'custom_data', 'container', 'template_type'
])
except ObjectDoesNotExist:
if template_name in provider.templates:
if template_info.datestamp is None:
self.logger.warning(
"Ignoring template {} because it does not have a date!"
.format(template_name))
continue
template_version = template_info.version
if template_version is None:
# Make up a faux version
# First 3 fields of version get parsed as a zstream
# therefore ... makes it a "nil" stream
template_version = "...{}".format(
template_info.datestamp.strftime("%Y%m%d"))
# openshift has two templates bound to one build version
# sometimes sprout tries using second template -extdb that's wrong
# so, such template has to be marked as not usable inside sprout
usable = not template_name.endswith('-extdb')
with transaction.atomic():
tpl = Template(provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False,
date=template_info.datestamp,
ready=True,
exists=True,
usable=usable,
version=template_version)
tpl.save()
if provider.provider_type == 'openshift':
tpl.custom_data = processed_custom_data
tpl.container = 'cloudforms-0'
tpl.template_type = Template.OPENSHIFT_POD
tpl.save(update_fields=[
'container', 'template_type', 'custom_data'
])
original_template = tpl
self.logger.info("Created a new template #{}".format(
tpl.id))
# If the provider is set to not preconfigure templates, do not bother even doing it.
if provider.num_simultaneous_configuring > 0:
# Preconfigured one
try:
# openshift providers don't have preconfigured templates.
# so regular template should be used
if provider.provider_type != 'openshift':
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
preconfigured=True)
else:
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
name=template_name,
preconfigured=True)
preconfigured_template.custom_data = processed_custom_data
preconfigured_template.container = 'cloudforms-0'
preconfigured_template.template_type = Template.OPENSHIFT_POD
preconfigured_template.save(update_fields=[
'container', 'template_type', 'custom_data'
])
if preconfigured_template.ga_released != ga_released:
preconfigured_template.ga_released = ga_released
preconfigured_template.save(update_fields=['ga_released'])
except ObjectDoesNotExist:
if template_name in provider.templates and provider.provider_type != 'openshift':
original_id = original_template.id if original_template is not None else None
create_appliance_template.delay(
provider.id,
group.id,
template_name,
source_template_id=original_id)
# If any of the templates becomes unusable, let sprout know about it
# Similarly if some of them becomes usable ...
for provider_id, template_name, usability in template_usability:
if conf.cfme_data.management_systems.get(provider_id, {}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(id=provider_id)
else:
continue
if not provider.working or provider.disabled:
continue
with transaction.atomic():
for template in Template.objects.filter(
provider=provider, original_name=template_name):
template.usable = usability
template.save(update_fields=['usable'])
# Kill all shepherd appliances if they were accidentally spun up
if not usability:
for appliance in Appliance.objects.filter(
template=template,
marked_for_deletion=False,
appliance_pool=None):
self.logger.info('Killing an appliance {}/{} '
'because its template was marked '
'as unusable'.format(
appliance.id, appliance.name))
Appliance.kill(appliance)
def generic_shepherd(self, preconfigured):
"""This task takes care of having the required templates spinned into required number of
appliances. For each template group, it keeps the last template's appliances spinned up in
required quantity. If new template comes out of the door, it automatically kills the older
running template's appliances and spins up new ones. Sorts the groups by the fulfillment."""
for gs in sorted(
GroupShepherd.objects.all(),
key=lambda g: g.get_fulfillment_percentage(preconfigured)):
prov_filter = {'provider__user_groups': gs.user_group}
group_versions = Template.get_versions(
template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
group_dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
if group_versions:
# Downstream - by version (downstream releases)
version = group_versions[0]
# Find the latest date (one version can have new build)
dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
version=group_versions[0],
preconfigured=preconfigured,
**prov_filter)
if not dates:
# No template yet?
continue
date = dates[0]
filter_keep = {"version": version, "date": date}
filters_kill = []
for kill_date in dates[1:]:
filters_kill.append({"version": version, "date": kill_date})
for kill_version in group_versions[1:]:
filters_kill.append({"version": kill_version})
elif group_dates:
# Upstream - by date (upstream nightlies)
filter_keep = {"date": group_dates[0]}
filters_kill = [{"date": v} for v in group_dates[1:]]
else:
continue # Ignore this group, no templates detected yet
filter_keep.update(prov_filter)
for filt in filters_kill:
filt.update(prov_filter)
# Keeping current appliances
# Retrieve list of all templates for given group
# I know joins might be a bit better solution but I'll leave that for later.
possible_templates = list(
Template.objects.filter(usable=True,
ready=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_keep).all())
# If it can be deployed, it must exist
possible_templates_for_provision = [
tpl for tpl in possible_templates if tpl.exists
]
appliances = []
for template in possible_templates:
appliances.extend(
Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False))
# If we then want to delete some templates, better kill the eldest. status_changed
# says which one was provisioned when, because nothing else then touches that field.
appliances.sort(key=lambda appliance: appliance.status_changed)
pool_size = gs.template_pool_size if preconfigured else gs.unconfigured_template_pool_size
if len(appliances) < pool_size and possible_templates_for_provision:
# There must be some templates in order to run the provisioning
# Provision ONE appliance at time for each group, that way it is possible to maintain
# reasonable balancing
with transaction.atomic():
# Now look for templates that are on non-busy providers
tpl_free = [
t for t in possible_templates_for_provision
if not t.provider.disabled and t.provider.free
]
if tpl_free:
chosen_template = sorted(
tpl_free, key=lambda t: t.provider.appliance_load)[0]
new_appliance_name = gen_appliance_name(chosen_template.id)
appliance = Appliance(template=chosen_template,
name=new_appliance_name)
appliance.save()
self.logger.info("Adding an appliance to shepherd: %s/%s",
appliance.id, appliance.name)
clone_template_to_appliance.delay(appliance.id, None)
elif len(appliances) > pool_size:
# Too many appliances, kill the surplus
# Only kill those that are visible only for one group. This is necessary so the groups
# don't "fight"
for appliance in appliances[:len(appliances) - pool_size]:
if appliance.is_visible_only_in_group(gs.user_group):
self.logger.info(
"Killing an extra appliance {}/{} in shepherd".format(
appliance.id, appliance.name))
Appliance.kill(appliance)
# Killing old appliances
for filter_kill in filters_kill:
for template in Template.objects.filter(
ready=True,
usable=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_kill):
for a in Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False):
self.logger.info(
"Killing appliance {}/{} in shepherd because it is obsolete now"
.format(a.id, a.name))
Appliance.kill(a)
