def clone_template(self, template_id):
self.logger.info("Cloning template {}".format(template_id))
template = Template.objects.get(id=template_id)
new_appliance_name = gen_appliance_name(template_id)
appliance = Appliance(template=template, name=new_appliance_name)
appliance.save()
clone_template_to_appliance.delay(appliance.id)
def clone_template_to_pool(template_id, appliance_pool_id, time_minutes):
template = Template.objects.get(id=template_id)
with transaction.atomic():
pool = AppliancePool.objects.get(id=appliance_pool_id)
if pool.not_needed_anymore:
return
new_appliance_name = gen_appliance_name(template_id,
username=pool.owner.username)
appliance = Appliance(template=template,
name=new_appliance_name,
appliance_pool=pool)
appliance.save()
appliance.set_lease_time()
# Set pool to these params to keep the appliances with same versions/dates
pool.version = template.version
pool.date = template.date
pool.save(update_fields=['version', 'date'])
clone_template_to_appliance.delay(appliance.id, time_minutes,
pool.yum_update)
def newHw(request):
if request.POST:
txt_hwid = request.POST.get('txt_hwid').strip()
txt_psw = request.POST.get('txt_sn').strip()
txt_title = request.POST.get('txt_title').strip()
hwtype_radio = request.POST.get('hwtype')
import re
reg = re.compile(r'^[\w\s]+$')
if reg.search(txt_hwid) is None:
messages.info(
request,
'Please use only alphanumeric symbols and whitespaces for hwid.'
)
return render(request, 'hardware/newhw.html')
if reg.search(txt_psw) is None:
messages.info(
request,
'Please use only alphanumeric symbols and whitespaces for serial number.'
)
return render(request, 'hardware/newhw.html')
if reg.search(txt_title) is None:
messages.info(
request,
'Please use only alphanumeric symbols and whitespaces for title.'
)
return render(request, 'hardware/newhw.html')
from django.db import IntegrityError
if hwtype_radio is not None:
if hwtype_radio == '0':
from appliances.models import Appliance, ApplianceChore
try:
a = Appliance()
a.hwid = txt_hwid
a.password = txt_psw
a.name = txt_title
a.generatePad()
a.save()
ac = ApplianceChore()
ac.appliance = a
ac.name = a.name + " Chore"
ac.save()
except IntegrityError:
messages.error(request,
'Appliance with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request,
a.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
elif hwtype_radio == '1':
from dishes.models import ItemTracker
try:
h = ItemTracker()
h.hwid = txt_hwid
h.password = txt_psw
h.name = txt_title
h.generatePad()
h.save()
except IntegrityError:
messages.error(
request, 'Item tracker with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request,
h.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
else:
messages.info(request, 'Please select hardware type')
return render(request, 'hardware/newhw.html')
return render(request, 'hardware/newhw.html')
def newHw(request):
if request.POST:
txt_hwid = request.POST.get('txt_hwid').strip()
txt_psw = request.POST.get('txt_sn').strip()
txt_title = request.POST.get('txt_title').strip()
hwtype_radio = request.POST.get('hwtype')
import re
reg = re.compile(r'^[\w\s]+$')
if reg.search(txt_hwid) is None:
messages.info(request, 'Please use only alphanumeric symbols and whitespaces for hwid.')
return render(request, 'hardware/newhw.html')
if reg.search(txt_psw) is None:
messages.info(request, 'Please use only alphanumeric symbols and whitespaces for serial number.')
return render(request, 'hardware/newhw.html')
if reg.search(txt_title) is None:
messages.info(request, 'Please use only alphanumeric symbols and whitespaces for title.')
return render(request, 'hardware/newhw.html')
from django.db import IntegrityError
if hwtype_radio is not None:
if hwtype_radio == '0':
from appliances.models import Appliance, ApplianceChore
try:
a = Appliance()
a.hwid = txt_hwid
a.password = txt_psw
a.name = txt_title
a.generatePad()
a.save()
ac = ApplianceChore()
ac.appliance = a
ac.name = a.name + " Chore"
ac.save()
except IntegrityError:
messages.error(request, 'Appliance with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request, a.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
elif hwtype_radio == '1':
from dishes.models import ItemTracker
try:
h = ItemTracker()
h.hwid = txt_hwid
h.password = txt_psw
h.name = txt_title
h.generatePad()
h.save()
except IntegrityError:
messages.error(request, 'Item tracker with this id already exists.')
return render(request, 'hardware/newhw.html')
else:
messages.info(request, h.getName() + ' successfully registered.')
return redirect('hardware:hwlist')
else:
messages.info(request, 'Please select hardware type')
return render(request, 'hardware/newhw.html')
return render(request, 'hardware/newhw.html')
def synchronize_untracked_vms_in_provider(self, provider_id):
"""'re'-synchronizes any vms that might be lost during outages."""
provider = Provider.objects.get(id=provider_id, working=True, disabled=False)
provider_api = provider.api
if not hasattr(provider_api, 'list_vms'):
# This provider does not have VMs
return
for vm in sorted(provider_api.list_vms(), key=lambda pvm: getattr(pvm, 'name', pvm)):
if (
Appliance.objects.filter(name=getattr(vm, 'name', vm),
template__provider=provider).count() != 0
):
continue
# We have an untracked VM. Let's investigate
try:
appliance_id = vm.get_meta_value('sprout_id')
except KeyError:
continue
except (AttributeError, NotImplementedError):
# Do not bother if not implemented in the VM object's API
return
# just check it again ...
if Appliance.objects.filter(id=appliance_id).count() == 1:
# For some reason it is already in
continue
# Now it appears that this is a VM that was in Sprout
construct = {'id': appliance_id}
# Retrieve appliance data
try:
self.logger.info('Trying to reconstruct appliance %d/%s', appliance_id, vm.name)
construct['name'] = vm.name
template_id = vm.get_meta_value('sprout_source_template_id')
# Templates are not deleted from the DB so this should be OK.
construct['template'] = Template.objects.get(id=template_id)
construct['name'] = vm.name
construct['ready'] = vm.get_meta_value('sprout_ready')
construct['description'] = vm.get_meta_value('sprout_description')
construct['lun_disk_connected'] = vm.get_meta_value('sprout_lun_disk_connected')
construct['swap'] = vm.get_meta_value('sprout_swap')
construct['ssh_failed'] = vm.get_meta_value('sprout_ssh_failed')
# Time fields
construct['datetime_leased'] = parsedate(vm.get_meta_value('sprout_datetime_leased'))
construct['leased_until'] = parsedate(vm.get_meta_value('sprout_leased_until'))
construct['status_changed'] = parsedate(vm.get_meta_value('sprout_status_changed'))
construct['created_on'] = parsedate(vm.get_meta_value('sprout_created_on'))
construct['modified_on'] = parsedate(vm.get_meta_value('sprout_modified_on'))
except KeyError as e:
self.logger.error('Failed to reconstruct %d/%s', appliance_id, vm.name)
self.logger.exception(e)
continue
# Retrieve pool data if applicable
try:
pool_id = vm.get_meta_value('sprout_pool_id')
pool_construct = dict(id=pool_id)
pool_construct['total_count'] = vm.get_meta_value('sprout_pool_total_count')
group_id = vm.get_meta_value('sprout_pool_group')
pool_construct['group'] = Group.objects.get(id=group_id)
try:
construct_provider_id = vm.get_meta_value('sprout_pool_provider')
pool_construct['provider'] = Provider.objects.get(id=construct_provider_id)
except (KeyError, ObjectDoesNotExist):
# optional
pool_construct['provider'] = None
pool_construct['version'] = vm.get_meta_value('sprout_pool_version')
pool_construct['date'] = parsedate(vm.get_meta_value('sprout_pool_appliance_date'))
owner_id = vm.get_meta_value('sprout_pool_owner_id')
try:
owner = User.objects.get(id=owner_id)
except ObjectDoesNotExist:
owner_username = vm.get_meta_value('sprout_pool_owner_username')
owner = User(id=owner_id, username=owner_username)
owner.save()
pool_construct['owner'] = owner
pool_construct['preconfigured'] = vm.get_meta_value('sprout_pool_preconfigured')
pool_construct['description'] = vm.get_meta_value('sprout_pool_description')
pool_construct['not_needed_anymore'] = vm.get_meta_value(
'sprout_pool_not_needed_anymore')
pool_construct['finished'] = vm.get_meta_value('sprout_pool_finished')
pool_construct['yum_update'] = vm.get_meta_value('sprout_pool_yum_update')
try:
construct['appliance_pool'] = AppliancePool.objects.get(id=pool_id)
except ObjectDoesNotExist:
pool = AppliancePool(**pool_construct)
pool.save()
construct['appliance_pool'] = pool
except KeyError:
pass
appliance = Appliance(**construct)
appliance.save()
# And now, refresh!
refresh_appliances_provider.delay(provider.id)
def generic_shepherd(self, preconfigured):
"""This task takes care of having the required templates spinned into required number of
appliances. For each template group, it keeps the last template's appliances spinned up in
required quantity. If new template comes out of the door, it automatically kills the older
running template's appliances and spins up new ones. Sorts the groups by the fulfillment."""
for gs in sorted(
GroupShepherd.objects.all(),
key=lambda g: g.get_fulfillment_percentage(preconfigured)):
prov_filter = {'provider__user_groups': gs.user_group}
group_versions = Template.get_versions(
template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
group_dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
if group_versions:
# Downstream - by version (downstream releases)
version = group_versions[0]
# Find the latest date (one version can have new build)
dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
version=group_versions[0],
preconfigured=preconfigured,
**prov_filter)
if not dates:
# No template yet?
continue
date = dates[0]
filter_keep = {"version": version, "date": date}
filters_kill = []
for kill_date in dates[1:]:
filters_kill.append({"version": version, "date": kill_date})
for kill_version in group_versions[1:]:
filters_kill.append({"version": kill_version})
elif group_dates:
# Upstream - by date (upstream nightlies)
filter_keep = {"date": group_dates[0]}
filters_kill = [{"date": v} for v in group_dates[1:]]
else:
continue # Ignore this group, no templates detected yet
filter_keep.update(prov_filter)
for filt in filters_kill:
filt.update(prov_filter)
# Keeping current appliances
# Retrieve list of all templates for given group
# I know joins might be a bit better solution but I'll leave that for later.
possible_templates = list(
Template.objects.filter(usable=True,
ready=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_keep).all())
# If it can be deployed, it must exist
possible_templates_for_provision = [
tpl for tpl in possible_templates if tpl.exists
]
appliances = []
for template in possible_templates:
appliances.extend(
Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False))
# If we then want to delete some templates, better kill the eldest. status_changed
# says which one was provisioned when, because nothing else then touches that field.
appliances.sort(key=lambda appliance: appliance.status_changed)
pool_size = gs.template_pool_size if preconfigured else gs.unconfigured_template_pool_size
if len(appliances) < pool_size and possible_templates_for_provision:
# There must be some templates in order to run the provisioning
# Provision ONE appliance at time for each group, that way it is possible to maintain
# reasonable balancing
with transaction.atomic():
# Now look for templates that are on non-busy providers
tpl_free = [
t for t in possible_templates_for_provision
if not t.provider.disabled and t.provider.free
]
if tpl_free:
chosen_template = sorted(
tpl_free, key=lambda t: t.provider.appliance_load)[0]
new_appliance_name = gen_appliance_name(chosen_template.id)
appliance = Appliance(template=chosen_template,
name=new_appliance_name)
appliance.save()
self.logger.info("Adding an appliance to shepherd: %s/%s",
appliance.id, appliance.name)
clone_template_to_appliance.delay(appliance.id, None)
elif len(appliances) > pool_size:
# Too many appliances, kill the surplus
# Only kill those that are visible only for one group. This is necessary so the groups
# don't "fight"
for appliance in appliances[:len(appliances) - pool_size]:
if appliance.is_visible_only_in_group(gs.user_group):
self.logger.info(
"Killing an extra appliance {}/{} in shepherd".format(
appliance.id, appliance.name))
Appliance.kill(appliance)
# Killing old appliances
for filter_kill in filters_kill:
for template in Template.objects.filter(
ready=True,
usable=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_kill):
for a in Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False):
self.logger.info(
"Killing appliance {}/{} in shepherd because it is obsolete now"
.format(a.id, a.name))
Appliance.kill(a)
