def create_docker_vm(self, group_id, provider_id, version, date, pull_url):
group = Group.objects.get(id=group_id)
provider = Provider.objects.get(id=provider_id,
working=True,
disabled=False)
with transaction.atomic():
if provider.remaining_configuring_slots < 1:
self.retry(args=(group_id, provider_id, version, date, pull_url),
countdown=60,
max_retries=60)
new_name = docker_vm_name(version, date)
new_template = Template(template_group=group,
provider=provider,
container='cfme',
name=new_name,
original_name=provider.container_base_template,
version=version,
date=date,
ready=False,
exists=False,
usable=True,
preconfigured=True,
template_type=Template.DOCKER_VM)
new_template.save()
workflow = chain(
prepare_template_deploy.si(new_template.id),
configure_docker_template.si(new_template.id, pull_url),
prepare_template_seal.si(new_template.id),
prepare_template_poweroff.si(new_template.id),
prepare_template_finish.si(new_template.id),
)
workflow.link_error(prepare_template_delete_on_error.si(new_template.id))
workflow()
def poke_trackerbot(self):
"""This beat-scheduled task periodically polls the trackerbot if there are any new templates.
"""
template_usability = []
# Extract data from trackerbot
tbapi = trackerbot()
objects = depaginate(
tbapi,
tbapi.providertemplate().get(limit=TRACKERBOT_PAGINATE))["objects"]
per_group = {}
for obj in objects:
if obj["template"]["group"]["name"] == 'unknown':
continue
if obj["template"]["group"]["name"] not in per_group:
per_group[obj["template"]["group"]["name"]] = []
per_group[obj["template"]["group"]["name"]].append(obj)
# Sort them using the build date
for group in list(per_group.keys()):
per_group[group] = sorted(per_group[group],
reverse=True,
key=lambda o: o["template"]["datestamp"])
objects = []
# And interleave the the groups
while any(per_group.values()):
for key in list(per_group.keys()):
if per_group[key]:
objects.append(per_group[key].pop(0))
for template in objects:
if template["provider"]["key"] not in list(
conf.cfme_data.management_systems.keys()):
# If we don't use that provider in yamls, set the template as not usable
# 1) It will prevent adding this template if not added
# 2) It'll mark the template as unusable if it already exists
template["usable"] = False
template_usability.append(
(template["provider"]["key"], template["template"]["name"],
template["usable"]))
if not template["usable"]:
continue
group, create = Group.objects.get_or_create(
id=template["template"]["group"]["name"])
# Check if the template is already obsolete
if group.template_obsolete_days is not None:
build_date = parsetime.from_iso_date(
template["template"]["datestamp"])
if build_date <= (parsetime.today() -
timedelta(days=group.template_obsolete_days)):
# It is already obsolete, so ignore it
continue
if conf.cfme_data.management_systems.get(
template["provider"]["key"],
{}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(
id=template["provider"]["key"])
else:
continue
if not provider.is_working:
continue
if "sprout" not in provider.provider_data:
continue
if not provider.provider_type:
provider.provider_type = provider.provider_data.get('type')
provider.save(update_fields=['provider_type'])
template_name = template["template"]["name"]
ga_released = template['template']['ga_released']
custom_data = template['template'].get('custom_data', "{}")
processed_custom_data = yaml.safe_load(custom_data)
template_info = TemplateName.parse_template(template_name)
if not template_info.datestamp:
# Not a CFME/MIQ template, ignore it.
continue
# Original one
original_template = None
try:
original_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False)
if original_template.ga_released != ga_released:
original_template.ga_released = ga_released
original_template.save(update_fields=['ga_released'])
if provider.provider_type == 'openshift':
if original_template.custom_data != custom_data:
original_template.custom_data = processed_custom_data
original_template.template_type = Template.OPENSHIFT_POD
original_template.container = 'cloudforms-0'
original_template.save(update_fields=[
'custom_data', 'container', 'template_type'
])
except ObjectDoesNotExist:
if template_name in provider.templates:
if template_info.datestamp is None:
self.logger.warning(
"Ignoring template {} because it does not have a date!"
.format(template_name))
continue
template_version = template_info.version
if template_version is None:
# Make up a faux version
# First 3 fields of version get parsed as a zstream
# therefore ... makes it a "nil" stream
template_version = "...{}".format(
template_info.datestamp.strftime("%Y%m%d"))
# openshift has two templates bound to one build version
# sometimes sprout tries using second template -extdb that's wrong
# so, such template has to be marked as not usable inside sprout
usable = not template_name.endswith('-extdb')
with transaction.atomic():
tpl = Template(provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False,
date=template_info.datestamp,
ready=True,
exists=True,
usable=usable,
version=template_version)
tpl.save()
if provider.provider_type == 'openshift':
tpl.custom_data = processed_custom_data
tpl.container = 'cloudforms-0'
tpl.template_type = Template.OPENSHIFT_POD
tpl.save(update_fields=[
'container', 'template_type', 'custom_data'
])
original_template = tpl
self.logger.info("Created a new template #{}".format(
tpl.id))
# If the provider is set to not preconfigure templates, do not bother even doing it.
if provider.num_simultaneous_configuring > 0:
# Preconfigured one
try:
# openshift providers don't have preconfigured templates.
# so regular template should be used
if provider.provider_type != 'openshift':
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
preconfigured=True)
else:
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
name=template_name,
preconfigured=True)
preconfigured_template.custom_data = processed_custom_data
preconfigured_template.container = 'cloudforms-0'
preconfigured_template.template_type = Template.OPENSHIFT_POD
preconfigured_template.save(update_fields=[
'container', 'template_type', 'custom_data'
])
if preconfigured_template.ga_released != ga_released:
preconfigured_template.ga_released = ga_released
preconfigured_template.save(update_fields=['ga_released'])
except ObjectDoesNotExist:
if template_name in provider.templates and provider.provider_type != 'openshift':
original_id = original_template.id if original_template is not None else None
create_appliance_template.delay(
provider.id,
group.id,
template_name,
source_template_id=original_id)
# If any of the templates becomes unusable, let sprout know about it
# Similarly if some of them becomes usable ...
for provider_id, template_name, usability in template_usability:
if conf.cfme_data.management_systems.get(provider_id, {}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(id=provider_id)
else:
continue
if not provider.working or provider.disabled:
continue
with transaction.atomic():
for template in Template.objects.filter(
provider=provider, original_name=template_name):
template.usable = usability
template.save(update_fields=['usable'])
# Kill all shepherd appliances if they were accidentally spun up
if not usability:
for appliance in Appliance.objects.filter(
template=template,
marked_for_deletion=False,
appliance_pool=None):
self.logger.info('Killing an appliance {}/{} '
'because its template was marked '
'as unusable'.format(
appliance.id, appliance.name))
Appliance.kill(appliance)
def create_appliance_template(self,
provider_id,
group_id,
template_name,
source_template_id=None):
"""This task creates a template from a fresh CFME template. In case of fatal error during the
operation, the template object is deleted to make sure the operation will be retried next time
when poke_trackerbot runs."""
provider = Provider.objects.get(id=provider_id,
working=True,
disabled=False)
provider.cleanup() # Precaution
group = Group.objects.get(id=group_id)
with transaction.atomic():
# Limit the number of concurrent template configurations
if provider.remaining_configuring_slots == 0:
return False # It will be kicked again when trackerbot gets poked
try:
Template.objects.get(template_group=group,
provider=provider,
original_name=template_name,
preconfigured=True)
return False
except ObjectDoesNotExist:
pass
# Fire off the template preparation
template_info = TemplateName.parse_template(template_name)
template_date_fmt = template_info.datestamp.strftime("%Y%m%d")
if not template_info.datestamp:
return
# Make up a faux version
# First 3 fields of version get parsed as a zstream
# therefore ... makes it a "nil" stream
template_version = template_info.version or "...{}".format(
template_date_fmt)
new_template_name = settings.TEMPLATE_FORMAT.format(
group=group.id,
date=template_date_fmt,
rnd=fauxfactory.gen_alphanumeric(8))
if provider.template_name_length is not None:
allowed_length = provider.template_name_length
# There is some limit
if len(new_template_name) > allowed_length:
# Cut it down
randoms_length = len(new_template_name.rsplit("_", 1)[-1])
minimum_length = (len(new_template_name) -
randoms_length) + 1 # One random must be
if minimum_length <= allowed_length:
# Just cut it
new_template_name = new_template_name[:allowed_length]
else:
# Another solution
new_template_name = settings.TEMPLATE_FORMAT.format(
group=group.id[:2],
date=template_date_fmt, # Use only first 2 of grp
rnd=fauxfactory.gen_alphanumeric(
2)) # And just 2 chars random
# TODO: If anything larger comes, do fix that!
if source_template_id is not None:
try:
source_template = Template.objects.get(id=source_template_id)
except ObjectDoesNotExist:
source_template = None
else:
source_template = None
template = Template(provider=provider,
template_group=group,
name=new_template_name,
date=template_info.datestamp,
version=template_version,
original_name=template_name,
parent_template=source_template,
exists=False)
template.save()
workflow = chain(
prepare_template_deploy.si(template.id),
prepare_template_verify_version.si(template.id),
prepare_template_configure.si(template.id),
prepare_template_seal.si(template.id),
prepare_template_poweroff.si(template.id),
prepare_template_finish.si(template.id),
)
workflow.link_error(prepare_template_delete_on_error.si(template.id))
workflow()
