def date_for_group_and_version(request):
group_id = request.POST.get("stream")
latest_date = None
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
if group_id == "<None>":
dates = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
dates = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
}
if version == "latest":
try:
versions = Template.get_versions(**filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
dates = Template.get_dates(**filters)
if dates:
latest_date = dates[0]
return render(request, 'appliances/_dates.html', locals())
def versions_for_group(request):
if not request.user.is_authenticated():
return go_home(request)
group_id = request.POST.get("stream")
latest_version = None
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
container = request.POST.get("container", "false").lower() == "true"
container_q = ~Q(container=None) if container else Q(container=None)
if group_id == "<None>":
versions = []
group = None
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
versions = []
else:
versions = [
(version, Template.ga_version(version))
for version in Template.get_versions(
container_q,
template_group=group, ready=True, usable=True, exists=True,
preconfigured=preconfigured, provider__working=True, provider__disabled=False,
provider__user_groups__in=request.user.groups.all())]
if versions:
if versions[0][1]:
latest_version = '{} (GA)'.format(versions[0][0])
else:
latest_version = versions[0][0]
return render(request, 'appliances/_versions.html', locals())
def providers_for_date_group_and_version(request):
total_provisioning_slots = 0
total_appliance_slots = 0
group_id = request.POST.get("stream")
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
if group_id == "<None>":
providers = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
providers = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
}
if version == "latest":
try:
versions = Template.get_versions(**filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
date = request.POST.get("date")
if date == "latest":
try:
dates = Template.get_dates(**filters)
filters["date"] = dates[0]
except IndexError:
pass # No such thing as date for this template group
else:
filters["date"] = parser.parse(date)
providers = Template.objects.filter(**filters).values("provider").distinct()
providers = sorted([p.values()[0] for p in providers])
providers = [Provider.objects.get(id=provider) for provider in providers]
for provider in providers:
total_appliance_slots += provider.remaining_appliance_slots
total_provisioning_slots += provider.remaining_provisioning_slots
return render(request, 'appliances/_providers.html', locals())
def date_for_group_and_version(request):
if not request.user.is_authenticated():
return go_home(request)
group_id = request.POST.get("stream")
latest_date = None
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
container = request.POST.get("container", "false").lower() == "true"
container_q = ~Q(container=None) if container else Q(container=None)
if group_id == "<None>":
dates = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
dates = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
'provider__disabled': False,
"provider__user_groups__in": request.user.groups.all(),
}
if version == "latest":
try:
versions = Template.get_versions(container_q, **filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
dates = Template.get_dates(container_q, **filters)
if dates:
latest_date = dates[0]
return render(request, 'appliances/_dates.html', locals())
def num_shepherd_appliances(group, version=None, date=None, provider=None):
"""Provides number of currently available shepherd appliances."""
group = Group.objects.get(id=group)
if provider is not None:
provider = Provider.objects.get(id=provider)
if version is None:
if provider is None:
try:
version = Template.get_versions(template_group=group)[0]
except IndexError:
# No version
pass
else:
try:
version = Template.get_versions(template_group=group, provider=provider)[0]
except IndexError:
# No version
pass
if date is None:
filter_kwargs = {"template_group": group}
if provider is not None:
filter_kwargs["provider"] = provider
if version is not None:
filter_kwargs["version"] = version
try:
date = Template.get_dates(**filter_kwargs)[0]
except IndexError:
# No date
pass
filter_kwargs = {"template__template_group": group, "ready": True, "appliance_pool": None}
if version is not None:
filter_kwargs["template__version"] = version
if date is not None:
filter_kwargs["template__date"] = date
if provider is not None:
filter_kwargs["template__provider"] = provider
return len(Appliance.objects.filter(**filter_kwargs))
def versions_for_group(request):
group_id = request.POST.get("stream")
template_type = request.POST.get("template_type")
latest_version = None
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
if group_id == "<None>":
versions = []
group = None
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
versions = []
else:
filters = {
'template_group': group,
'ready': True,
'usable': True,
'exists': True,
'provider__working': True,
'provider__disabled': False,
'provider__user_groups__in': request.user.groups.all(),
'preconfigured': preconfigured,
'template_type': template_type
}
versions = [
(version, Template.ga_version(version))
for version in Template.get_versions(**filters)]
if versions:
if versions[0][1]:
latest_version = '{} (GA)'.format(versions[0][0])
else:
latest_version = versions[0][0]
return render(request, 'appliances/_versions.html', locals())
def versions_for_group(request):
group_id = request.POST.get("stream")
latest_version = None
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
if group_id == "<None>":
versions = []
group = None
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
versions = []
else:
versions = Template.get_versions(
template_group=group, ready=True, usable=True, exists=True,
preconfigured=preconfigured, provider__working=True, provider__disabled=False)
if versions:
latest_version = versions[0]
return render(request, 'appliances/_versions.html', locals())
def providers_for_date_group_and_version(request):
if not request.user.is_authenticated():
return go_home(request)
total_provisioning_slots = 0
total_appliance_slots = 0
total_shepherd_slots = 0
shepherd_appliances = {}
group_id = request.POST.get("stream")
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
container = request.POST.get("container", "false").lower() == "true"
container_q = ~Q(container=None) if container else Q(container=None)
if container:
appliance_container_q = ~Q(template__container=None)
else:
appliance_container_q = Q(template__container=None)
if group_id == "<None>":
providers = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
providers = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
"provider__user_groups__in": request.user.groups.all(),
}
if version == "latest":
try:
versions = Template.get_versions(container_q, **filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
date = request.POST.get("date")
if date == "latest":
try:
dates = Template.get_dates(container_q, **filters)
filters["date"] = dates[0]
except IndexError:
pass # No such thing as date for this template group
else:
filters["date"] = parser.parse(date)
providers = Template.objects.filter(
container_q, **filters).values("provider").distinct()
providers = sorted([p.values()[0] for p in providers])
providers = [Provider.objects.get(id=provider) for provider in providers]
for provider in providers:
appl_filter = dict(
appliance_pool=None, ready=True, template__provider=provider,
template__preconfigured=filters["preconfigured"],
template__template_group=filters["template_group"])
if "date" in filters:
appl_filter["template__date"] = filters["date"]
if "version" in filters:
appl_filter["template__version"] = filters["version"]
shepherd_appliances[provider.id] = len(
Appliance.objects.filter(appliance_container_q, **appl_filter))
total_shepherd_slots += shepherd_appliances[provider.id]
total_appliance_slots += provider.remaining_appliance_slots
total_provisioning_slots += provider.remaining_provisioning_slots
render_providers = {}
for provider in providers:
render_providers[provider.id] = {
"shepherd_count": shepherd_appliances[provider.id], "object": provider}
return render(request, 'appliances/_providers.html', locals())
def available_cfme_versions(preconfigured=True):
"""Lists all versions that are available"""
return Template.get_versions(preconfigured=preconfigured)
def providers_for_date_group_and_version(request):
total_provisioning_slots = 0
total_appliance_slots = 0
total_shepherd_slots = 0
shepherd_appliances = {}
group_id = request.POST.get("stream")
provider_type = request.POST.get("provider_type")
template_type = request.POST.get("template_type")
if provider_type == 'any' or not provider_type:
provider_type = None
preconfigured = request.POST.get("preconfigured",
"false").lower() == "true"
if group_id == "<None>":
providers = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
providers = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
"provider__disabled": False,
"provider__user_groups__in": request.user.groups.all(),
"template_type": template_type,
}
if version == "latest":
try:
versions = Template.get_versions(**filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
date = request.POST.get("date")
if date == "latest":
try:
dates = Template.get_dates(**filters)
filters["date"] = dates[0]
except IndexError:
pass # No such thing as date for this template group
else:
filters["date"] = parser.parse(date)
providers = Template.objects.filter(
**filters).values("provider").distinct()
providers = sorted([p.values()[0] for p in providers])
providers = list(Provider.objects.filter(id__in=providers))
if provider_type is None:
providers = list(providers)
else:
providers = [
provider for provider in providers
if provider.provider_type == provider_type
]
for provider in providers:
appl_filter = dict(
appliance_pool=None,
ready=True,
template__provider=provider,
template__preconfigured=filters["preconfigured"],
template__template_group=filters["template_group"],
template__template_type=filters["template_type"])
if "date" in filters:
appl_filter["template__date"] = filters["date"]
if "version" in filters:
appl_filter["template__version"] = filters["version"]
shepherd_appliances[provider.id] = len(
Appliance.objects.filter(**appl_filter))
total_shepherd_slots += shepherd_appliances[provider.id]
total_appliance_slots += provider.remaining_appliance_slots
total_provisioning_slots += provider.remaining_provisioning_slots
render_providers = {}
for provider in providers:
render_providers[provider.id] = {
"shepherd_count": shepherd_appliances[provider.id],
"object": provider
}
return render(request, 'appliances/_providers.html', locals())
def available_cfme_versions(preconfigured=True):
"""Lists all versions that are available"""
return Template.get_versions(preconfigured=preconfigured)
def poke_trackerbot(self):
"""This beat-scheduled task periodically polls the trackerbot if there are any new templates.
"""
template_usability = []
# Extract data from trackerbot
tbapi = trackerbot()
objects = depaginate(
tbapi,
tbapi.providertemplate().get(limit=TRACKERBOT_PAGINATE))["objects"]
per_group = {}
for obj in objects:
if obj["template"]["group"]["name"] == 'unknown':
continue
if obj["template"]["group"]["name"] not in per_group:
per_group[obj["template"]["group"]["name"]] = []
per_group[obj["template"]["group"]["name"]].append(obj)
# Sort them using the build date
for group in list(per_group.keys()):
per_group[group] = sorted(per_group[group],
reverse=True,
key=lambda o: o["template"]["datestamp"])
objects = []
# And interleave the the groups
while any(per_group.values()):
for key in list(per_group.keys()):
if per_group[key]:
objects.append(per_group[key].pop(0))
for template in objects:
if template["provider"]["key"] not in list(
conf.cfme_data.management_systems.keys()):
# If we don't use that provider in yamls, set the template as not usable
# 1) It will prevent adding this template if not added
# 2) It'll mark the template as unusable if it already exists
template["usable"] = False
template_usability.append(
(template["provider"]["key"], template["template"]["name"],
template["usable"]))
if not template["usable"]:
continue
group, create = Group.objects.get_or_create(
id=template["template"]["group"]["name"])
# Check if the template is already obsolete
if group.template_obsolete_days is not None:
build_date = parsetime.from_iso_date(
template["template"]["datestamp"])
if build_date <= (parsetime.today() -
timedelta(days=group.template_obsolete_days)):
# It is already obsolete, so ignore it
continue
if conf.cfme_data.management_systems.get(
template["provider"]["key"],
{}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(
id=template["provider"]["key"])
else:
continue
if not provider.is_working:
continue
if "sprout" not in provider.provider_data:
continue
if not provider.provider_type:
provider.provider_type = provider.provider_data.get('type')
provider.save(update_fields=['provider_type'])
template_name = template["template"]["name"]
ga_released = template['template']['ga_released']
custom_data = template['template'].get('custom_data', "{}")
processed_custom_data = yaml.safe_load(custom_data)
template_info = TemplateName.parse_template(template_name)
if not template_info.datestamp:
# Not a CFME/MIQ template, ignore it.
continue
# Original one
original_template = None
try:
original_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False)
if original_template.ga_released != ga_released:
original_template.ga_released = ga_released
original_template.save(update_fields=['ga_released'])
if provider.provider_type == 'openshift':
if original_template.custom_data != custom_data:
original_template.custom_data = processed_custom_data
original_template.template_type = Template.OPENSHIFT_POD
original_template.container = 'cloudforms-0'
original_template.save(update_fields=[
'custom_data', 'container', 'template_type'
])
except ObjectDoesNotExist:
if template_name in provider.templates:
if template_info.datestamp is None:
self.logger.warning(
"Ignoring template {} because it does not have a date!"
.format(template_name))
continue
template_version = template_info.version
if template_version is None:
# Make up a faux version
# First 3 fields of version get parsed as a zstream
# therefore ... makes it a "nil" stream
template_version = "...{}".format(
template_info.datestamp.strftime("%Y%m%d"))
# openshift has two templates bound to one build version
# sometimes sprout tries using second template -extdb that's wrong
# so, such template has to be marked as not usable inside sprout
usable = not template_name.endswith('-extdb')
with transaction.atomic():
tpl = Template(provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False,
date=template_info.datestamp,
ready=True,
exists=True,
usable=usable,
version=template_version)
tpl.save()
if provider.provider_type == 'openshift':
tpl.custom_data = processed_custom_data
tpl.container = 'cloudforms-0'
tpl.template_type = Template.OPENSHIFT_POD
tpl.save(update_fields=[
'container', 'template_type', 'custom_data'
])
original_template = tpl
self.logger.info("Created a new template #{}".format(
tpl.id))
# If the provider is set to not preconfigure templates, do not bother even doing it.
if provider.num_simultaneous_configuring > 0:
# Preconfigured one
try:
# openshift providers don't have preconfigured templates.
# so regular template should be used
if provider.provider_type != 'openshift':
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
preconfigured=True)
else:
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
name=template_name,
preconfigured=True)
preconfigured_template.custom_data = processed_custom_data
preconfigured_template.container = 'cloudforms-0'
preconfigured_template.template_type = Template.OPENSHIFT_POD
preconfigured_template.save(update_fields=[
'container', 'template_type', 'custom_data'
])
if preconfigured_template.ga_released != ga_released:
preconfigured_template.ga_released = ga_released
preconfigured_template.save(update_fields=['ga_released'])
except ObjectDoesNotExist:
if template_name in provider.templates and provider.provider_type != 'openshift':
original_id = original_template.id if original_template is not None else None
create_appliance_template.delay(
provider.id,
group.id,
template_name,
source_template_id=original_id)
# If any of the templates becomes unusable, let sprout know about it
# Similarly if some of them becomes usable ...
for provider_id, template_name, usability in template_usability:
if conf.cfme_data.management_systems.get(provider_id, {}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(id=provider_id)
else:
continue
if not provider.working or provider.disabled:
continue
with transaction.atomic():
for template in Template.objects.filter(
provider=provider, original_name=template_name):
template.usable = usability
template.save(update_fields=['usable'])
# Kill all shepherd appliances if they were accidentally spun up
if not usability:
for appliance in Appliance.objects.filter(
template=template,
marked_for_deletion=False,
appliance_pool=None):
self.logger.info('Killing an appliance {}/{} '
'because its template was marked '
'as unusable'.format(
appliance.id, appliance.name))
Appliance.kill(appliance)
def create_appliance_template(self,
provider_id,
group_id,
template_name,
source_template_id=None):
"""This task creates a template from a fresh CFME template. In case of fatal error during the
operation, the template object is deleted to make sure the operation will be retried next time
when poke_trackerbot runs."""
provider = Provider.objects.get(id=provider_id,
working=True,
disabled=False)
provider.cleanup() # Precaution
group = Group.objects.get(id=group_id)
with transaction.atomic():
# Limit the number of concurrent template configurations
if provider.remaining_configuring_slots == 0:
return False # It will be kicked again when trackerbot gets poked
try:
Template.objects.get(template_group=group,
provider=provider,
original_name=template_name,
preconfigured=True)
return False
except ObjectDoesNotExist:
pass
# Fire off the template preparation
template_info = TemplateName.parse_template(template_name)
template_date_fmt = template_info.datestamp.strftime("%Y%m%d")
if not template_info.datestamp:
return
# Make up a faux version
# First 3 fields of version get parsed as a zstream
# therefore ... makes it a "nil" stream
template_version = template_info.version or "...{}".format(
template_date_fmt)
new_template_name = settings.TEMPLATE_FORMAT.format(
group=group.id,
date=template_date_fmt,
rnd=fauxfactory.gen_alphanumeric(8))
if provider.template_name_length is not None:
allowed_length = provider.template_name_length
# There is some limit
if len(new_template_name) > allowed_length:
# Cut it down
randoms_length = len(new_template_name.rsplit("_", 1)[-1])
minimum_length = (len(new_template_name) -
randoms_length) + 1 # One random must be
if minimum_length <= allowed_length:
# Just cut it
new_template_name = new_template_name[:allowed_length]
else:
# Another solution
new_template_name = settings.TEMPLATE_FORMAT.format(
group=group.id[:2],
date=template_date_fmt, # Use only first 2 of grp
rnd=fauxfactory.gen_alphanumeric(
2)) # And just 2 chars random
# TODO: If anything larger comes, do fix that!
if source_template_id is not None:
try:
source_template = Template.objects.get(id=source_template_id)
except ObjectDoesNotExist:
source_template = None
else:
source_template = None
template = Template(provider=provider,
template_group=group,
name=new_template_name,
date=template_info.datestamp,
version=template_version,
original_name=template_name,
parent_template=source_template,
exists=False)
template.save()
workflow = chain(
prepare_template_deploy.si(template.id),
prepare_template_verify_version.si(template.id),
prepare_template_configure.si(template.id),
prepare_template_seal.si(template.id),
prepare_template_poweroff.si(template.id),
prepare_template_finish.si(template.id),
)
workflow.link_error(prepare_template_delete_on_error.si(template.id))
workflow()
def generic_shepherd(self, preconfigured):
"""This task takes care of having the required templates spinned into required number of
appliances. For each template group, it keeps the last template's appliances spinned up in
required quantity. If new template comes out of the door, it automatically kills the older
running template's appliances and spins up new ones. Sorts the groups by the fulfillment."""
for gs in sorted(
GroupShepherd.objects.all(),
key=lambda g: g.get_fulfillment_percentage(preconfigured)):
prov_filter = {'provider__user_groups': gs.user_group}
group_versions = Template.get_versions(
template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
group_dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
if group_versions:
# Downstream - by version (downstream releases)
version = group_versions[0]
# Find the latest date (one version can have new build)
dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
version=group_versions[0],
preconfigured=preconfigured,
**prov_filter)
if not dates:
# No template yet?
continue
date = dates[0]
filter_keep = {"version": version, "date": date}
filters_kill = []
for kill_date in dates[1:]:
filters_kill.append({"version": version, "date": kill_date})
for kill_version in group_versions[1:]:
filters_kill.append({"version": kill_version})
elif group_dates:
# Upstream - by date (upstream nightlies)
filter_keep = {"date": group_dates[0]}
filters_kill = [{"date": v} for v in group_dates[1:]]
else:
continue # Ignore this group, no templates detected yet
filter_keep.update(prov_filter)
for filt in filters_kill:
filt.update(prov_filter)
# Keeping current appliances
# Retrieve list of all templates for given group
# I know joins might be a bit better solution but I'll leave that for later.
possible_templates = list(
Template.objects.filter(usable=True,
ready=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_keep).all())
# If it can be deployed, it must exist
possible_templates_for_provision = [
tpl for tpl in possible_templates if tpl.exists
]
appliances = []
for template in possible_templates:
appliances.extend(
Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False))
# If we then want to delete some templates, better kill the eldest. status_changed
# says which one was provisioned when, because nothing else then touches that field.
appliances.sort(key=lambda appliance: appliance.status_changed)
pool_size = gs.template_pool_size if preconfigured else gs.unconfigured_template_pool_size
if len(appliances) < pool_size and possible_templates_for_provision:
# There must be some templates in order to run the provisioning
# Provision ONE appliance at time for each group, that way it is possible to maintain
# reasonable balancing
with transaction.atomic():
# Now look for templates that are on non-busy providers
tpl_free = [
t for t in possible_templates_for_provision
if not t.provider.disabled and t.provider.free
]
if tpl_free:
chosen_template = sorted(
tpl_free, key=lambda t: t.provider.appliance_load)[0]
new_appliance_name = gen_appliance_name(chosen_template.id)
appliance = Appliance(template=chosen_template,
name=new_appliance_name)
appliance.save()
self.logger.info("Adding an appliance to shepherd: %s/%s",
appliance.id, appliance.name)
clone_template_to_appliance.delay(appliance.id, None)
elif len(appliances) > pool_size:
# Too many appliances, kill the surplus
# Only kill those that are visible only for one group. This is necessary so the groups
# don't "fight"
for appliance in appliances[:len(appliances) - pool_size]:
if appliance.is_visible_only_in_group(gs.user_group):
self.logger.info(
"Killing an extra appliance {}/{} in shepherd".format(
appliance.id, appliance.name))
Appliance.kill(appliance)
# Killing old appliances
for filter_kill in filters_kill:
for template in Template.objects.filter(
ready=True,
usable=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_kill):
for a in Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False):
self.logger.info(
"Killing appliance {}/{} in shepherd because it is obsolete now"
.format(a.id, a.name))
Appliance.kill(a)
