def date_for_group_and_version(request):
group_id = request.POST.get("stream")
latest_date = None
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
if group_id == "<None>":
dates = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
dates = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
}
if version == "latest":
try:
versions = Template.get_versions(**filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
dates = Template.get_dates(**filters)
if dates:
latest_date = dates[0]
return render(request, 'appliances/_dates.html', locals())
def date_for_group_and_version(request):
group_id = request.POST.get("stream")
latest_date = None
preconfigured = request.POST.get("preconfigured",
"false").lower() == "true"
if group_id == "<None>":
dates = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
dates = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
}
if version == "latest":
try:
versions = Template.get_versions(**filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
dates = Template.get_dates(**filters)
if dates:
latest_date = dates[0]
return render(request, 'appliances/_dates.html', locals())
def num_shepherd_appliances(user,
group,
version=None,
date=None,
provider=None):
"""Provides number of currently available shepherd appliances."""
group = Group.objects.get(id=group)
if provider is not None:
provider = Provider.objects.get(id=provider)
if version is None:
if provider is None:
try:
version = Template.get_versions(template_group=group)[0]
except IndexError:
# No version
pass
else:
try:
version = Template.get_versions(template_group=group,
provider=provider)[0]
except IndexError:
# No version
pass
if date is None:
filter_kwargs = {"template_group": group}
if provider is not None:
filter_kwargs["provider"] = provider
if version is not None:
filter_kwargs["version"] = version
try:
date = Template.get_dates(**filter_kwargs)[0]
except IndexError:
# No date
pass
filter_kwargs = {
"template__template_group": group,
"ready": True,
"appliance_pool": None
}
if version is not None:
filter_kwargs["template__version"] = version
if date is not None:
filter_kwargs["template__date"] = date
if provider is not None:
filter_kwargs["template__provider"] = provider
return len(Appliance.objects.filter(**filter_kwargs))
def providers_for_date_group_and_version(request):
total_provisioning_slots = 0
total_appliance_slots = 0
group_id = request.POST.get("stream")
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
if group_id == "<None>":
providers = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
providers = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
}
if version == "latest":
try:
versions = Template.get_versions(**filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
date = request.POST.get("date")
if date == "latest":
try:
dates = Template.get_dates(**filters)
filters["date"] = dates[0]
except IndexError:
pass # No such thing as date for this template group
else:
filters["date"] = parser.parse(date)
providers = Template.objects.filter(**filters).values("provider").distinct()
providers = sorted([p.values()[0] for p in providers])
providers = [Provider.objects.get(id=provider) for provider in providers]
for provider in providers:
total_appliance_slots += provider.remaining_appliance_slots
total_provisioning_slots += provider.remaining_provisioning_slots
return render(request, 'appliances/_providers.html', locals())
def date_for_group_and_version(request):
if not request.user.is_authenticated():
return go_home(request)
group_id = request.POST.get("stream")
latest_date = None
preconfigured = request.POST.get("preconfigured",
"false").lower() == "true"
container = request.POST.get("container", "false").lower() == "true"
container_q = ~Q(container=None) if container else Q(container=None)
if group_id == "<None>":
dates = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
dates = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
'provider__disabled': False,
"provider__user_groups__in": request.user.groups.all(),
}
if version == "latest":
try:
versions = Template.get_versions(container_q, **filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
dates = Template.get_dates(container_q, **filters)
if dates:
latest_date = dates[0]
return render(request, 'appliances/_dates.html', locals())
def date_for_group_and_version(request):
if not request.user.is_authenticated():
return go_home(request)
group_id = request.POST.get("stream")
latest_date = None
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
container = request.POST.get("container", "false").lower() == "true"
container_q = ~Q(container=None) if container else Q(container=None)
if group_id == "<None>":
dates = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
dates = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
'provider__disabled': False,
"provider__user_groups__in": request.user.groups.all(),
}
if version == "latest":
try:
versions = Template.get_versions(container_q, **filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
dates = Template.get_dates(container_q, **filters)
if dates:
latest_date = dates[0]
return render(request, 'appliances/_dates.html', locals())
def num_shepherd_appliances(group, version=None, date=None, provider=None):
"""Provides number of currently available shepherd appliances."""
group = Group.objects.get(id=group)
if provider is not None:
provider = Provider.objects.get(id=provider)
if version is None:
if provider is None:
try:
version = Template.get_versions(template_group=group)[0]
except IndexError:
# No version
pass
else:
try:
version = Template.get_versions(template_group=group, provider=provider)[0]
except IndexError:
# No version
pass
if date is None:
filter_kwargs = {"template_group": group}
if provider is not None:
filter_kwargs["provider"] = provider
if version is not None:
filter_kwargs["version"] = version
try:
date = Template.get_dates(**filter_kwargs)[0]
except IndexError:
# No date
pass
filter_kwargs = {"template__template_group": group, "ready": True, "appliance_pool": None}
if version is not None:
filter_kwargs["template__version"] = version
if date is not None:
filter_kwargs["template__date"] = date
if provider is not None:
filter_kwargs["template__provider"] = provider
return len(Appliance.objects.filter(**filter_kwargs))
def providers_for_date_group_and_version(request):
if not request.user.is_authenticated():
return go_home(request)
total_provisioning_slots = 0
total_appliance_slots = 0
total_shepherd_slots = 0
shepherd_appliances = {}
group_id = request.POST.get("stream")
preconfigured = request.POST.get("preconfigured",
"false").lower() == "true"
container = request.POST.get("container", "false").lower() == "true"
container_q = ~Q(container=None) if container else Q(container=None)
if container:
appliance_container_q = ~Q(template__container=None)
else:
appliance_container_q = Q(template__container=None)
if group_id == "<None>":
providers = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
providers = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
"provider__disabled": False,
"provider__user_groups__in": request.user.groups.all(),
}
if version == "latest":
try:
versions = Template.get_versions(container_q, **filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
date = request.POST.get("date")
if date == "latest":
try:
dates = Template.get_dates(container_q, **filters)
filters["date"] = dates[0]
except IndexError:
pass # No such thing as date for this template group
else:
filters["date"] = parser.parse(date)
providers = Template.objects.filter(
container_q, **filters).values("provider").distinct()
providers = sorted([p.values()[0] for p in providers])
providers = [
Provider.objects.get(id=provider) for provider in providers
]
for provider in providers:
appl_filter = dict(
appliance_pool=None,
ready=True,
template__provider=provider,
template__preconfigured=filters["preconfigured"],
template__template_group=filters["template_group"])
if "date" in filters:
appl_filter["template__date"] = filters["date"]
if "version" in filters:
appl_filter["template__version"] = filters["version"]
shepherd_appliances[provider.id] = len(
Appliance.objects.filter(appliance_container_q,
**appl_filter))
total_shepherd_slots += shepherd_appliances[provider.id]
total_appliance_slots += provider.remaining_appliance_slots
total_provisioning_slots += provider.remaining_provisioning_slots
render_providers = {}
for provider in providers:
render_providers[provider.id] = {
"shepherd_count": shepherd_appliances[provider.id],
"object": provider
}
return render(request, 'appliances/_providers.html', locals())
def providers_for_date_group_and_version(request):
if not request.user.is_authenticated():
return go_home(request)
total_provisioning_slots = 0
total_appliance_slots = 0
total_shepherd_slots = 0
shepherd_appliances = {}
group_id = request.POST.get("stream")
preconfigured = request.POST.get("preconfigured", "false").lower() == "true"
container = request.POST.get("container", "false").lower() == "true"
container_q = ~Q(container=None) if container else Q(container=None)
if container:
appliance_container_q = ~Q(template__container=None)
else:
appliance_container_q = Q(template__container=None)
if group_id == "<None>":
providers = []
else:
try:
group = Group.objects.get(id=group_id)
except ObjectDoesNotExist:
providers = []
else:
version = request.POST.get("version")
filters = {
"template_group": group,
"ready": True,
"exists": True,
"usable": True,
"preconfigured": preconfigured,
"provider__working": True,
"provider__user_groups__in": request.user.groups.all(),
}
if version == "latest":
try:
versions = Template.get_versions(container_q, **filters)
filters["version"] = versions[0]
except IndexError:
pass # No such thing as version for this template group
else:
filters["version"] = version
date = request.POST.get("date")
if date == "latest":
try:
dates = Template.get_dates(container_q, **filters)
filters["date"] = dates[0]
except IndexError:
pass # No such thing as date for this template group
else:
filters["date"] = parser.parse(date)
providers = Template.objects.filter(
container_q, **filters).values("provider").distinct()
providers = sorted([p.values()[0] for p in providers])
providers = [Provider.objects.get(id=provider) for provider in providers]
for provider in providers:
appl_filter = dict(
appliance_pool=None, ready=True, template__provider=provider,
template__preconfigured=filters["preconfigured"],
template__template_group=filters["template_group"])
if "date" in filters:
appl_filter["template__date"] = filters["date"]
if "version" in filters:
appl_filter["template__version"] = filters["version"]
shepherd_appliances[provider.id] = len(
Appliance.objects.filter(appliance_container_q, **appl_filter))
total_shepherd_slots += shepherd_appliances[provider.id]
total_appliance_slots += provider.remaining_appliance_slots
total_provisioning_slots += provider.remaining_provisioning_slots
render_providers = {}
for provider in providers:
render_providers[provider.id] = {
"shepherd_count": shepherd_appliances[provider.id], "object": provider}
return render(request, 'appliances/_providers.html', locals())
def generic_shepherd(self, preconfigured):
"""This task takes care of having the required templates spinned into required number of
appliances. For each template group, it keeps the last template's appliances spinned up in
required quantity. If new template comes out of the door, it automatically kills the older
running template's appliances and spins up new ones. Sorts the groups by the fulfillment."""
for gs in sorted(
GroupShepherd.objects.all(),
key=lambda g: g.get_fulfillment_percentage(preconfigured)):
prov_filter = {'provider__user_groups': gs.user_group}
group_versions = Template.get_versions(
template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
group_dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
preconfigured=preconfigured,
**prov_filter)
if group_versions:
# Downstream - by version (downstream releases)
version = group_versions[0]
# Find the latest date (one version can have new build)
dates = Template.get_dates(template_group=gs.template_group,
ready=True,
usable=True,
version=group_versions[0],
preconfigured=preconfigured,
**prov_filter)
if not dates:
# No template yet?
continue
date = dates[0]
filter_keep = {"version": version, "date": date}
filters_kill = []
for kill_date in dates[1:]:
filters_kill.append({"version": version, "date": kill_date})
for kill_version in group_versions[1:]:
filters_kill.append({"version": kill_version})
elif group_dates:
# Upstream - by date (upstream nightlies)
filter_keep = {"date": group_dates[0]}
filters_kill = [{"date": v} for v in group_dates[1:]]
else:
continue # Ignore this group, no templates detected yet
filter_keep.update(prov_filter)
for filt in filters_kill:
filt.update(prov_filter)
# Keeping current appliances
# Retrieve list of all templates for given group
# I know joins might be a bit better solution but I'll leave that for later.
possible_templates = list(
Template.objects.filter(usable=True,
ready=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_keep).all())
# If it can be deployed, it must exist
possible_templates_for_provision = [
tpl for tpl in possible_templates if tpl.exists
]
appliances = []
for template in possible_templates:
appliances.extend(
Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False))
# If we then want to delete some templates, better kill the eldest. status_changed
# says which one was provisioned when, because nothing else then touches that field.
appliances.sort(key=lambda appliance: appliance.status_changed)
pool_size = gs.template_pool_size if preconfigured else gs.unconfigured_template_pool_size
if len(appliances) < pool_size and possible_templates_for_provision:
# There must be some templates in order to run the provisioning
# Provision ONE appliance at time for each group, that way it is possible to maintain
# reasonable balancing
with transaction.atomic():
# Now look for templates that are on non-busy providers
tpl_free = [
t for t in possible_templates_for_provision
if not t.provider.disabled and t.provider.free
]
if tpl_free:
chosen_template = sorted(
tpl_free, key=lambda t: t.provider.appliance_load)[0]
new_appliance_name = gen_appliance_name(chosen_template.id)
appliance = Appliance(template=chosen_template,
name=new_appliance_name)
appliance.save()
self.logger.info("Adding an appliance to shepherd: %s/%s",
appliance.id, appliance.name)
clone_template_to_appliance.delay(appliance.id, None)
elif len(appliances) > pool_size:
# Too many appliances, kill the surplus
# Only kill those that are visible only for one group. This is necessary so the groups
# don't "fight"
for appliance in appliances[:len(appliances) - pool_size]:
if appliance.is_visible_only_in_group(gs.user_group):
self.logger.info(
"Killing an extra appliance {}/{} in shepherd".format(
appliance.id, appliance.name))
Appliance.kill(appliance)
# Killing old appliances
for filter_kill in filters_kill:
for template in Template.objects.filter(
ready=True,
usable=True,
template_group=gs.template_group,
preconfigured=preconfigured,
**filter_kill):
for a in Appliance.objects.filter(template=template,
appliance_pool=None,
marked_for_deletion=False):
self.logger.info(
"Killing appliance {}/{} in shepherd because it is obsolete now"
.format(a.id, a.name))
Appliance.kill(a)
