def date_for_group_and_version(request): group_id = request.POST.get("stream") latest_date = None preconfigured = request.POST.get("preconfigured", "false").lower() == "true" if group_id == "<None>": dates = [] else: try: group = Group.objects.get(id=group_id) except ObjectDoesNotExist: dates = [] else: version = request.POST.get("version") filters = { "template_group": group, "ready": True, "exists": True, "usable": True, "preconfigured": preconfigured, "provider__working": True, } if version == "latest": try: versions = Template.get_versions(**filters) filters["version"] = versions[0] except IndexError: pass # No such thing as version for this template group else: filters["version"] = version dates = Template.get_dates(**filters) if dates: latest_date = dates[0] return render(request, 'appliances/_dates.html', locals())
def date_for_group_and_version(request): group_id = request.POST.get("stream") latest_date = None preconfigured = request.POST.get("preconfigured", "false").lower() == "true" if group_id == "<None>": dates = [] else: try: group = Group.objects.get(id=group_id) except ObjectDoesNotExist: dates = [] else: version = request.POST.get("version") filters = { "template_group": group, "ready": True, "exists": True, "usable": True, "preconfigured": preconfigured, "provider__working": True, } if version == "latest": try: versions = Template.get_versions(**filters) filters["version"] = versions[0] except IndexError: pass # No such thing as version for this template group else: filters["version"] = version dates = Template.get_dates(**filters) if dates: latest_date = dates[0] return render(request, 'appliances/_dates.html', locals())
def num_shepherd_appliances(user, group, version=None, date=None, provider=None): """Provides number of currently available shepherd appliances.""" group = Group.objects.get(id=group) if provider is not None: provider = Provider.objects.get(id=provider) if version is None: if provider is None: try: version = Template.get_versions(template_group=group)[0] except IndexError: # No version pass else: try: version = Template.get_versions(template_group=group, provider=provider)[0] except IndexError: # No version pass if date is None: filter_kwargs = {"template_group": group} if provider is not None: filter_kwargs["provider"] = provider if version is not None: filter_kwargs["version"] = version try: date = Template.get_dates(**filter_kwargs)[0] except IndexError: # No date pass filter_kwargs = { "template__template_group": group, "ready": True, "appliance_pool": None } if version is not None: filter_kwargs["template__version"] = version if date is not None: filter_kwargs["template__date"] = date if provider is not None: filter_kwargs["template__provider"] = provider return len(Appliance.objects.filter(**filter_kwargs))
def providers_for_date_group_and_version(request): total_provisioning_slots = 0 total_appliance_slots = 0 group_id = request.POST.get("stream") preconfigured = request.POST.get("preconfigured", "false").lower() == "true" if group_id == "<None>": providers = [] else: try: group = Group.objects.get(id=group_id) except ObjectDoesNotExist: providers = [] else: version = request.POST.get("version") filters = { "template_group": group, "ready": True, "exists": True, "usable": True, "preconfigured": preconfigured, } if version == "latest": try: versions = Template.get_versions(**filters) filters["version"] = versions[0] except IndexError: pass # No such thing as version for this template group else: filters["version"] = version date = request.POST.get("date") if date == "latest": try: dates = Template.get_dates(**filters) filters["date"] = dates[0] except IndexError: pass # No such thing as date for this template group else: filters["date"] = parser.parse(date) providers = Template.objects.filter(**filters).values("provider").distinct() providers = sorted([p.values()[0] for p in providers]) providers = [Provider.objects.get(id=provider) for provider in providers] for provider in providers: total_appliance_slots += provider.remaining_appliance_slots total_provisioning_slots += provider.remaining_provisioning_slots return render(request, 'appliances/_providers.html', locals())
def date_for_group_and_version(request): if not request.user.is_authenticated(): return go_home(request) group_id = request.POST.get("stream") latest_date = None preconfigured = request.POST.get("preconfigured", "false").lower() == "true" container = request.POST.get("container", "false").lower() == "true" container_q = ~Q(container=None) if container else Q(container=None) if group_id == "<None>": dates = [] else: try: group = Group.objects.get(id=group_id) except ObjectDoesNotExist: dates = [] else: version = request.POST.get("version") filters = { "template_group": group, "ready": True, "exists": True, "usable": True, "preconfigured": preconfigured, "provider__working": True, 'provider__disabled': False, "provider__user_groups__in": request.user.groups.all(), } if version == "latest": try: versions = Template.get_versions(container_q, **filters) filters["version"] = versions[0] except IndexError: pass # No such thing as version for this template group else: filters["version"] = version dates = Template.get_dates(container_q, **filters) if dates: latest_date = dates[0] return render(request, 'appliances/_dates.html', locals())
def date_for_group_and_version(request): if not request.user.is_authenticated(): return go_home(request) group_id = request.POST.get("stream") latest_date = None preconfigured = request.POST.get("preconfigured", "false").lower() == "true" container = request.POST.get("container", "false").lower() == "true" container_q = ~Q(container=None) if container else Q(container=None) if group_id == "<None>": dates = [] else: try: group = Group.objects.get(id=group_id) except ObjectDoesNotExist: dates = [] else: version = request.POST.get("version") filters = { "template_group": group, "ready": True, "exists": True, "usable": True, "preconfigured": preconfigured, "provider__working": True, 'provider__disabled': False, "provider__user_groups__in": request.user.groups.all(), } if version == "latest": try: versions = Template.get_versions(container_q, **filters) filters["version"] = versions[0] except IndexError: pass # No such thing as version for this template group else: filters["version"] = version dates = Template.get_dates(container_q, **filters) if dates: latest_date = dates[0] return render(request, 'appliances/_dates.html', locals())
def num_shepherd_appliances(group, version=None, date=None, provider=None): """Provides number of currently available shepherd appliances.""" group = Group.objects.get(id=group) if provider is not None: provider = Provider.objects.get(id=provider) if version is None: if provider is None: try: version = Template.get_versions(template_group=group)[0] except IndexError: # No version pass else: try: version = Template.get_versions(template_group=group, provider=provider)[0] except IndexError: # No version pass if date is None: filter_kwargs = {"template_group": group} if provider is not None: filter_kwargs["provider"] = provider if version is not None: filter_kwargs["version"] = version try: date = Template.get_dates(**filter_kwargs)[0] except IndexError: # No date pass filter_kwargs = {"template__template_group": group, "ready": True, "appliance_pool": None} if version is not None: filter_kwargs["template__version"] = version if date is not None: filter_kwargs["template__date"] = date if provider is not None: filter_kwargs["template__provider"] = provider return len(Appliance.objects.filter(**filter_kwargs))
def providers_for_date_group_and_version(request): if not request.user.is_authenticated(): return go_home(request) total_provisioning_slots = 0 total_appliance_slots = 0 total_shepherd_slots = 0 shepherd_appliances = {} group_id = request.POST.get("stream") preconfigured = request.POST.get("preconfigured", "false").lower() == "true" container = request.POST.get("container", "false").lower() == "true" container_q = ~Q(container=None) if container else Q(container=None) if container: appliance_container_q = ~Q(template__container=None) else: appliance_container_q = Q(template__container=None) if group_id == "<None>": providers = [] else: try: group = Group.objects.get(id=group_id) except ObjectDoesNotExist: providers = [] else: version = request.POST.get("version") filters = { "template_group": group, "ready": True, "exists": True, "usable": True, "preconfigured": preconfigured, "provider__working": True, "provider__disabled": False, "provider__user_groups__in": request.user.groups.all(), } if version == "latest": try: versions = Template.get_versions(container_q, **filters) filters["version"] = versions[0] except IndexError: pass # No such thing as version for this template group else: filters["version"] = version date = request.POST.get("date") if date == "latest": try: dates = Template.get_dates(container_q, **filters) filters["date"] = dates[0] except IndexError: pass # No such thing as date for this template group else: filters["date"] = parser.parse(date) providers = Template.objects.filter( container_q, **filters).values("provider").distinct() providers = sorted([p.values()[0] for p in providers]) providers = [ Provider.objects.get(id=provider) for provider in providers ] for provider in providers: appl_filter = dict( appliance_pool=None, ready=True, template__provider=provider, template__preconfigured=filters["preconfigured"], template__template_group=filters["template_group"]) if "date" in filters: appl_filter["template__date"] = filters["date"] if "version" in filters: appl_filter["template__version"] = filters["version"] shepherd_appliances[provider.id] = len( Appliance.objects.filter(appliance_container_q, **appl_filter)) total_shepherd_slots += shepherd_appliances[provider.id] total_appliance_slots += provider.remaining_appliance_slots total_provisioning_slots += provider.remaining_provisioning_slots render_providers = {} for provider in providers: render_providers[provider.id] = { "shepherd_count": shepherd_appliances[provider.id], "object": provider } return render(request, 'appliances/_providers.html', locals())
def providers_for_date_group_and_version(request): if not request.user.is_authenticated(): return go_home(request) total_provisioning_slots = 0 total_appliance_slots = 0 total_shepherd_slots = 0 shepherd_appliances = {} group_id = request.POST.get("stream") preconfigured = request.POST.get("preconfigured", "false").lower() == "true" container = request.POST.get("container", "false").lower() == "true" container_q = ~Q(container=None) if container else Q(container=None) if container: appliance_container_q = ~Q(template__container=None) else: appliance_container_q = Q(template__container=None) if group_id == "<None>": providers = [] else: try: group = Group.objects.get(id=group_id) except ObjectDoesNotExist: providers = [] else: version = request.POST.get("version") filters = { "template_group": group, "ready": True, "exists": True, "usable": True, "preconfigured": preconfigured, "provider__working": True, "provider__user_groups__in": request.user.groups.all(), } if version == "latest": try: versions = Template.get_versions(container_q, **filters) filters["version"] = versions[0] except IndexError: pass # No such thing as version for this template group else: filters["version"] = version date = request.POST.get("date") if date == "latest": try: dates = Template.get_dates(container_q, **filters) filters["date"] = dates[0] except IndexError: pass # No such thing as date for this template group else: filters["date"] = parser.parse(date) providers = Template.objects.filter( container_q, **filters).values("provider").distinct() providers = sorted([p.values()[0] for p in providers]) providers = [Provider.objects.get(id=provider) for provider in providers] for provider in providers: appl_filter = dict( appliance_pool=None, ready=True, template__provider=provider, template__preconfigured=filters["preconfigured"], template__template_group=filters["template_group"]) if "date" in filters: appl_filter["template__date"] = filters["date"] if "version" in filters: appl_filter["template__version"] = filters["version"] shepherd_appliances[provider.id] = len( Appliance.objects.filter(appliance_container_q, **appl_filter)) total_shepherd_slots += shepherd_appliances[provider.id] total_appliance_slots += provider.remaining_appliance_slots total_provisioning_slots += provider.remaining_provisioning_slots render_providers = {} for provider in providers: render_providers[provider.id] = { "shepherd_count": shepherd_appliances[provider.id], "object": provider} return render(request, 'appliances/_providers.html', locals())
def generic_shepherd(self, preconfigured): """This task takes care of having the required templates spinned into required number of appliances. For each template group, it keeps the last template's appliances spinned up in required quantity. If new template comes out of the door, it automatically kills the older running template's appliances and spins up new ones. Sorts the groups by the fulfillment.""" for gs in sorted( GroupShepherd.objects.all(), key=lambda g: g.get_fulfillment_percentage(preconfigured)): prov_filter = {'provider__user_groups': gs.user_group} group_versions = Template.get_versions( template_group=gs.template_group, ready=True, usable=True, preconfigured=preconfigured, **prov_filter) group_dates = Template.get_dates(template_group=gs.template_group, ready=True, usable=True, preconfigured=preconfigured, **prov_filter) if group_versions: # Downstream - by version (downstream releases) version = group_versions[0] # Find the latest date (one version can have new build) dates = Template.get_dates(template_group=gs.template_group, ready=True, usable=True, version=group_versions[0], preconfigured=preconfigured, **prov_filter) if not dates: # No template yet? continue date = dates[0] filter_keep = {"version": version, "date": date} filters_kill = [] for kill_date in dates[1:]: filters_kill.append({"version": version, "date": kill_date}) for kill_version in group_versions[1:]: filters_kill.append({"version": kill_version}) elif group_dates: # Upstream - by date (upstream nightlies) filter_keep = {"date": group_dates[0]} filters_kill = [{"date": v} for v in group_dates[1:]] else: continue # Ignore this group, no templates detected yet filter_keep.update(prov_filter) for filt in filters_kill: filt.update(prov_filter) # Keeping current appliances # Retrieve list of all templates for given group # I know joins might be a bit better solution but I'll leave that for later. possible_templates = list( Template.objects.filter(usable=True, ready=True, template_group=gs.template_group, preconfigured=preconfigured, **filter_keep).all()) # If it can be deployed, it must exist possible_templates_for_provision = [ tpl for tpl in possible_templates if tpl.exists ] appliances = [] for template in possible_templates: appliances.extend( Appliance.objects.filter(template=template, appliance_pool=None, marked_for_deletion=False)) # If we then want to delete some templates, better kill the eldest. status_changed # says which one was provisioned when, because nothing else then touches that field. appliances.sort(key=lambda appliance: appliance.status_changed) pool_size = gs.template_pool_size if preconfigured else gs.unconfigured_template_pool_size if len(appliances) < pool_size and possible_templates_for_provision: # There must be some templates in order to run the provisioning # Provision ONE appliance at time for each group, that way it is possible to maintain # reasonable balancing with transaction.atomic(): # Now look for templates that are on non-busy providers tpl_free = [ t for t in possible_templates_for_provision if not t.provider.disabled and t.provider.free ] if tpl_free: chosen_template = sorted( tpl_free, key=lambda t: t.provider.appliance_load)[0] new_appliance_name = gen_appliance_name(chosen_template.id) appliance = Appliance(template=chosen_template, name=new_appliance_name) appliance.save() self.logger.info("Adding an appliance to shepherd: %s/%s", appliance.id, appliance.name) clone_template_to_appliance.delay(appliance.id, None) elif len(appliances) > pool_size: # Too many appliances, kill the surplus # Only kill those that are visible only for one group. This is necessary so the groups # don't "fight" for appliance in appliances[:len(appliances) - pool_size]: if appliance.is_visible_only_in_group(gs.user_group): self.logger.info( "Killing an extra appliance {}/{} in shepherd".format( appliance.id, appliance.name)) Appliance.kill(appliance) # Killing old appliances for filter_kill in filters_kill: for template in Template.objects.filter( ready=True, usable=True, template_group=gs.template_group, preconfigured=preconfigured, **filter_kill): for a in Appliance.objects.filter(template=template, appliance_pool=None, marked_for_deletion=False): self.logger.info( "Killing appliance {}/{} in shepherd because it is obsolete now" .format(a.id, a.name)) Appliance.kill(a)