def get_queryset(self): request_user = self.request.user # Showing non-end dated, public ProviderMachines public_set = ProviderMachine.objects.filter( only_current_source(), only_public_providers(), application_version__application__private=False) if not isinstance(request_user, AnonymousUser): # Showing non-end dated, public ProviderMachines shared_set = ProviderMachine.objects.filter( only_current_source(), members__id__in=request_user.memberships.values_list( 'group__id', flat=True)) # NOTE: Showing 'my pms' EVEN if they are end-dated. my_set = ProviderMachine.objects.filter( user_provider_machine_set(request_user)) if request_user.is_staff: admin_set = get_admin_machines(request_user) else: admin_set = ProviderMachine.objects.none() else: shared_set = my_set = admin_set = ProviderMachine.objects.none() # Order them by date, make sure no dupes. queryset = ( public_set | shared_set | my_set | admin_set).distinct().order_by('-instance_source__start_date') if not isinstance(request_user, AnonymousUser): queryset = queryset.filter( in_provider_list(request_user.current_providers)) return queryset
def get_queryset(self): request_user = self.request.user # Showing non-end dated, public ProviderMachines public_set = ProviderMachine.objects.filter( only_current_source(), only_public_providers(), application_version__application__private=False) if not isinstance(request_user, AnonymousUser): # Showing non-end dated, public ProviderMachines shared_set = ProviderMachine.objects.filter( only_current_source(), members__id__in=request_user.memberships.values_list('group__id', flat=True)) # NOTE: Showing 'my pms' EVEN if they are end-dated. my_set = ProviderMachine.objects.filter(user_provider_machine_set(request_user)) if request_user.is_staff: admin_set = get_admin_machines(request_user) else: admin_set = ProviderMachine.objects.none() else: shared_set = my_set = admin_set = ProviderMachine.objects.none() # Order them by date, make sure no dupes. queryset = ( public_set | shared_set | my_set | admin_set).distinct().order_by( '-instance_source__start_date') if not isinstance(request_user, AnonymousUser): launchable_providers = request_user.current_providers.filter(active=True) queryset = queryset.filter(in_provider_list(launchable_providers)) return queryset
def get_queryset(self): request_user = self.request.user # Showing non-end dated, public ProviderMachines public_set = ProviderMachine.objects.filter( only_current_source(), application_version__application__private=False) if not isinstance(request_user, AnonymousUser): # Showing non-end dated, public ProviderMachines shared_set = ProviderMachine.objects.filter( only_current_source(), members__in=request_user.group_set.values('id')) # NOTE: Showing 'my pms' EVEN if they are end-dated. my_set = ProviderMachine.objects.filter( Q( application_version__application__created_by=request_user ) | Q( instance_source__created_by=request_user)) if request_user.is_staff: admin_set = get_admin_machines(request_user) else: admin_set = ProviderMachine.objects.none() else: shared_set = my_set = admin_set = ProviderMachine.objects.none() # Order them by date, make sure no dupes. queryset = ( public_set | shared_set | my_set | admin_set).distinct().order_by( '-instance_source__start_date') return queryset
def last_machine(self): providermachine_set = self.all_machines # Out of all non-end dated machines in this application last = providermachine_set.filter( query.only_current_source() ).order_by('instance_source__start_date').last() return last
def get_queryset(self): """ Filter out tags for deleted volumes """ user = self.request.user return Volume.objects.filter(only_current_source(), project__owner__user=user)
def search(cls, query, identity=None): if identity: base_apps = Application.objects.filter( # Privately owned OR public machines Q(private=True, providermachine__instance_source__created_by_identity=identity) | Q(private=False, providermachine__instance_source__provider=identity.provider)) else: active_providers = Provider.get_active() base_apps = Application.objects.filter( # Public machines private=False, #Providermachine's provider is active providermachine__instance_source__provider__in=active_providers) # AND query matches on: query_match = base_apps.filter( # app tag name Q(tags__name__icontains=query) # OR app tag desc | Q(tags__description__icontains=query) # OR app name | Q(name__icontains=query) # OR app desc | Q(description__icontains=query), only_current_source()) return query_match.distinct()
def last_machine(self): providermachine_set = self.all_machines # Out of all non-end dated machines in this application last = providermachine_set.filter( only_current_source() ).order_by('instance_source__start_date').last() return last
def handle_machine(args, provider): if not args.machine_alias: print "Error: A machine-alias is required." sys.exit(1) if args.machine_alias == 'all': return ProviderMachine.objects.filter( only_current_source(), instance_source__provider_id=provider.id, ).distinct() elif args.machine_alias == 'most_used': return sort_most_used_machines(provider, limit=20) elif ',' not in args.machine_alias: return [ ProviderMachine.objects.get( instance_source__identifier=args.machine_alias, instance_source__provider_id=provider.id) ] else: machines = args.machine_alias.split(",") print "Batch launch of images detected: %s" % machines return [ ProviderMachine.objects.get( instance_source__identifier=machine_alias, instance_source__provider_id=provider.id) for machine_alias in machines ]
def search(cls, query, identity=None): if identity: base_apps = Application.objects.filter( # Privately owned OR public machines Q(private=True, providermachine__instance_source__created_by_identity=identity ) | Q(private=False, providermachine__instance_source__provider=identity. provider)) else: active_providers = Provider.get_active() base_apps = Application.objects.filter( # Public machines private=False, #Providermachine's provider is active providermachine__instance_source__provider__in=active_providers ) # AND query matches on: query_match = base_apps.filter( # app tag name Q(tags__name__icontains=query) # OR app tag desc | Q(tags__description__icontains=query) # OR app name | Q(name__icontains=query) # OR app desc | Q(description__icontains=query), only_current_source()) return query_match.distinct()
def handle_machine(args, provider): if not args.machine_alias: print "Error: A machine-alias is required." sys.exit(1) if args.machine_alias == "all": return ProviderMachine.objects.filter( only_current_source(), instance_source__provider_id=provider.id ).distinct() elif args.machine_alias == "most_used": return sort_most_used_machines(provider, limit=20) elif "," not in args.machine_alias: return [ ProviderMachine.objects.get( instance_source__identifier=args.machine_alias, instance_source__provider_id=provider.id ) ] else: machines = args.machine_alias.split(",") print "Batch launch of images detected: %s" % machines return [ ProviderMachine.objects.get( instance_source__identifier=machine_alias, instance_source__provider_id=provider.id ) for machine_alias in machines ]
def add_membership(image_version, group): """ This function will add *all* users in the group to *all* providers/machines using this image_version O(N^2) """ for provider_machine in image_version.machines.filter(only_current_source()): add_machine_membership(provider_machine, group)
def get_queryset(self): """ Filter projects by current user """ user = self.request.user return Volume.objects.filter( only_current_source(), instance_source__created_by=user)
def get_queryset(self): """ Filter out tags for deleted volumes """ user = self.request.user return Volume.objects.filter( only_current_source(), project__owner__user=user )
def prune_machines_for( provider_id, print_logs=False, dry_run=False, forced_removal=False): """ Look at the list of machines (as seen by the AccountProvider) if a machine cannot be found in the list, remove it. NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any image unseen by the admin to be prematurely end-dated and removed from the API/UI. """ provider = Provider.objects.get(id=provider_id) now = timezone.now() if print_logs: import logging import sys consolehandler = logging.StreamHandler(sys.stdout) consolehandler.setLevel(logging.DEBUG) celery_logger.addHandler(consolehandler) celery_logger.info("Starting prune_machines for Provider %s @ %s" % (provider, now)) if provider.is_active(): account_driver = get_account_driver(provider) db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) cloud_machines = account_driver.list_all_images() else: db_machines = ProviderMachine.objects.filter( source_in_range(), # like 'only_current..' w/o active_provider instance_source__provider=provider) cloud_machines = [] # Don't do anything if cloud machines == [None,[]] if not cloud_machines and not forced_removal: return # Loop 1 - End-date All machines in the DB that # can NOT be found in the cloud. mach_count = _end_date_missing_database_machines( db_machines, cloud_machines, now=now, dry_run=dry_run) # Loop 2 and 3 - Capture all (still-active) versions without machines, # and all applications without versions. # These are 'outliers' and mainly here for safety-check purposes. ver_count = _remove_versions_without_machines(now=now) app_count = _remove_applications_without_versions(now=now) # Loop 4 - All 'Application' DB objects require # >=1 Version with >=1 ProviderMachine (ACTIVE!) # Apps that don't meet this criteria should be end-dated. app_count += _update_improperly_enddated_applications(now) celery_logger.info( "prune_machines completed for Provider %s : " "%s Applications, %s versions and %s machines pruned." % (provider, app_count, ver_count, mach_count)) if print_logs: celery_logger.removeHandler(consolehandler)
def get_queryset(self): """ Filter projects by current user """ user = self.request.user identity_ids = user.current_identities.values_list('id', flat=True) return Volume.objects.filter( only_current_source(), instance_source__created_by_identity__in=identity_ids)
def get_queryset(self): """ Filter projects by current user """ user = self.request.user identity_ids = user.current_identities.values_list('id',flat=True) return Volume.objects.filter( only_current_source(), instance_source__created_by_identity__in=identity_ids)
def get_queryset(self): """ Filter projects by current user """ user = self.request.user now = timezone.now() return Volume.objects.filter( only_current_source(now), instance_source__created_by=user)
def get_user_volumes(self, project): return [ VolumeSerializer(item, context={ 'request': self.context.get('request') }).data for item in project.volumes.filter( only_current_source(), instance_source__provider__active=True) ]
def prune_machines_for(provider_id, print_logs=False, dry_run=False, forced_removal=False): """ Look at the list of machines (as seen by the AccountProvider) if a machine cannot be found in the list, remove it. NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any image unseen by the admin to be prematurely end-dated and removed from the API/UI. """ provider = Provider.objects.get(id=provider_id) now = timezone.now() if print_logs: console_handler = _init_stdout_logging() celery_logger.info("Starting prune_machines for Provider %s @ %s" % (provider, now)) if provider.is_active(): account_driver = get_account_driver(provider) db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) cloud_machines = account_driver.list_all_images() else: db_machines = ProviderMachine.objects.filter( source_in_range(), # like 'only_current..' w/o active_provider instance_source__provider=provider) cloud_machines = [] # Don't do anything if cloud machines == [None,[]] if not cloud_machines and not forced_removal: return # Loop 1 - End-date All machines in the DB that # can NOT be found in the cloud. mach_count = _end_date_missing_database_machines(db_machines, cloud_machines, now=now, dry_run=dry_run) # Loop 2 and 3 - Capture all (still-active) versions without machines, # and all applications without versions. # These are 'outliers' and mainly here for safety-check purposes. ver_count = _remove_versions_without_machines(now=now) app_count = _remove_applications_without_versions(now=now) # Loop 4 - All 'Application' DB objects require # >=1 Version with >=1 ProviderMachine (ACTIVE!) # Apps that don't meet this criteria should be end-dated. app_count += _update_improperly_enddated_applications(now) celery_logger.info("prune_machines completed for Provider %s : " "%s Applications, %s versions and %s machines pruned." % (provider, app_count, ver_count, mach_count)) if print_logs: _exit_stdout_logging(console_handler)
def get_user_volumes(self, atmo_user): return [ VolumeSerializer( item, context={ 'request': self.context.get('request')}).data for item in atmo_user.volume_set().filter( * only_current_source(), instance_source__provider__active=True, projects=None)]
def get_user_volumes(self, atmo_user): return [ VolumeSerializer(item, context={ 'request': self.context.get('request') }).data for item in atmo_user.volume_set().filter( *only_current_source(), instance_source__provider__active=True, projects=None) ]
def get_user_volumes(self, project): return [ VolumeSerializer( item, context={ 'request': self.context.get('request') } ).data for item in project.volumes.filter( only_current_source(), instance_source__provider__active=True ) ]
def monitor_volumes_for(provider_id, print_logs=False): """ Run the set of tasks related to monitoring sizes for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. """ from service.driver import get_account_driver from core.models import Identity if print_logs: console_handler = _init_stdout_logging() provider = Provider.objects.get(id=provider_id) account_driver = get_account_driver(provider) # Non-End dated volumes on this provider db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider) all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30) seen_volumes = [] for cloud_volume in all_volumes: try: core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid) seen_volumes.append(core_volume) except ObjectDoesNotExist: tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id'] tenant = account_driver.get_project_by_id(tenant_id) tenant_name = tenant.name if tenant else tenant_id try: if not tenant: celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume) raise ObjectDoesNotExist() identity = Identity.objects.filter( contains_credential('ex_project_name', tenant_name), provider=provider ).first() if not identity: raise ObjectDoesNotExist() core_volume = convert_esh_volume( cloud_volume, provider.uuid, identity.uuid, identity.created_by) except ObjectDoesNotExist: celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name)) pass now_time = timezone.now() needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes] for volume in needs_end_date: celery_logger.debug("End dating inactive volume: %s" % volume) volume.end_date = now_time volume.save() if print_logs: _exit_stdout_logging(console_handler) for vol in seen_volumes: vol.esh = None return [vol.instance_source.identifier for vol in seen_volumes]
def remove_membership(image_version, group, accounts=None): """ This function will remove *all* users in the group to *all* providers/machines using this image_version """ for provider_machine in image_version.machines.filter( only_current_source()): prov = provider_machine.instance_source.provider if not accounts: accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) approved_projects = accounts.get_image_members(img.id) for identity_membership in group.identitymembership_set.order_by( 'identity__created_by__username'): if identity_membership.identity.provider != prov: continue # Get project name from the identity's credential-list project_name = identity_membership.identity.get_credential( 'ex_project_name') project = accounts.get_project(project_name) if project and project not in approved_projects: continue # Perform a *DATABASE* remove first. application = provider_machine.application application_version = provider_machine.application_version models.ApplicationMembership.objects.filter( group=group, application=application).delete() logger.info("Removed ApplicationMembership: %s-%s" % (application, group)) models.ApplicationVersionMembership.objects.filter( group=group, image_version=application_version).delete() logger.info("Removed ApplicationVersionMembership: %s-%s" % (application_version, group)) models.ProviderMachineMembership.objects.filter( group=group, provider_machine=provider_machine).delete() logger.info("Removed ProviderMachineMembership: %s-%s" % (provider_machine, group)) # Perform a *CLOUD* remove last. try: accounts.image_manager.unshare_image(img, project_name) except Exception as exc: logger.exception( "Exception occurred while removing user from cloud: %s", exc) logger.info("Removed Cloud Access: %s-%s" % (img, project_name)) return
def remove_membership(image_version, group, accounts=None): """ This function will remove *all* users in the group to *all* providers/machines using this image_version """ for provider_machine in image_version.machines.filter(only_current_source()): prov = provider_machine.instance_source.provider if not accounts: accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) approved_projects = accounts.get_image_members(img.id) for identity_membership in group.identitymembership_set.order_by('identity__created_by__username'): if identity_membership.identity.provider != prov: continue # Get project name from the identity's credential-list project_name = identity_membership.identity.get_credential( 'ex_project_name') project = accounts.get_project(project_name) if project and project not in approved_projects: continue # Perform a *DATABASE* remove first. application = provider_machine.application application_version = provider_machine.application_version models.ApplicationMembership.objects.filter( group=group, application=application).delete() logger.info("Removed ApplicationMembership: %s-%s" % (application, group)) models.ApplicationVersionMembership.objects.filter( group=group, image_version=application_version).delete() logger.info("Removed ApplicationVersionMembership: %s-%s" % (application_version, group)) models.ProviderMachineMembership.objects.filter( group=group, provider_machine=provider_machine).delete() logger.info("Removed ProviderMachineMembership: %s-%s" % (provider_machine, group)) # Perform a *CLOUD* remove last. try: accounts.image_manager.unshare_image(img, project_name) except Exception as exc: logger.exception("Exception occurred while removing user from cloud: %s", exc) logger.info("Removed Cloud Access: %s-%s" % (img, project_name)) return
def add_membership(image_version, group): """ This function will add *all* users in the group to *all* providers/machines using this image_version O(N^2) """ for provider_machine in image_version.machines.filter( only_current_source()): prov = provider_machine.instance_source.provider accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) projects = get_current_projects_for_image(accounts, img.id) for identity_membership in group.identitymembership_set.all(): if identity_membership.identity.provider != prov: continue # Get project name from the identity's credential-list project_name = identity_membership.identity.get_credential( 'ex_project_name') project = accounts.get_project(project_name) if project and project in projects: continue # Share with the *database* first! obj, created = models.ApplicationMembership.objects.get_or_create( group=group, application=provider_machine.application) if created: print "Created new ApplicationMembership: %s" \ % (obj,) obj, created = models.ApplicationVersionMembership.objects.get_or_create( group=group, image_version=provider_machine.application_version) if created: print "Created new ApplicationVersionMembership: %s" \ % (obj,) obj, created = models.ProviderMachineMembership.objects.get_or_create( group=group, provider_machine=provider_machine) if created: print "Created new ProviderMachineMembership: %s" \ % (obj,) # Share with the *cloud* last! accounts.image_manager.share_image(img, project_name) accounts.accept_shared_image(img, project_name) logger.info("Added Cloud Access: %s-%s" % (img, project_name)) continue
def get_queryset(self): """ Filter projects by current user """ user = self.request.user qs = Volume.shared_with_user(user) if 'archived' not in self.request.query_params: qs = qs.filter(only_current_source()) qs = qs\ .select_related('instance_source')\ .select_related("instance_source__created_by")\ .select_related('instance_source__created_by_identity')\ .select_related('project') return qs
def add_membership(image_version, group): """ This function will add *all* users in the group to *all* providers/machines using this image_version O(N^2) """ for provider_machine in image_version.machines.filter(only_current_source()): prov = provider_machine.instance_source.provider accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) projects = get_current_projects_for_image(accounts, img.id) for identity_membership in group.identitymembership_set.all(): if identity_membership.identity.provider != prov: continue # Get project name from the identity's credential-list project_name = identity_membership.identity.get_credential('ex_project_name') project = accounts.get_project(project_name) if project and project in projects: continue # Share with the *database* first! obj, created = models.ApplicationMembership.objects.get_or_create( group=group, application=provider_machine.application) if created: print "Created new ApplicationMembership: %s" \ % (obj,) obj, created = models.ApplicationVersionMembership.objects.get_or_create( group=group, image_version=provider_machine.application_version) if created: print "Created new ApplicationVersionMembership: %s" \ % (obj,) obj, created = models.ProviderMachineMembership.objects.get_or_create( group=group, provider_machine=provider_machine) if created: print "Created new ProviderMachineMembership: %s" \ % (obj,) # Share with the *cloud* last! accounts.image_manager.share_image(img, project_name) accounts.accept_shared_image(img, project_name) logger.info("Added Cloud Access: %s-%s" % (img, project_name)) continue
def search(cls, identity, query): return ProviderMachine.objects.filter( # Privately owned OR public machines Q(application__private=True, instance_source__created_by_identity=identity) | Q(application__private=False, instance_source__provider=identity.provider), # AND query matches on: # app tag name OR # app tag desc OR # app name OR # app desc Q(application__tags__name__icontains=query) | Q(application__tags__description__icontains=query) | Q(application__name__icontains=query) | Q(application__description__icontains=query), only_current_source())
def get_admin_machines(user): """ TODO: This 'just works' and is probably very slow... Look for a better way? """ provider_id_list = user.identity_set.values_list('provider', flat=True) account_providers_list = AccountProvider.objects.filter( provider__id__in=provider_id_list) admin_users = [ap.identity.created_by for ap in account_providers_list] machine_ids = [] for user in admin_users: machine_ids.extend( user.source_set.filter(providermachine__isnull=False).values_list( 'providermachine', flat=True)) admin_list = ProviderMachine.objects.filter(only_current_source(), id__in=machine_ids) return admin_list
def _current_machines(self, request_user=None): """ Return a list of current provider machines. NOTE: Defined as: * Provider is still listed as active * Provider has not exceeded its end_date * The ProviderMachine has not exceeded its end_date """ providermachine_set = self.all_machines pms = providermachine_set.filter(only_current_source(), instance_source__provider__active=True) if request_user: if isinstance(request_user, AnonymousUser): providers = Provider.objects.filter(public=True) else: providers = [identity.provider for identity in request_user.identity_set.all()] pms = pms.filter(instance_source__provider__in=providers) return pms
def get_public_and_private_apps(provider): """ INPUT: Provider provider OUTPUT: 2-tuple ( new_public_apps [], private_apps(key) + super-set-membership(value) {}) """ account_driver = get_account_driver(provider) all_projects_map = tenant_id_to_name_map(account_driver) cloud_machines = account_driver.list_all_images() db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) new_public_apps = [] private_apps = {} # ASSERT: All non-end-dated machines in the DB can be found in the cloud # if you do not believe this is the case, you should call 'prune_machines_for' for cloud_machine in cloud_machines: #Filter out: ChromoSnapShot, eri-, eki-, ... (Or dont..) if any( cloud_machine.name.startswith(prefix) for prefix in ['eri-', 'eki-', 'ChromoSnapShot']): #celery_logger.debug("Skipping cloud machine %s" % cloud_machine) continue app_name, version_name = ProviderMachine._split_cloud_name( cloud_machine.name) db_machine = get_or_create_provider_machine(cloud_machine.id, app_name, provider.uuid, version_name=version_name) db_version = db_machine.application_version db_application = db_version.application if cloud_machine.get('visibility') == 'public': if db_application.private and db_application not in new_public_apps: new_public_apps.append(db_application) #Distinct list.. #Else the db app is public and no changes are necessary. else: # cloud machine is private membership = get_shared_identities(account_driver, cloud_machine, all_projects_map) all_members = private_apps.get(db_application, []) all_members.extend(membership) #Distinct list.. private_apps[db_application] = all_members return new_public_apps, private_apps
def prune_machines_for(provider_id, print_logs=False, dry_run=False, forced_removal=False): """ Look at the list of machines (as seen by the AccountProvider) if a machine cannot be found in the list, remove it. NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any unseen image to be prematurely end-dated and removed from the API/UI. """ provider = Provider.objects.get(id=provider_id) if print_logs: import logging import sys consolehandler = logging.StreamHandler(sys.stdout) consolehandler.setLevel(logging.DEBUG) celery_logger.addHandler(consolehandler) if provider.is_active(): account_driver = get_account_driver(provider) db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) all_projects_map = tenant_id_to_name_map(account_driver) cloud_machines = account_driver.list_all_images() else: db_machines = ProviderMachine.objects.filter( source_in_range(), instance_source__provider=provider) cloud_machines = [] # Don't do anything if cloud machines == [None,[]] if not cloud_machines and not forced_removal: return # Loop1 - End-date All machines in the DB that can NOT be found in the cloud. cloud_machine_ids = [mach.id for mach in cloud_machines] for machine in db_machines: cloud_match = [ mach for mach in cloud_machine_ids if mach == machine.identifier ] if not cloud_match: remove_machine(machine, dry_run=dry_run) if print_logs: celery_logger.removeHandler(consolehandler)
def get_admin_machines(user): """ TODO: This 'just works' and is probably very slow... Look for a better way? """ provider_id_list = user.identity_set.values_list('provider', flat=True) account_providers_list = AccountProvider.objects.filter( provider__id__in=provider_id_list) admin_users = [ap.identity.created_by for ap in account_providers_list] machine_ids = [] for user in admin_users: machine_ids.extend( user.source_set.filter( providermachine__isnull=False).values_list( 'providermachine', flat=True)) admin_list = ProviderMachine.objects.filter( only_current_source(), id__in=machine_ids) return admin_list
def get_public_and_private_apps(provider): """ INPUT: Provider provider OUTPUT: 2-tuple ( new_public_apps [], private_apps(key) + super-set-membership(value) {}) """ account_driver = get_account_driver(provider) all_projects_map = tenant_id_to_name_map(account_driver) cloud_machines = account_driver.list_all_images() db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) new_public_apps = [] private_apps = {} # ASSERT: All non-end-dated machines in the DB can be found in the cloud # if you do not believe this is the case, you should call 'prune_machines_for' for cloud_machine in cloud_machines: #Filter out: ChromoSnapShot, eri-, eki-, ... (Or dont..) if any( cloud_machine.name.startswith(prefix) for prefix in ['eri-', 'eki-', 'ChromoSnapShot']): #celery_logger.debug("Skipping cloud machine %s" % cloud_machine) continue db_machine = get_or_create_provider_machine( cloud_machine.id, cloud_machine.name, provider.uuid) db_version = db_machine.application_version db_application = db_version.application if cloud_machine.is_public: if db_application.private and db_application not in new_public_apps: new_public_apps.append(db_application) #Distinct list.. #Else the db app is public and no changes are necessary. else: # cloud machine is private membership = get_shared_identities(account_driver, cloud_machine, all_projects_map) all_members = private_apps.get(db_application, []) all_members.extend(membership) #Distinct list.. private_apps[db_application] = all_members return new_public_apps, private_apps
def visible_applications(user): apps = [] if not user: return apps # Look only for 'Active' private applications for app in Application.objects.filter(only_current(), private=True): # Retrieve the machines associated with this app providermachine_set = app.all_machines machine_set = providermachine_set.filter(only_current_source()) # Skip app if all their machines are on inactive providers. if all(not pm.instance_source.provider.is_active() for pm in machine_set): continue # Add the application if 'user' is a member of the application or PM if app.members.filter(user=user): _add_app(apps, app) for pm in machine_set: if pm.members.filter(user=user): _add_app(apps, app) break return apps
def _current_machines(self, request_user=None): """ Return a list of current provider machines. NOTE: Defined as: * Provider is still listed as active * Provider has not exceeded its end_date * The ProviderMachine has not exceeded its end_date """ providermachine_set = self.all_machines pms = providermachine_set.filter( query.only_current_source(), instance_source__provider__active=True) if request_user: if isinstance(request_user, AnonymousUser): providers = Provider.objects.filter(public=True) else: providers = [identity.provider for identity in request_user.identity_set.all()] pms = pms.filter(instance_source__provider__in=providers) return pms
def visible_applications(user): apps = [] if not user: return apps from core.models import Provider, ProviderMachineMembership active_providers = Provider.get_active() now_time = timezone.now() #Look only for 'Active' private applications for app in Application.objects.filter(only_current(), private=True): #Retrieve the machines associated with this app machine_set = app.providermachine_set.filter(only_current_source()) #Skip app if all their machines are on inactive providers. if all(not pm.instance_source.provider.is_active() for pm in machine_set): continue #Add the application if 'user' is a member of the application or PM if app.members.filter(user=user): _add_app(apps, app) for pm in machine_set: if pm.members.filter(user=user): _add_app(apps, app) break return apps
def active_machines(self): """ Show machines that are from an active provider and non-end-dated. """ return self.machines.filter(only_current_source())
def first_machine(self): #Out of all non-end dated machines in this application first = self.providermachine_set.filter( only_current_source()).order_by( 'instance_source__start_date').first() return first
def first_machine(self): # Out of all non-end dated machines in this application providermachine_set = self.all_machines first = providermachine_set.filter(only_current_source()).order_by("instance_source__start_date").first() return first
def prune_machines_for( provider_id, print_logs=False, dry_run=False, forced_removal=False, validate=True ): """ Look at the list of machines (as seen by the AccountProvider) if a machine cannot be found in the list, remove it. NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any image unseen by the admin to be prematurely end-dated and removed from the API/UI. """ provider = Provider.objects.get(id=provider_id) now = timezone.now() if print_logs: console_handler = _init_stdout_logging() celery_logger.info( "Starting prune_machines for Provider %s @ %s" % (provider, now) ) if provider.is_active(): account_driver = get_account_driver(provider) db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider ) cloud_machines = account_driver.list_all_images() else: db_machines = ProviderMachine.objects.filter( source_in_range(), # like 'only_current..' w/o active_provider instance_source__provider=provider ) cloud_machines = [] machine_validator = MachineValidationPluginManager.get_validator( account_driver ) cloud_machines = [ cloud_machine for cloud_machine in cloud_machines if not validate or machine_validator.machine_is_valid(cloud_machine) ] # Don't do anything if cloud machines == [None,[]] if not cloud_machines and not forced_removal: return # Loop 1 - End-date All machines in the DB that # can NOT be found in the cloud. mach_count = _end_date_missing_database_machines( db_machines, cloud_machines, now=now, dry_run=dry_run ) # Loop 2 and 3 - Capture all (still-active) versions without machines, # and all applications without versions. # These are 'outliers' and mainly here for safety-check purposes. ver_count = _remove_versions_without_machines(now=now) app_count = _remove_applications_without_versions(now=now) # Loop 4 - All 'Application' DB objects require # >=1 Version with >=1 ProviderMachine (ACTIVE!) # Apps that don't meet this criteria should be end-dated. app_count += _update_improperly_enddated_applications(now) # Clear out application, provider machine, and version memberships # if the result is >128. # Additionally, remove all users who are not in the machine request (if one exists). _clean_memberships(db_machines, account_driver) celery_logger.info( "prune_machines completed for Provider %s : " "%s Applications, %s versions and %s machines pruned." % (provider, app_count, ver_count, mach_count) ) if print_logs: _exit_stdout_logging(console_handler)
def get_queryset(self): return super(ActiveVolumesManager, self)\ .get_queryset().filter(only_current_source())