def get_queryset(self):
request_user = self.request.user
# Showing non-end dated, public ProviderMachines
public_set = ProviderMachine.objects.filter(
only_current_source(),
only_public_providers(),
application_version__application__private=False)
if not isinstance(request_user, AnonymousUser):
# Showing non-end dated, public ProviderMachines
shared_set = ProviderMachine.objects.filter(
only_current_source(),
members__id__in=request_user.memberships.values_list(
'group__id', flat=True))
# NOTE: Showing 'my pms' EVEN if they are end-dated.
my_set = ProviderMachine.objects.filter(
user_provider_machine_set(request_user))
if request_user.is_staff:
admin_set = get_admin_machines(request_user)
else:
admin_set = ProviderMachine.objects.none()
else:
shared_set = my_set = admin_set = ProviderMachine.objects.none()
# Order them by date, make sure no dupes.
queryset = (
public_set | shared_set | my_set
| admin_set).distinct().order_by('-instance_source__start_date')
if not isinstance(request_user, AnonymousUser):
queryset = queryset.filter(
in_provider_list(request_user.current_providers))
return queryset
def get_queryset(self):
request_user = self.request.user
# Showing non-end dated, public ProviderMachines
public_set = ProviderMachine.objects.filter(
only_current_source(),
only_public_providers(),
application_version__application__private=False)
if not isinstance(request_user, AnonymousUser):
# Showing non-end dated, public ProviderMachines
shared_set = ProviderMachine.objects.filter(
only_current_source(),
members__id__in=request_user.memberships.values_list('group__id', flat=True))
# NOTE: Showing 'my pms' EVEN if they are end-dated.
my_set = ProviderMachine.objects.filter(user_provider_machine_set(request_user))
if request_user.is_staff:
admin_set = get_admin_machines(request_user)
else:
admin_set = ProviderMachine.objects.none()
else:
shared_set = my_set = admin_set = ProviderMachine.objects.none()
# Order them by date, make sure no dupes.
queryset = (
public_set | shared_set | my_set | admin_set).distinct().order_by(
'-instance_source__start_date')
if not isinstance(request_user, AnonymousUser):
launchable_providers = request_user.current_providers.filter(active=True)
queryset = queryset.filter(in_provider_list(launchable_providers))
return queryset
def get_queryset(self):
request_user = self.request.user
# Showing non-end dated, public ProviderMachines
public_set = ProviderMachine.objects.filter(
only_current_source(),
application_version__application__private=False)
if not isinstance(request_user, AnonymousUser):
# Showing non-end dated, public ProviderMachines
shared_set = ProviderMachine.objects.filter(
only_current_source(),
members__in=request_user.group_set.values('id'))
# NOTE: Showing 'my pms' EVEN if they are end-dated.
my_set = ProviderMachine.objects.filter(
Q(
application_version__application__created_by=request_user
) | Q(
instance_source__created_by=request_user))
if request_user.is_staff:
admin_set = get_admin_machines(request_user)
else:
admin_set = ProviderMachine.objects.none()
else:
shared_set = my_set = admin_set = ProviderMachine.objects.none()
# Order them by date, make sure no dupes.
queryset = (
public_set | shared_set | my_set | admin_set).distinct().order_by(
'-instance_source__start_date')
return queryset
def last_machine(self):
providermachine_set = self.all_machines
# Out of all non-end dated machines in this application
last = providermachine_set.filter(
query.only_current_source()
).order_by('instance_source__start_date').last()
return last
def get_queryset(self):
"""
Filter out tags for deleted volumes
"""
user = self.request.user
return Volume.objects.filter(only_current_source(),
project__owner__user=user)
def search(cls, query, identity=None):
if identity:
base_apps = Application.objects.filter(
# Privately owned OR public machines
Q(private=True,
providermachine__instance_source__created_by_identity=identity)
| Q(private=False,
providermachine__instance_source__provider=identity.provider))
else:
active_providers = Provider.get_active()
base_apps = Application.objects.filter(
# Public machines
private=False,
#Providermachine's provider is active
providermachine__instance_source__provider__in=active_providers)
# AND query matches on:
query_match = base_apps.filter(
# app tag name
Q(tags__name__icontains=query)
# OR app tag desc
| Q(tags__description__icontains=query)
# OR app name
| Q(name__icontains=query)
# OR app desc
| Q(description__icontains=query),
only_current_source())
return query_match.distinct()
def last_machine(self):
providermachine_set = self.all_machines
# Out of all non-end dated machines in this application
last = providermachine_set.filter(
only_current_source()
).order_by('instance_source__start_date').last()
return last
def handle_machine(args, provider):
if not args.machine_alias:
print "Error: A machine-alias is required."
sys.exit(1)
if args.machine_alias == 'all':
return ProviderMachine.objects.filter(
only_current_source(),
instance_source__provider_id=provider.id,
).distinct()
elif args.machine_alias == 'most_used':
return sort_most_used_machines(provider, limit=20)
elif ',' not in args.machine_alias:
return [
ProviderMachine.objects.get(
instance_source__identifier=args.machine_alias,
instance_source__provider_id=provider.id)
]
else:
machines = args.machine_alias.split(",")
print "Batch launch of images detected: %s" % machines
return [
ProviderMachine.objects.get(
instance_source__identifier=machine_alias,
instance_source__provider_id=provider.id)
for machine_alias in machines
]
def search(cls, query, identity=None):
if identity:
base_apps = Application.objects.filter(
# Privately owned OR public machines
Q(private=True,
providermachine__instance_source__created_by_identity=identity
)
| Q(private=False,
providermachine__instance_source__provider=identity.
provider))
else:
active_providers = Provider.get_active()
base_apps = Application.objects.filter(
# Public machines
private=False,
#Providermachine's provider is active
providermachine__instance_source__provider__in=active_providers
)
# AND query matches on:
query_match = base_apps.filter(
# app tag name
Q(tags__name__icontains=query)
# OR app tag desc
| Q(tags__description__icontains=query)
# OR app name
| Q(name__icontains=query)
# OR app desc
| Q(description__icontains=query),
only_current_source())
return query_match.distinct()
def handle_machine(args, provider):
if not args.machine_alias:
print "Error: A machine-alias is required."
sys.exit(1)
if args.machine_alias == "all":
return ProviderMachine.objects.filter(
only_current_source(), instance_source__provider_id=provider.id
).distinct()
elif args.machine_alias == "most_used":
return sort_most_used_machines(provider, limit=20)
elif "," not in args.machine_alias:
return [
ProviderMachine.objects.get(
instance_source__identifier=args.machine_alias, instance_source__provider_id=provider.id
)
]
else:
machines = args.machine_alias.split(",")
print "Batch launch of images detected: %s" % machines
return [
ProviderMachine.objects.get(
instance_source__identifier=machine_alias, instance_source__provider_id=provider.id
)
for machine_alias in machines
]
def add_membership(image_version, group):
"""
This function will add *all* users in the group
to *all* providers/machines using this image_version
O(N^2)
"""
for provider_machine in image_version.machines.filter(only_current_source()):
add_machine_membership(provider_machine, group)
def add_membership(image_version, group):
"""
This function will add *all* users in the group
to *all* providers/machines using this image_version
O(N^2)
"""
for provider_machine in image_version.machines.filter(only_current_source()):
add_machine_membership(provider_machine, group)
def get_queryset(self):
"""
Filter projects by current user
"""
user = self.request.user
return Volume.objects.filter(
only_current_source(),
instance_source__created_by=user)
def get_queryset(self):
"""
Filter out tags for deleted volumes
"""
user = self.request.user
return Volume.objects.filter(
only_current_source(), project__owner__user=user
)
def prune_machines_for(
provider_id, print_logs=False, dry_run=False, forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
import logging
import sys
consolehandler = logging.StreamHandler(sys.stdout)
consolehandler.setLevel(logging.DEBUG)
celery_logger.addHandler(consolehandler)
celery_logger.info("Starting prune_machines for Provider %s @ %s"
% (provider, now))
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(
db_machines, cloud_machines, now=now, dry_run=dry_run)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
celery_logger.info(
"prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned."
% (provider, app_count, ver_count, mach_count))
if print_logs:
celery_logger.removeHandler(consolehandler)
def get_queryset(self):
"""
Filter projects by current user
"""
user = self.request.user
identity_ids = user.current_identities.values_list('id', flat=True)
return Volume.objects.filter(
only_current_source(),
instance_source__created_by_identity__in=identity_ids)
def get_queryset(self):
"""
Filter projects by current user
"""
user = self.request.user
identity_ids = user.current_identities.values_list('id',flat=True)
return Volume.objects.filter(
only_current_source(),
instance_source__created_by_identity__in=identity_ids)
def get_queryset(self):
"""
Filter projects by current user
"""
user = self.request.user
now = timezone.now()
return Volume.objects.filter(
only_current_source(now),
instance_source__created_by=user)
def get_user_volumes(self, project):
return [
VolumeSerializer(item,
context={
'request': self.context.get('request')
}).data
for item in project.volumes.filter(
only_current_source(), instance_source__provider__active=True)
]
def prune_machines_for(provider_id,
print_logs=False,
dry_run=False,
forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
console_handler = _init_stdout_logging()
celery_logger.info("Starting prune_machines for Provider %s @ %s" %
(provider, now))
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(db_machines,
cloud_machines,
now=now,
dry_run=dry_run)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
celery_logger.info("prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned." %
(provider, app_count, ver_count, mach_count))
if print_logs:
_exit_stdout_logging(console_handler)
def get_user_volumes(self, atmo_user):
return [
VolumeSerializer(
item,
context={
'request': self.context.get('request')}).data for item in atmo_user.volume_set().filter(
*
only_current_source(),
instance_source__provider__active=True,
projects=None)]
def get_user_volumes(self, atmo_user):
return [
VolumeSerializer(item,
context={
'request': self.context.get('request')
}).data for item in atmo_user.volume_set().filter(
*only_current_source(),
instance_source__provider__active=True,
projects=None)
]
def get_user_volumes(self, project):
return [
VolumeSerializer(
item, context={
'request': self.context.get('request')
}
).data for item in project.volumes.filter(
only_current_source(), instance_source__provider__active=True
)
]
def monitor_volumes_for(provider_id, print_logs=False):
"""
Run the set of tasks related to monitoring sizes for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
"""
from service.driver import get_account_driver
from core.models import Identity
if print_logs:
console_handler = _init_stdout_logging()
provider = Provider.objects.get(id=provider_id)
account_driver = get_account_driver(provider)
# Non-End dated volumes on this provider
db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider)
all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30)
seen_volumes = []
for cloud_volume in all_volumes:
try:
core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid)
seen_volumes.append(core_volume)
except ObjectDoesNotExist:
tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id']
tenant = account_driver.get_project_by_id(tenant_id)
tenant_name = tenant.name if tenant else tenant_id
try:
if not tenant:
celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume)
raise ObjectDoesNotExist()
identity = Identity.objects.filter(
contains_credential('ex_project_name', tenant_name), provider=provider
).first()
if not identity:
raise ObjectDoesNotExist()
core_volume = convert_esh_volume(
cloud_volume,
provider.uuid, identity.uuid,
identity.created_by)
except ObjectDoesNotExist:
celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name))
pass
now_time = timezone.now()
needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes]
for volume in needs_end_date:
celery_logger.debug("End dating inactive volume: %s" % volume)
volume.end_date = now_time
volume.save()
if print_logs:
_exit_stdout_logging(console_handler)
for vol in seen_volumes:
vol.esh = None
return [vol.instance_source.identifier for vol in seen_volumes]
def monitor_volumes_for(provider_id, print_logs=False):
"""
Run the set of tasks related to monitoring sizes for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
"""
from service.driver import get_account_driver
from core.models import Identity
if print_logs:
console_handler = _init_stdout_logging()
provider = Provider.objects.get(id=provider_id)
account_driver = get_account_driver(provider)
# Non-End dated volumes on this provider
db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider)
all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30)
seen_volumes = []
for cloud_volume in all_volumes:
try:
core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid)
seen_volumes.append(core_volume)
except ObjectDoesNotExist:
tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id']
tenant = account_driver.get_project_by_id(tenant_id)
tenant_name = tenant.name if tenant else tenant_id
try:
if not tenant:
celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume)
raise ObjectDoesNotExist()
identity = Identity.objects.filter(
contains_credential('ex_project_name', tenant_name), provider=provider
).first()
if not identity:
raise ObjectDoesNotExist()
core_volume = convert_esh_volume(
cloud_volume,
provider.uuid, identity.uuid,
identity.created_by)
except ObjectDoesNotExist:
celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name))
pass
now_time = timezone.now()
needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes]
for volume in needs_end_date:
celery_logger.debug("End dating inactive volume: %s" % volume)
volume.end_date = now_time
volume.save()
if print_logs:
_exit_stdout_logging(console_handler)
for vol in seen_volumes:
vol.esh = None
return [vol.instance_source.identifier for vol in seen_volumes]
def remove_membership(image_version, group, accounts=None):
"""
This function will remove *all* users in the group
to *all* providers/machines using this image_version
"""
for provider_machine in image_version.machines.filter(
only_current_source()):
prov = provider_machine.instance_source.provider
if not accounts:
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" %
prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" %
prov)
img = accounts.get_image(provider_machine.identifier)
approved_projects = accounts.get_image_members(img.id)
for identity_membership in group.identitymembership_set.order_by(
'identity__created_by__username'):
if identity_membership.identity.provider != prov:
continue
# Get project name from the identity's credential-list
project_name = identity_membership.identity.get_credential(
'ex_project_name')
project = accounts.get_project(project_name)
if project and project not in approved_projects:
continue
# Perform a *DATABASE* remove first.
application = provider_machine.application
application_version = provider_machine.application_version
models.ApplicationMembership.objects.filter(
group=group, application=application).delete()
logger.info("Removed ApplicationMembership: %s-%s" %
(application, group))
models.ApplicationVersionMembership.objects.filter(
group=group, image_version=application_version).delete()
logger.info("Removed ApplicationVersionMembership: %s-%s" %
(application_version, group))
models.ProviderMachineMembership.objects.filter(
group=group, provider_machine=provider_machine).delete()
logger.info("Removed ProviderMachineMembership: %s-%s" %
(provider_machine, group))
# Perform a *CLOUD* remove last.
try:
accounts.image_manager.unshare_image(img, project_name)
except Exception as exc:
logger.exception(
"Exception occurred while removing user from cloud: %s",
exc)
logger.info("Removed Cloud Access: %s-%s" % (img, project_name))
return
def remove_membership(image_version, group, accounts=None):
"""
This function will remove *all* users in the group
to *all* providers/machines using this image_version
"""
for provider_machine in image_version.machines.filter(only_current_source()):
prov = provider_machine.instance_source.provider
if not accounts:
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" % prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" % prov)
img = accounts.get_image(provider_machine.identifier)
approved_projects = accounts.get_image_members(img.id)
for identity_membership in group.identitymembership_set.order_by('identity__created_by__username'):
if identity_membership.identity.provider != prov:
continue
# Get project name from the identity's credential-list
project_name = identity_membership.identity.get_credential(
'ex_project_name')
project = accounts.get_project(project_name)
if project and project not in approved_projects:
continue
# Perform a *DATABASE* remove first.
application = provider_machine.application
application_version = provider_machine.application_version
models.ApplicationMembership.objects.filter(
group=group,
application=application).delete()
logger.info("Removed ApplicationMembership: %s-%s"
% (application, group))
models.ApplicationVersionMembership.objects.filter(
group=group,
image_version=application_version).delete()
logger.info("Removed ApplicationVersionMembership: %s-%s"
% (application_version, group))
models.ProviderMachineMembership.objects.filter(
group=group,
provider_machine=provider_machine).delete()
logger.info("Removed ProviderMachineMembership: %s-%s"
% (provider_machine, group))
# Perform a *CLOUD* remove last.
try:
accounts.image_manager.unshare_image(img, project_name)
except Exception as exc:
logger.exception("Exception occurred while removing user from cloud: %s", exc)
logger.info("Removed Cloud Access: %s-%s"
% (img, project_name))
return
def add_membership(image_version, group):
"""
This function will add *all* users in the group
to *all* providers/machines using this image_version
O(N^2)
"""
for provider_machine in image_version.machines.filter(
only_current_source()):
prov = provider_machine.instance_source.provider
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" %
prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" %
prov)
img = accounts.get_image(provider_machine.identifier)
projects = get_current_projects_for_image(accounts, img.id)
for identity_membership in group.identitymembership_set.all():
if identity_membership.identity.provider != prov:
continue
# Get project name from the identity's credential-list
project_name = identity_membership.identity.get_credential(
'ex_project_name')
project = accounts.get_project(project_name)
if project and project in projects:
continue
# Share with the *database* first!
obj, created = models.ApplicationMembership.objects.get_or_create(
group=group, application=provider_machine.application)
if created:
print "Created new ApplicationMembership: %s" \
% (obj,)
obj, created = models.ApplicationVersionMembership.objects.get_or_create(
group=group,
image_version=provider_machine.application_version)
if created:
print "Created new ApplicationVersionMembership: %s" \
% (obj,)
obj, created = models.ProviderMachineMembership.objects.get_or_create(
group=group, provider_machine=provider_machine)
if created:
print "Created new ProviderMachineMembership: %s" \
% (obj,)
# Share with the *cloud* last!
accounts.image_manager.share_image(img, project_name)
accounts.accept_shared_image(img, project_name)
logger.info("Added Cloud Access: %s-%s" % (img, project_name))
continue
def get_queryset(self):
"""
Filter projects by current user
"""
user = self.request.user
qs = Volume.shared_with_user(user)
if 'archived' not in self.request.query_params:
qs = qs.filter(only_current_source())
qs = qs\
.select_related('instance_source')\
.select_related("instance_source__created_by")\
.select_related('instance_source__created_by_identity')\
.select_related('project')
return qs
def get_queryset(self):
"""
Filter projects by current user
"""
user = self.request.user
qs = Volume.shared_with_user(user)
if 'archived' not in self.request.query_params:
qs = qs.filter(only_current_source())
qs = qs\
.select_related('instance_source')\
.select_related("instance_source__created_by")\
.select_related('instance_source__created_by_identity')\
.select_related('project')
return qs
def add_membership(image_version, group):
"""
This function will add *all* users in the group
to *all* providers/machines using this image_version
O(N^2)
"""
for provider_machine in image_version.machines.filter(only_current_source()):
prov = provider_machine.instance_source.provider
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" % prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" % prov)
img = accounts.get_image(provider_machine.identifier)
projects = get_current_projects_for_image(accounts, img.id)
for identity_membership in group.identitymembership_set.all():
if identity_membership.identity.provider != prov:
continue
# Get project name from the identity's credential-list
project_name = identity_membership.identity.get_credential('ex_project_name')
project = accounts.get_project(project_name)
if project and project in projects:
continue
# Share with the *database* first!
obj, created = models.ApplicationMembership.objects.get_or_create(
group=group,
application=provider_machine.application)
if created:
print "Created new ApplicationMembership: %s" \
% (obj,)
obj, created = models.ApplicationVersionMembership.objects.get_or_create(
group=group,
image_version=provider_machine.application_version)
if created:
print "Created new ApplicationVersionMembership: %s" \
% (obj,)
obj, created = models.ProviderMachineMembership.objects.get_or_create(
group=group,
provider_machine=provider_machine)
if created:
print "Created new ProviderMachineMembership: %s" \
% (obj,)
# Share with the *cloud* last!
accounts.image_manager.share_image(img, project_name)
accounts.accept_shared_image(img, project_name)
logger.info("Added Cloud Access: %s-%s"
% (img, project_name))
continue
def search(cls, identity, query):
return ProviderMachine.objects.filter(
# Privately owned OR public machines
Q(application__private=True, instance_source__created_by_identity=identity)
| Q(application__private=False, instance_source__provider=identity.provider),
# AND query matches on:
# app tag name OR
# app tag desc OR
# app name OR
# app desc
Q(application__tags__name__icontains=query)
| Q(application__tags__description__icontains=query)
| Q(application__name__icontains=query)
| Q(application__description__icontains=query),
only_current_source())
def get_admin_machines(user):
"""
TODO: This 'just works' and is probably very slow... Look for a better way?
"""
provider_id_list = user.identity_set.values_list('provider', flat=True)
account_providers_list = AccountProvider.objects.filter(
provider__id__in=provider_id_list)
admin_users = [ap.identity.created_by for ap in account_providers_list]
machine_ids = []
for user in admin_users:
machine_ids.extend(
user.source_set.filter(providermachine__isnull=False).values_list(
'providermachine', flat=True))
admin_list = ProviderMachine.objects.filter(only_current_source(),
id__in=machine_ids)
return admin_list
def _current_machines(self, request_user=None):
"""
Return a list of current provider machines.
NOTE: Defined as:
* Provider is still listed as active
* Provider has not exceeded its end_date
* The ProviderMachine has not exceeded its end_date
"""
providermachine_set = self.all_machines
pms = providermachine_set.filter(only_current_source(), instance_source__provider__active=True)
if request_user:
if isinstance(request_user, AnonymousUser):
providers = Provider.objects.filter(public=True)
else:
providers = [identity.provider for identity in request_user.identity_set.all()]
pms = pms.filter(instance_source__provider__in=providers)
return pms
def search(cls, identity, query):
return ProviderMachine.objects.filter(
# Privately owned OR public machines
Q(application__private=True,
instance_source__created_by_identity=identity)
| Q(application__private=False,
instance_source__provider=identity.provider),
# AND query matches on:
# app tag name OR
# app tag desc OR
# app name OR
# app desc
Q(application__tags__name__icontains=query)
| Q(application__tags__description__icontains=query)
| Q(application__name__icontains=query)
| Q(application__description__icontains=query),
only_current_source())
def get_public_and_private_apps(provider):
"""
INPUT: Provider provider
OUTPUT: 2-tuple (
new_public_apps [],
private_apps(key) + super-set-membership(value) {})
"""
account_driver = get_account_driver(provider)
all_projects_map = tenant_id_to_name_map(account_driver)
cloud_machines = account_driver.list_all_images()
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
new_public_apps = []
private_apps = {}
# ASSERT: All non-end-dated machines in the DB can be found in the cloud
# if you do not believe this is the case, you should call 'prune_machines_for'
for cloud_machine in cloud_machines:
#Filter out: ChromoSnapShot, eri-, eki-, ... (Or dont..)
if any(
cloud_machine.name.startswith(prefix)
for prefix in ['eri-', 'eki-', 'ChromoSnapShot']):
#celery_logger.debug("Skipping cloud machine %s" % cloud_machine)
continue
app_name, version_name = ProviderMachine._split_cloud_name(
cloud_machine.name)
db_machine = get_or_create_provider_machine(cloud_machine.id,
app_name,
provider.uuid,
version_name=version_name)
db_version = db_machine.application_version
db_application = db_version.application
if cloud_machine.get('visibility') == 'public':
if db_application.private and db_application not in new_public_apps:
new_public_apps.append(db_application) #Distinct list..
#Else the db app is public and no changes are necessary.
else:
# cloud machine is private
membership = get_shared_identities(account_driver, cloud_machine,
all_projects_map)
all_members = private_apps.get(db_application, [])
all_members.extend(membership)
#Distinct list..
private_apps[db_application] = all_members
return new_public_apps, private_apps
def prune_machines_for(provider_id,
print_logs=False,
dry_run=False,
forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any unseen image to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
if print_logs:
import logging
import sys
consolehandler = logging.StreamHandler(sys.stdout)
consolehandler.setLevel(logging.DEBUG)
celery_logger.addHandler(consolehandler)
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
all_projects_map = tenant_id_to_name_map(account_driver)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop1 - End-date All machines in the DB that can NOT be found in the cloud.
cloud_machine_ids = [mach.id for mach in cloud_machines]
for machine in db_machines:
cloud_match = [
mach for mach in cloud_machine_ids if mach == machine.identifier
]
if not cloud_match:
remove_machine(machine, dry_run=dry_run)
if print_logs:
celery_logger.removeHandler(consolehandler)
def prune_machines_for(provider_id,
print_logs=False,
dry_run=False,
forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any unseen image to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
if print_logs:
import logging
import sys
consolehandler = logging.StreamHandler(sys.stdout)
consolehandler.setLevel(logging.DEBUG)
celery_logger.addHandler(consolehandler)
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
all_projects_map = tenant_id_to_name_map(account_driver)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop1 - End-date All machines in the DB that can NOT be found in the cloud.
cloud_machine_ids = [mach.id for mach in cloud_machines]
for machine in db_machines:
cloud_match = [
mach for mach in cloud_machine_ids if mach == machine.identifier
]
if not cloud_match:
remove_machine(machine, dry_run=dry_run)
if print_logs:
celery_logger.removeHandler(consolehandler)
def get_admin_machines(user):
"""
TODO: This 'just works' and is probably very slow... Look for a better way?
"""
provider_id_list = user.identity_set.values_list('provider', flat=True)
account_providers_list = AccountProvider.objects.filter(
provider__id__in=provider_id_list)
admin_users = [ap.identity.created_by for ap in account_providers_list]
machine_ids = []
for user in admin_users:
machine_ids.extend(
user.source_set.filter(
providermachine__isnull=False).values_list(
'providermachine',
flat=True))
admin_list = ProviderMachine.objects.filter(
only_current_source(),
id__in=machine_ids)
return admin_list
def get_public_and_private_apps(provider):
"""
INPUT: Provider provider
OUTPUT: 2-tuple (
new_public_apps [],
private_apps(key) + super-set-membership(value) {})
"""
account_driver = get_account_driver(provider)
all_projects_map = tenant_id_to_name_map(account_driver)
cloud_machines = account_driver.list_all_images()
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
new_public_apps = []
private_apps = {}
# ASSERT: All non-end-dated machines in the DB can be found in the cloud
# if you do not believe this is the case, you should call 'prune_machines_for'
for cloud_machine in cloud_machines:
#Filter out: ChromoSnapShot, eri-, eki-, ... (Or dont..)
if any(
cloud_machine.name.startswith(prefix)
for prefix in ['eri-', 'eki-', 'ChromoSnapShot']):
#celery_logger.debug("Skipping cloud machine %s" % cloud_machine)
continue
db_machine = get_or_create_provider_machine(
cloud_machine.id, cloud_machine.name, provider.uuid)
db_version = db_machine.application_version
db_application = db_version.application
if cloud_machine.is_public:
if db_application.private and db_application not in new_public_apps:
new_public_apps.append(db_application) #Distinct list..
#Else the db app is public and no changes are necessary.
else:
# cloud machine is private
membership = get_shared_identities(account_driver, cloud_machine,
all_projects_map)
all_members = private_apps.get(db_application, [])
all_members.extend(membership)
#Distinct list..
private_apps[db_application] = all_members
return new_public_apps, private_apps
def visible_applications(user):
apps = []
if not user:
return apps
# Look only for 'Active' private applications
for app in Application.objects.filter(only_current(), private=True):
# Retrieve the machines associated with this app
providermachine_set = app.all_machines
machine_set = providermachine_set.filter(only_current_source())
# Skip app if all their machines are on inactive providers.
if all(not pm.instance_source.provider.is_active() for pm in machine_set):
continue
# Add the application if 'user' is a member of the application or PM
if app.members.filter(user=user):
_add_app(apps, app)
for pm in machine_set:
if pm.members.filter(user=user):
_add_app(apps, app)
break
return apps
def _current_machines(self, request_user=None):
"""
Return a list of current provider machines.
NOTE: Defined as:
* Provider is still listed as active
* Provider has not exceeded its end_date
* The ProviderMachine has not exceeded its end_date
"""
providermachine_set = self.all_machines
pms = providermachine_set.filter(
query.only_current_source(),
instance_source__provider__active=True)
if request_user:
if isinstance(request_user, AnonymousUser):
providers = Provider.objects.filter(public=True)
else:
providers = [identity.provider for identity in
request_user.identity_set.all()]
pms = pms.filter(instance_source__provider__in=providers)
return pms
def visible_applications(user):
apps = []
if not user:
return apps
# Look only for 'Active' private applications
for app in Application.objects.filter(only_current(), private=True):
# Retrieve the machines associated with this app
providermachine_set = app.all_machines
machine_set = providermachine_set.filter(only_current_source())
# Skip app if all their machines are on inactive providers.
if all(not pm.instance_source.provider.is_active()
for pm in machine_set):
continue
# Add the application if 'user' is a member of the application or PM
if app.members.filter(user=user):
_add_app(apps, app)
for pm in machine_set:
if pm.members.filter(user=user):
_add_app(apps, app)
break
return apps
def visible_applications(user):
apps = []
if not user:
return apps
from core.models import Provider, ProviderMachineMembership
active_providers = Provider.get_active()
now_time = timezone.now()
#Look only for 'Active' private applications
for app in Application.objects.filter(only_current(), private=True):
#Retrieve the machines associated with this app
machine_set = app.providermachine_set.filter(only_current_source())
#Skip app if all their machines are on inactive providers.
if all(not pm.instance_source.provider.is_active() for pm in machine_set):
continue
#Add the application if 'user' is a member of the application or PM
if app.members.filter(user=user):
_add_app(apps, app)
for pm in machine_set:
if pm.members.filter(user=user):
_add_app(apps, app)
break
return apps
def visible_applications(user):
apps = []
if not user:
return apps
from core.models import Provider, ProviderMachineMembership
active_providers = Provider.get_active()
now_time = timezone.now()
#Look only for 'Active' private applications
for app in Application.objects.filter(only_current(), private=True):
#Retrieve the machines associated with this app
machine_set = app.providermachine_set.filter(only_current_source())
#Skip app if all their machines are on inactive providers.
if all(not pm.instance_source.provider.is_active()
for pm in machine_set):
continue
#Add the application if 'user' is a member of the application or PM
if app.members.filter(user=user):
_add_app(apps, app)
for pm in machine_set:
if pm.members.filter(user=user):
_add_app(apps, app)
break
return apps
def active_machines(self):
"""
Show machines that are from an active provider and non-end-dated.
"""
return self.machines.filter(only_current_source())
def active_machines(self):
"""
Show machines that are from an active provider and non-end-dated.
"""
return self.machines.filter(only_current_source())
def first_machine(self):
#Out of all non-end dated machines in this application
first = self.providermachine_set.filter(
only_current_source()).order_by(
'instance_source__start_date').first()
return first
def first_machine(self):
# Out of all non-end dated machines in this application
providermachine_set = self.all_machines
first = providermachine_set.filter(only_current_source()).order_by("instance_source__start_date").first()
return first
def prune_machines_for(
provider_id,
print_logs=False,
dry_run=False,
forced_removal=False,
validate=True
):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
console_handler = _init_stdout_logging()
celery_logger.info(
"Starting prune_machines for Provider %s @ %s" % (provider, now)
)
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider
)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider
)
cloud_machines = []
machine_validator = MachineValidationPluginManager.get_validator(
account_driver
)
cloud_machines = [
cloud_machine for cloud_machine in cloud_machines
if not validate or machine_validator.machine_is_valid(cloud_machine)
]
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(
db_machines, cloud_machines, now=now, dry_run=dry_run
)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
# Clear out application, provider machine, and version memberships
# if the result is >128.
# Additionally, remove all users who are not in the machine request (if one exists).
_clean_memberships(db_machines, account_driver)
celery_logger.info(
"prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned." %
(provider, app_count, ver_count, mach_count)
)
if print_logs:
_exit_stdout_logging(console_handler)
def get_queryset(self):
return super(ActiveVolumesManager, self)\
.get_queryset().filter(only_current_source())
def get_queryset(self):
return super(ActiveVolumesManager, self)\
.get_queryset().filter(only_current_source())
