def __init__(self):
self.user = UserFactory.create()
provider_type = ProviderType(name='Test_%s' % self.user.username)
provider_type.save()
platform_type = PlatformType(name='test-platform-%s' %
self.user.username)
platform_type.save()
provider = Provider(location='BIO5',
type=provider_type,
virtualization=platform_type)
provider.save()
active_status = InstanceStatus.objects.filter(name='active').last()
if not active_status:
active_status = InstanceStatus(name='active')
active_status.save()
suspended_status = InstanceStatus.objects.filter(
name='suspended').last()
if not suspended_status:
suspended_status = InstanceStatus(name='suspended')
suspended_status.save()
self.provider = provider
self.active_status = active_status
self.suspended_status = suspended_status
def __init__(self):
self.user = UserFactory.create()
provider_type = ProviderType(name='Test_%s' % self.user.username)
provider_type.save()
platform_type = PlatformType(
name='test-platform-%s' % self.user.username
)
platform_type.save()
provider = Provider(
location='BIO5', type=provider_type, virtualization=platform_type
)
provider.save()
active_status = InstanceStatus.objects.filter(name='active').last()
if not active_status:
active_status = InstanceStatus(name='active')
active_status.save()
suspended_status = InstanceStatus.objects.filter(name='suspended'
).last()
if not suspended_status:
suspended_status = InstanceStatus(name='suspended')
suspended_status.save()
self.provider = provider
self.active_status = active_status
self.suspended_status = suspended_status
def visible_applications(user):
apps = []
if not user:
return apps
from core.models import Provider, ProviderMachineMembership
active_providers = Provider.get_active()
now_time = timezone.now()
#Look only for 'Active' private applications
for app in Application.objects.filter(
Q(end_date=None) | Q(end_date__gt=now_time),
private=True):
#Retrieve the machines associated with this app
machine_set = app.providermachine_set.filter(
Q(end_date=None) | Q(end_date__gt=now_time))
#Skip app if all their machines are on inactive providers.
if all(not pm.provider.is_active() for pm in machine_set):
continue
#Add the application if 'user' is a member of the application or PM
if app.members.filter(user=user):
_add_app(apps, app)
for pm in machine_set:
if pm.members.filter(user=user):
_add_app(apps, app)
break
return apps
def get_os_account_driver(provider):
from service.accounts.openstack import AccountDriver as OSAccountDriver
if provider not in Provider.get_active(type_name='openstack'):
raise Exception("An active openstack provider is required to"
" update image owner")
accounts = OSAccountDriver(provider)
return accounts
def monitor_instance_allocations():
"""
Update instances for each active provider.
"""
for p in Provider.get_active():
monitor_instances_for.apply_async(args=[p.id],
kwargs={'check_allocations': True})
def _os_update_owner(provider_machine, tenant_name):
from core.models import Provider
from service.driver import get_admin_driver
from service.cache import get_cached_machines, get_cached_driver
provider = provider_machine.provider
if provider not in Provider.get_active(type_name='openstack'):
raise Exception("An active openstack provider is required to"
" update image owner")
esh_driver = get_cached_driver(provider)
if not esh_driver:
raise Exception("The account driver of Provider %s is required to"
" update image metadata" % provider)
esh_machines = get_cached_machines(provider, force=True)
esh_machine = [
mach for mach in esh_machines
if mach.alias == provider_machine.identifier
]
if not esh_machine:
raise Exception("Machine with ID %s not found" %
provider_machine.identifier)
esh_machine = esh_machine[0]
tenant_id = _tenant_name_to_id(provider_machine.provider, tenant_name)
update_machine_metadata(esh_driver, esh_machine, {
"owner": tenant_id,
"application_owner": tenant_name
})
def prune_machines():
"""
Query the cloud and remove any machines
that exist in the DB but can no longer be found.
"""
for p in Provider.get_active():
prune_machines_for.apply_async(args=[p.id])
def search(cls, query, identity=None):
if identity:
base_apps = Application.objects.filter(
# Privately owned OR public machines
Q(private=True,
providermachine__instance_source__created_by_identity=identity)
| Q(private=False,
providermachine__instance_source__provider=identity.provider))
else:
active_providers = Provider.get_active()
base_apps = Application.objects.filter(
# Public machines
private=False,
#Providermachine's provider is active
providermachine__instance_source__provider__in=active_providers)
# AND query matches on:
query_match = base_apps.filter(
# app tag name
Q(tags__name__icontains=query)
# OR app tag desc
| Q(tags__description__icontains=query)
# OR app name
| Q(name__icontains=query)
# OR app desc
| Q(description__icontains=query),
only_current_source())
return query_match.distinct()
def search(cls, query, identity=None):
if identity:
base_apps = Application.objects.filter(
# Privately owned OR public machines
Q(private=True, providermachine__created_by_identity=identity)
| Q(private=False, providermachine__provider=identity.provider)
)
else:
active_providers = Provider.get_active()
base_apps = Application.objects.filter(
# Public machines
private=False,
#Providermachine's provider is active
providermachine__provider__in=active_providers)
# AND query matches on:
query_match = base_apps.filter(
# app tag name
Q(tags__name__icontains=query)
# OR app tag desc
| Q(tags__description__icontains=query)
# OR app name
| Q(name__icontains=query)
# OR app desc
| Q(description__icontains=query),
only_current())
return query_match.distinct()
def prune_machines():
"""
Query the cloud and remove any machines
that exist in the DB but can no longer be found.
"""
for p in Provider.get_active():
prune_machines_for.apply_async(args=[p.id])
def get(self, request, project_uuid):
user = request.user
group = get_user_group(user.username)
project = get_group_project(group, project_uuid)
if not project:
return Response(
"Project with ID=%s does not "
"exist" % project_uuid,
status=status.HTTP_400_BAD_REQUEST
)
instances = project.instances.filter(
only_current(), provider_machine__provider__active=True
)
active_provider_uuids = [ap.uuid for ap in Provider.get_active()]
instances = instances.filter(
pk__in=[
i.id for i in instances
if i.instance.provider_uuid() in active_provider_uuids
]
)
serialized_data = InstanceSerializer(
instances, many=True, context={
"request": request
}
).data
response = Response(serialized_data)
return response
def get_os_account_driver(provider):
from service.accounts.openstack import AccountDriver as OSAccountDriver
if provider not in Provider.get_active(type_name='openstack'):
raise Exception("An active openstack provider is required to"
" update image owner")
accounts = OSAccountDriver(provider)
return accounts
def get(self, request, provider_uuid):
try:
provider = Provider.get_active(provider_uuid)
except Provider.DoesNotExist:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The provider does not exist.")
admin_driver = get_admin_driver(provider)
if not admin_driver:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The driver cannot be retrieved for this provider.")
if not hasattr(admin_driver._connection, "ex_hypervisor_statistics"):
return failure_response(
status.HTTP_404_NOT_FOUND,
"Occupancy statistics cannot be retrieved for this provider.")
try:
stats = admin_driver._connection.ex_hypervisor_statistics()
return Response(stats)
except (socket_error, ConnectionFailure):
return connection_failure(provider_uuid)
except Exception as exc:
return failure_response(
status.HTTP_503_SERVICE_UNAVAILABLE,
"Error occurred while retrieving statistics: %s" %
exc)
def monitor_instance_allocations():
"""
Update instances for each active provider.
"""
if settings.USE_ALLOCATION_SOURCE:
celery_logger.info("Skipping the old method of monitoring instance allocations")
return False
for p in Provider.get_active():
monitor_instances_for.apply_async(args=[p.id], kwargs={'check_allocations':True})
def _fix_existing_machines():
#NOTE: Run once on prod, then remove this function..
from core.models import Provider, ProviderMachine, MachineRequest
for pm in ProviderMachine.objects.filter(provider__in=Provider.get_active(type_name='openstack')):
owner = pm.application.created_by
if 'admin' not in owner.username:
try:
update_owner(pm, owner.username)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
def _fix_existing_machines():
#NOTE: Run once on prod, then remove this function..
from core.models import Provider, ProviderMachine, MachineRequest
for pm in ProviderMachine.objects.filter(provider__in=Provider.get_active(type_name='openstack')):
owner = pm.application.created_by
if 'admin' not in owner.username:
try:
update_owner(pm, owner.username)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
def get(self, request, provider_id):
try:
provider = Provider.get_active(provider_id)
except Provider.DoesNotExist:
return failure_response(status.HTTP_404_NOT_FOUND,
"The provider does not exist.")
admin_driver = get_admin_driver(provider)
if not admin_driver:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The driver cannot be retrieved for this provider.")
if hasattr(admin_driver._connection, "ex_hypervisor_statistics"):
return Response(
admin_driver._connection.ex_hypervisor_statistics())
else:
return failure_response(
status.HTTP_404_NOT_FOUND,
"Hypervisor statistics are unavailable for this provider.")
def get(self, request, provider_uuid):
try:
provider = Provider.get_active(provider_uuid)
except Provider.DoesNotExist:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The provider does not exist.")
admin_driver = get_admin_driver(provider)
if not admin_driver:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The driver cannot be retrieved for this provider.")
if hasattr(admin_driver._connection, "ex_hypervisor_statistics"):
return Response(
admin_driver._connection.ex_hypervisor_statistics())
else:
return failure_response(
status.HTTP_404_NOT_FOUND,
"Hypervisor statistics are unavailable for this provider.")
def _os_update_owner(provider_machine, tenant_name):
from core.models import Provider
from service.driver import get_admin_driver
provider = provider_machine.provider
if provider not in Provider.get_active(type_name='openstack'):
raise Exception("An active openstack provider is required to"
" update image owner")
esh_driver = get_admin_driver(provider)
if not esh_driver:
raise Exception("The account driver of Provider %s is required to"
" update image metadata" % provider)
esh_machine = esh_driver.get_machine(provider_machine.identifier)
if not esh_machine:
raise Exception("Machine with ID %s not found"
% provider_machine.identifier)
tenant_id = _tenant_name_to_id(provider_machine.provider, tenant_name)
update_machine_metadata(esh_driver, esh_machine,
{"owner": tenant_id,
"application_owner": tenant_name})
def get(self, request, provider_uuid):
"""
Returns occupancy data for the specific provider.
"""
try:
provider = Provider.get_active(provider_uuid)
except Provider.DoesNotExist:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The provider does not exist.")
admin_driver = get_admin_driver(provider)
if not admin_driver:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The driver cannot be retrieved for this provider.")
meta_driver = admin_driver.meta(admin_driver=admin_driver)
esh_size_list = meta_driver.occupancy()
core_size_list = [convert_esh_size(size, provider_uuid)
for size in esh_size_list]
serialized_data = ProviderSizeSerializer(core_size_list,
many=True).data
return Response(serialized_data)
def get(self, request, provider_uuid):
"""
Returns occupancy data for the specific provider.
"""
try:
provider = Provider.get_active(provider_uuid)
except Provider.DoesNotExist:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The provider does not exist.")
admin_driver = get_admin_driver(provider)
if not admin_driver:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The driver cannot be retrieved for this provider.")
meta_driver = admin_driver.meta(admin_driver=admin_driver)
esh_size_list = meta_driver.occupancy()
core_size_list = [convert_esh_size(size, provider_uuid)
for size in esh_size_list]
serialized_data = ProviderSizeSerializer(core_size_list,
many=True).data
return Response(serialized_data)
def visible_applications(user):
apps = []
if not user:
return apps
from core.models import Provider, ProviderMachineMembership
active_providers = Provider.get_active()
now_time = timezone.now()
#Look only for 'Active' private applications
for app in Application.objects.filter(only_current(), private=True):
#Retrieve the machines associated with this app
machine_set = app.providermachine_set.filter(only_current_source())
#Skip app if all their machines are on inactive providers.
if all(not pm.instance_source.provider.is_active() for pm in machine_set):
continue
#Add the application if 'user' is a member of the application or PM
if app.members.filter(user=user):
_add_app(apps, app)
for pm in machine_set:
if pm.members.filter(user=user):
_add_app(apps, app)
break
return apps
def get(self, request, provider_uuid):
try:
provider = Provider.get_active(provider_uuid)
except Provider.DoesNotExist:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The provider does not exist.")
admin_driver = get_admin_driver(provider)
if not admin_driver:
return failure_response(
status.HTTP_404_NOT_FOUND,
"The driver cannot be retrieved for this provider.")
if not hasattr(admin_driver._connection, "ex_hypervisor_statistics"):
return failure_response(
status.HTTP_404_NOT_FOUND,
"Occupancy statistics cannot be retrieved for this provider.")
try:
stats = admin_driver._connection.ex_hypervisor_statistics()
return Response(stats)
except (socket_error, ConnectionFailure):
return connection_failure(provider_uuid)
except Exception as exc:
return failure_response(status.HTTP_503_SERVICE_UNAVAILABLE,
"Error occurred while retrieving statistics: %s" % exc)
def monitor_sizes():
"""
Update sizes for each active provider.
"""
for p in Provider.get_active():
monitor_sizes_for.apply_async(args=[p.id])
def monitor_machines():
"""
Update machines by querying the Cloud for each active provider.
"""
for p in Provider.get_active():
monitor_machines_for.apply_async(args=[p.id])
def current_providers(self):
return Provider.shared_with_group(self).filter(only_current(), active=True)
def monitor_instance_allocations():
"""
Update instances for each active provider.
"""
for p in Provider.get_active():
monitor_instances_for.apply_async(args=[p.id], check_allocations=True)
def monitor_sizes():
"""
Update sizes for each active provider.
"""
for p in Provider.get_active():
monitor_sizes_for.apply_async(args=[p.id])
def monitor_resources():
"""
Update instances for each active provider.
"""
for p in Provider.get_active():
monitor_resources_for.apply_async(args=[p.id])
def monitor_machines():
"""
Update machines by querying the Cloud for each active provider.
"""
for p in Provider.get_active():
monitor_machines_for.apply_async(args=[p.id])
def monitor_instances():
"""
Update instances for each active provider.
"""
for p in Provider.get_active():
monitor_instances_for(p)
from threepio import logger
def _fix_existing_machines():
#NOTE: Run once on prod, then remove this function..
from core.models import Provider, ProviderMachine, MachineRequest
for pm in ProviderMachine.objects.filter(provider__in=Provider.get_active(type_name='openstack')):
owner = pm.application.created_by
if 'admin' not in owner.username:
try:
update_owner(pm, owner.username)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
for mr in MachineRequest.objects.filter(new_machine__isnull=False, new_machine_provider__in=Provider.get_active(type_name='openstack')):
owner = mr.new_machine_owner.username
try:
update_owner(mr.new_machine, owner)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
def update_owner(provider_machine, tenant_name):
#TODO: If we switch from user-group model this will have to do some
# lookup from username on provider to select specific tenant for
# the _os_update portion..
_db_update_owner(provider_machine.application, tenant_name)
_os_update_owner(provider_machine, tenant_name)
def _fix_existing_machines():
#NOTE: Run once on prod, then remove this function..
from core.models import Provider, ProviderMachine, MachineRequest
for pm in ProviderMachine.objects.filter(provider__in=Provider.get_active(
type_name='openstack')):
owner = pm.application.created_by
if 'admin' not in owner.username:
try:
update_owner(pm, owner.username)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
for mr in MachineRequest.objects.filter(
new_machine__isnull=False,
new_machine_provider__in=Provider.get_active(
type_name='openstack')):
owner = mr.new_machine_owner.username
try:
update_owner(mr.new_machine, owner)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
def update_owner(provider_machine, tenant_name):
#TODO: If we switch from user-group model this will have to do some
# lookup from username on provider to select specific tenant for
# the _os_update portion..
_db_update_owner(provider_machine.application, tenant_name)
_os_update_owner(provider_machine, tenant_name)
def monitor_instances():
"""
Update instances for each active provider.
"""
for p in Provider.get_active():
monitor_instances_for(p)
def current_providers(self):
return Provider.shared_with_group(self).filter(only_current(), active=True)
from threepio import logger
def _fix_existing_machines():
#NOTE: Run once on prod, then remove this function..
from core.models import Provider, ProviderMachine, MachineRequest
for pm in ProviderMachine.objects.filter(provider__in=Provider.get_active(type_name='openstack')):
owner = pm.application.created_by
if 'admin' not in owner.username:
try:
update_owner(pm, owner.username)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
for mr in MachineRequest.objects.filter(new_machine__isnull=False, new_machine_provider__in=Provider.get_active(type_name='openstack')):
owner = mr.new_machine_owner.username
try:
update_owner(mr.new_machine, owner)
print "Updated owner for %s" % owner
except Exception, exc:
print "Cannot update owner %s" % owner
print "Exception = %s" % exc
def update_owner(provider_machine, tenant_name, update_cloud=True):
#TODO: If we switch from user-group model this will have to do some
# lookup from username on provider to select specific tenant for
# the _os_update portion..
_db_update_owner(provider_machine.application, tenant_name)
if update_cloud:
