def enable_image_validation(machine_request, init_task, final_task, original_status="", error_handler_task=None):
if not error_handler_task:
error_handler_task = machine_request_error.s(machine_request.id)
# Task 3 = Validate the new image by launching an instance
admin_ident = machine_request.new_admin_identity()
admin_driver = get_admin_driver(machine_request.new_machine_provider)
if 'validating' == original_status:
image_id = machine_request.new_machine.identifier
celery_logger.info("Start with validating:%s" % image_id)
# If validating, seed the image_id and start here..
validate_task = validate_new_image.s(image_id, machine_request.id)
init_task = validate_task
else:
validate_task = validate_new_image.s(machine_request.id)
init_task.link(validate_task)
#Validate task returns an instance_id
# Task 4 = Wait for new instance to be 'active'
wait_for_task = wait_for_instance.s(
# NOTE: 1st arg, instance_id, passed from last task.
admin_driver.__class__,
admin_driver.provider,
admin_driver.identity,
"active",
test_tmp_status=True,
return_id=True)
validate_task.link(wait_for_task)
validate_task.link_error(error_handler_task)
# Task 5 = Terminate the new instance on completion
destroy_task = destroy_instance.s(
admin_ident.created_by, admin_ident.uuid)
wait_for_task.link(destroy_task)
wait_for_task.link_error(error_handler_task)
destroy_task.link_error(error_handler_task)
destroy_task.link(final_task)
return init_task
def send_email(subject, body, from_email, to, cc=None,
fail_silently=False, html=False):
"""
Use django.core.mail.EmailMessage to send and log an Atmosphere email.
"""
try:
msg = EmailMessage(subject=subject, body=body,
from_email=from_email,
to=to,
cc=cc)
if html:
msg.content_subtype = 'html'
email_logger.info("\n> From:%s\n> To:%s\n> Cc:%s\n> Subject:%s\n> Body:\n%s", from_email, to, cc, subject, body)
if getattr(settings, "SEND_EMAILS", True):
msg.send(fail_silently=fail_silently)
email_logger.info("NOTE: Above message sent successfully")
celery_logger.info("NOTE: Above message sent successfully")
else:
email_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False")
celery_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False")
return True
except Exception as e:
celery_logger.exception(e)
return False
def mount_failed(task_uuid,
driverCls,
provider,
identity,
volume_id,
unmount=False,
**celery_task_args):
from service import volume as volume_service
try:
celery_logger.debug("mount_failed task started at %s." %
datetime.now())
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "Mount Error Traceback:%s" % (result.traceback, )
celery_logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = 'umount_error'
else:
tmp_status = 'mount_error'
return volume_service._update_volume_metadata(
driver, volume, metadata={'tmp_status': tmp_status})
celery_logger.debug("mount_failed task finished at %s." %
datetime.now())
except Exception as exc:
celery_logger.warn(exc)
mount_failed.retry(exc=exc)
def enable_image_validation(machine_request, init_task, final_task, original_status="", error_handler_task=None):
if not error_handler_task:
error_handler_task = machine_request_error.s(machine_request.id)
# Task 3 = Validate the new image by launching an instance
admin_ident = machine_request.new_admin_identity()
admin_driver = get_admin_driver(machine_request.new_machine_provider)
if 'validating' in original_status:
image_id = machine_request.new_machine.identifier
celery_logger.info("Start with validating:%s" % image_id)
# If validating, seed the image_id and start here..
validate_task = validate_new_image.s(image_id, machine_request.id)
init_task = validate_task
else:
validate_task = validate_new_image.s(machine_request.id)
init_task.link(validate_task)
#Validate task returns an instance_id
# Task 4 = Wait for new instance to be 'active'
wait_for_task = wait_for_instance.s(
# NOTE: 1st arg, instance_id, passed from last task.
admin_driver.__class__,
admin_driver.provider,
admin_driver.identity,
"active",
test_tmp_status=True,
return_id=True)
validate_task.link(wait_for_task)
validate_task.link_error(error_handler_task)
# Task 5 = Terminate the new instance on completion
destroy_task = destroy_instance.s(
admin_ident.created_by, admin_ident.uuid)
wait_for_task.link(destroy_task)
wait_for_task.link_error(error_handler_task)
destroy_task.link_error(error_handler_task)
destroy_task.link(final_task)
return init_task
def send_email(subject, body, from_email, to, cc=None,
fail_silently=False, html=False):
"""
Use django.core.mail.EmailMessage to send and log an Atmosphere email.
"""
try:
msg = EmailMessage(subject=subject, body=body,
from_email=from_email,
to=to,
cc=cc)
if html:
msg.content_subtype = 'html'
log_message = "\n> From:{0}\n> To:{1}\n> Cc:{2}\n> Subject:{3}\n> Body:\n{4}"
args = (from_email, to, cc, subject, body)
email_logger.info(log_message.format(*args))
if getattr(settings, "SEND_EMAILS", True):
msg.send(fail_silently=fail_silently)
email_logger.info("NOTE: Above message sent successfully")
celery_logger.info("NOTE: Above message sent successfully")
else:
email_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False")
celery_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False")
return True
except Exception as e:
celery_logger.exception(e)
return False
def _share_image(account_driver,
cloud_machine,
identity,
members,
dry_run=False):
"""
INPUT: use account_driver to share cloud_machine with identity (if not in 'members' list)
"""
# Skip tenant-names who are NOT in the DB, and tenants who are already included
missing_tenant = identity.credential_set.filter(
~Q(value__in=members), key='ex_tenant_name')
if missing_tenant.count() == 0:
#celery_logger.debug("SKIPPED _ Image %s already shared with %s" % (cloud_machine.id, identity))
return
elif missing_tenant.count() > 1:
raise Exception("Safety Check -- You should not be here")
tenant_name = missing_tenant[0]
if cloud_machine.is_public == True:
celery_logger.info("Making Machine %s private" % cloud_machine.id)
cloud_machine.update(is_public=False)
celery_logger.info("Sharing image %s<%s>: %s with %s" %
(cloud_machine.id, cloud_machine.name,
identity.provider.location, tenant_name.value))
if not dry_run:
account_driver.image_manager.share_image(cloud_machine,
tenant_name.value)
def make_machines_private(application, identities, account_drivers={}, provider_tenant_mapping={}, image_maps={}, dry_run=False):
"""
This method is called when the DB has marked the Machine/Application as PUBLIC
But the CLOUD states that the machine is really private.
GOAL: All versions and machines will be listed as PRIVATE on the cloud and include AS MANY identities as exist.
"""
for version in application.active_versions():
for machine in version.active_machines():
# For each *active* machine in app/version..
# Loop over each identity and check the list of 'current tenants' as viewed by keystone.
account_driver = memoized_driver(machine, account_drivers)
tenant_name_mapping = memoized_tenant_name_map(account_driver, provider_tenant_mapping)
current_tenants = get_current_members(
account_driver, machine, tenant_name_mapping)
provider = machine.instance_source.provider
cloud_machine = memoized_image(account_driver, machine, image_maps)
for identity in identities:
if identity.provider == provider:
_share_image(account_driver, cloud_machine, identity, current_tenants, dry_run=dry_run)
add_application_membership(application, identity, dry_run=dry_run)
# All the cloud work has been completed, so "lock down" the application.
if application.private == False:
application.private = True
celery_logger.info("Making Application %s private" % application.name)
if not dry_run:
application.save()
def prep_instance_for_snapshot(identity_id, instance_id, **celery_task_args):
identity = Identity.objects.get(id=identity_id)
try:
celery_logger.debug("prep_instance_for_snapshot task started at %s." % timezone.now())
# NOTE: FIXMEIF the assumption that the 'linux username'
# is the 'created_by' AtmosphereUser changes.
username = identity.created_by.username
driver = get_esh_driver(identity)
instance = driver.get_instance(instance_id)
if instance.extra.get('status','') != 'active':
celery_logger.info("prep_instance_for_snapshot skipped")
return
playbooks = deploy_prepare_snapshot(
instance.ip, username, instance_id)
celery_logger.info(playbooks.__dict__)
hostname = build_host_name(instance.id, instance.ip)
result = False if execution_has_failures(playbooks, hostname)\
or execution_has_unreachable(playbooks, hostname) else True
if not result:
raise Exception(
"Error encountered while preparing instance for snapshot: %s"
% playbooks.stats.summarize(host=hostname))
except Exception as exc:
celery_logger.warn(exc)
prep_instance_for_snapshot.retry(exc=exc)
def _share_image(account_driver, cloud_machine, identity, members, dry_run=False):
"""
INPUT: use account_driver to share cloud_machine with identity (if not in 'members' list)
"""
# Skip tenant-names who are NOT in the DB, and tenants who are already included
missing_tenant = identity.credential_set.filter(~Q(value__in=members), key='ex_tenant_name')
if missing_tenant.count() == 0:
#celery_logger.debug("SKIPPED _ Image %s already shared with %s" % (cloud_machine.id, identity))
return
elif missing_tenant.count() > 1:
raise Exception("Safety Check -- You should not be here")
tenant_name = missing_tenant[0]
cloud_machine_is_public = cloud_machine.is_public if hasattr(cloud_machine,'is_public') else cloud_machine.get('visibility','') == 'public'
if cloud_machine_is_public == True:
celery_logger.info("Making Machine %s private" % cloud_machine.id)
account_driver.image_manager.glance.images.update(cloud_machine.id, visibility='private')
celery_logger.info("Sharing image %s<%s>: %s with %s" % (cloud_machine.id, cloud_machine.name, identity.provider.location, tenant_name.value))
if not dry_run:
try:
account_driver.image_manager.share_image(cloud_machine, tenant_name.value)
except GlanceConflict as exc:
if 'already associated with image' in exc.message:
pass
except GlanceForbidden as exc:
if 'Public images do not have members' in exc.message:
celery_logger.warn("CONFLICT -- This image should have been marked 'private'! %s" % cloud_machine)
pass
return
def _share_image(account_driver, cloud_machine, identity, members, dry_run=False):
"""
INPUT: use account_driver to share cloud_machine with identity (if not in 'members' list)
"""
# Skip tenant-names who are NOT in the DB, and tenants who are already included
missing_tenant = identity.credential_set.filter(~Q(value__in=members), key='ex_tenant_name')
if missing_tenant.count() == 0:
#celery_logger.debug("SKIPPED _ Image %s already shared with %s" % (cloud_machine.id, identity))
return
elif missing_tenant.count() > 1:
raise Exception("Safety Check -- You should not be here")
tenant_name = missing_tenant[0]
cloud_machine_is_public = cloud_machine.is_public if hasattr(cloud_machine,'is_public') else cloud_machine.get('visibility','') == 'public'
if cloud_machine_is_public == True:
celery_logger.info("Making Machine %s private" % cloud_machine.id)
if not dry_run:
account_driver.image_manager.glance.images.update(cloud_machine.id, visibility='shared')
celery_logger.info("Sharing image %s<%s>: %s with %s" % (cloud_machine.id, cloud_machine.name, identity.provider.location, tenant_name.value))
if not dry_run:
try:
account_driver.image_manager.share_image(cloud_machine, tenant_name.value)
except GlanceConflict as exc:
if 'already associated with image' in exc.message:
pass
except GlanceForbidden as exc:
if 'Public images do not have members' in exc.message:
celery_logger.warn("CONFLICT -- This image should have been marked 'shared'! %s" % cloud_machine)
pass
return
def send_email(subject, body, from_email, to, cc=None, html=False):
"""
Use django.core.mail.EmailMessage to send and log an Atmosphere email.
"""
try:
msg = EmailMessage(subject=subject,
body=body,
from_email=from_email,
to=to,
cc=cc)
if html:
msg.content_subtype = 'html'
email_logger.info(
"\n> From:%s\n> To:%s\n> Cc:%s\n> Subject:%s\n> Body:\n%s",
from_email, to, cc, subject, body)
if getattr(settings, "SEND_EMAILS", True):
msg.send()
email_logger.info("NOTE: Above message sent successfully")
celery_logger.info("NOTE: Above message sent successfully")
else:
email_logger.info(
"NOTE: Above message not sent -- SEND_EMAILS was False")
celery_logger.info(
"NOTE: Above message not sent -- SEND_EMAILS was False")
return True
except Exception as e:
celery_logger.exception(e)
return False
def make_machines_private(application, identities, account_drivers={}, provider_tenant_mapping={}, image_maps={}, dry_run=False):
"""
This method is called when the DB has marked the Machine/Application as PUBLIC
But the CLOUD states that the machine is really private.
GOAL: All versions and machines will be listed as PRIVATE on the cloud and include AS MANY identities as exist.
"""
for version in application.active_versions():
for machine in version.active_machines():
# For each *active* machine in app/version..
# Loop over each identity and check the list of 'current tenants' as viewed by keystone.
account_driver = memoized_driver(machine, account_drivers)
tenant_name_mapping = memoized_tenant_name_map(account_driver, provider_tenant_mapping)
current_tenants = get_current_members(
account_driver, machine, tenant_name_mapping)
provider = machine.instance_source.provider
cloud_machine = memoized_image(account_driver, machine, image_maps)
for identity in identities:
if identity.provider == provider:
_share_image(account_driver, cloud_machine, identity, current_tenants, dry_run=dry_run)
add_application_membership(application, identity, dry_run=dry_run)
# All the cloud work has been completed, so "lock down" the application.
if not application.private:
application.private = True
celery_logger.info("Making Application %s private" % application.name)
if not dry_run:
application.save()
def prep_instance_for_snapshot(identity_id, instance_id, **celery_task_args):
identity = Identity.objects.get(id=identity_id)
try:
celery_logger.debug("prep_instance_for_snapshot task started at %s." %
timezone.now())
# NOTE: FIXMEIF the assumption that the 'linux username'
# is the 'created_by' AtmosphereUser changes.
username = identity.created_by.username
driver = get_esh_driver(identity)
instance = driver.get_instance(instance_id)
if instance.extra.get('status', '') != 'active':
celery_logger.info("prep_instance_for_snapshot skipped")
return
playbooks = deploy_prepare_snapshot(instance.ip, username, instance_id)
celery_logger.info(playbooks.__dict__)
hostname = build_host_name(instance.id, instance.ip)
result = False if execution_has_failures(playbooks, hostname)\
or execution_has_unreachable(playbooks, hostname) else True
if not result:
raise Exception(
"Error encountered while preparing instance for snapshot: %s" %
playbooks.stats.summarize(host=hostname))
except Exception as exc:
celery_logger.warn(exc)
prep_instance_for_snapshot.retry(exc=exc)
def monitor_allocation_sources(usernames=()):
"""
Monitor allocation sources, if a snapshot shows that all compute has been used, then enforce as necessary
"""
celery_logger.debug('monitor_allocation_sources - usernames: %s',
usernames)
allocation_sources = AllocationSource.objects.all()
for allocation_source in allocation_sources.order_by('name'):
celery_logger.debug(
'monitor_allocation_sources - allocation_source: %s',
allocation_source)
for user in allocation_source.all_users.order_by('username'):
celery_logger.debug('monitor_allocation_sources - user: %s', user)
if usernames and user.username not in usernames:
celery_logger.info("Skipping User %s - not in the list" %
user.username)
continue
over_allocation = allocation_source.is_over_allocation(user)
celery_logger.debug(
'monitor_allocation_sources - user: %s, over_allocation: %s',
user, over_allocation)
if not over_allocation:
continue
celery_logger.debug(
'monitor_allocation_sources - Going to enforce on user user: %s',
user)
allocation_source_overage_enforcement_for_user.apply_async(
args=(allocation_source, user))
def mount_volume_task(
driverCls,
provider,
identity,
instance_id,
volume_id,
device_location,
mount_location,
device_type,
mount_prefix=None,
*args,
**kwargs
):
try:
celery_logger.debug("mount task started at %s." % timezone.now())
celery_logger.debug("mount_location: %s" % (mount_location, ))
driver = get_driver(driverCls, provider, identity)
username = identity.get_username()
instance = driver.get_instance(instance_id)
volume = driver.get_volume(volume_id)
try:
attach_data = volume.extra['attachments'][0]
if not device_location:
device_location = attach_data['device']
except (KeyError, IndexError):
celery_logger.warn(
"Volume %s missing attachments in Extra" % (volume, )
)
if not device_location:
raise Exception(
"No device_location found or inferred by volume %s" % volume
)
if not mount_prefix:
mount_prefix = "/vol_"
last_char = device_location[-1] # /dev/sdb --> b
if not mount_location:
mount_location = mount_prefix + last_char
playbook_results = deploy_mount_volume(
instance.ip,
username,
instance.id,
device_location,
mount_location=mount_location,
device_type=device_type
)
celery_logger.info(playbook_results)
if execution_has_failures(
playbook_results
) or execution_has_unreachable(playbook_results):
raise Exception(
"Error encountered while mounting volume: instance_id: {}, volume_id: {}"
.format(instance_id, volume_id)
)
return mount_location
except Exception as exc:
celery_logger.warn(exc)
mount_volume_task.retry(exc=exc)
def mount_failed(
context,
exception_msg,
traceback,
driverCls,
provider,
identity,
volume_id,
unmount=False,
**celery_task_args
):
from service import volume as volume_service
try:
celery_logger.debug("mount_failed task started at %s." % timezone.now())
celery_logger.info("task context=%s" % context)
err_str = "%s\nMount Error Traceback:%s" % (exception_msg, traceback)
celery_logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = 'umount_error'
else:
tmp_status = 'mount_error'
return volume_service._update_volume_metadata(
driver, volume, metadata={'tmp_status': tmp_status}
)
except Exception as exc:
celery_logger.warn(exc)
mount_failed.retry(exc=exc)
def _share_image(account_driver,
cloud_machine,
identity,
members,
dry_run=False):
"""
INPUT: use account_driver to share cloud_machine with identity (if not in 'members' list)
"""
# Skip tenant-names who are NOT in the DB, and tenants who are already included
missing_tenant = identity.credential_set.filter(~Q(value__in=members),
key='ex_tenant_name')
if missing_tenant.count() == 0:
#celery_logger.debug("SKIPPED _ Image %s already shared with %s" % (cloud_machine.id, identity))
return
elif missing_tenant.count() > 1:
raise Exception("Safety Check -- You should not be here")
tenant_name = missing_tenant[0]
if cloud_machine.is_public == True:
celery_logger.info("Making Machine %s private" % cloud_machine.id)
cloud_machine.update(is_public=False)
celery_logger.info("Sharing image %s<%s>: %s with %s" %
(cloud_machine.id, cloud_machine.name,
identity.provider.location, tenant_name.value))
if not dry_run:
account_driver.image_manager.share_image(cloud_machine,
tenant_name.value)
def distribute_image_membership(account_driver, cloud_machine, provider):
"""
Based on what we know about the DB, at a minimum, ensure that their projects are added to the image_members list for this cloud_machine.
"""
pm = ProviderMachine.objects.get(
instance_source__provider=provider,
instance_source__identifier=cloud_machine.id
)
group_ids = ProviderMachineMembership.objects.filter(provider_machine=pm
).values_list(
'group', flat=True
)
groups = Group.objects.filter(id__in=group_ids)
for group in groups:
try:
celery_logger.info(
"Add %s to cloud membership for %s" % (group, pm)
)
update_cloud_membership_for_machine(pm, group)
except TimeoutError:
celery_logger.warn(
"Failed to add cloud membership for %s - Operation timed out" %
group
)
return groups
def update_snapshot_cyverse(start_date=None, end_date=None):
logger.debug("update_snapshot_cyverse task started at %s." %
datetime.now())
end_date = timezone.now().replace(
microsecond=0) if not end_date else end_date
for allocation_source in AllocationSource.objects.order_by('name'):
# calculate and save snapshots here
allocation_source_name = allocation_source.name
last_renewal_event = EventTable.objects.filter(
name='allocation_source_created_or_renewed',
payload__allocation_source_name__exact=str(
allocation_source_name)).order_by('timestamp')
if not last_renewal_event:
logger.info('Allocation Source %s Create/Renewal event missing',
allocation_source_name)
continue
start_date = last_renewal_event.last().timestamp.replace(
microsecond=0) if not start_date else start_date
total_compute_used = 0
total_burn_rate = 0
for user in allocation_source.all_users:
compute_used, burn_rate = total_usage(
user.username,
start_date=start_date,
end_date=end_date,
allocation_source_name=allocation_source_name,
burn_rate=True)
UserAllocationSnapshot.objects.update_or_create(
allocation_source=allocation_source,
user=user,
defaults={
'compute_used': compute_used,
'burn_rate': burn_rate
})
total_compute_used += compute_used
total_burn_rate += burn_rate
AllocationSourceSnapshot.objects.update_or_create(
allocation_source=allocation_source,
defaults={
'compute_used': total_compute_used,
'global_burn_rate': total_burn_rate
})
run_all(
rule_list=cyverse_rules,
defined_variables=CyverseTestRenewalVariables(
allocation_source, end_date, start_date),
defined_actions=CyverseTestRenewalActions(allocation_source,
end_date),
)
# At the end of the task, fire-off an allocation threshold check
logger.debug("update_snapshot_cyverse task finished at %s." %
datetime.now())
allocation_threshold_check.apply_async()
def monitor_allocation_sources(usernames=()):
"""
Monitor allocation sources, if a snapshot shows that all compute has been used, then enforce as necessary
"""
celery_logger.debug('monitor_allocation_sources - usernames: %s', usernames)
allocation_sources = AllocationSource.objects.all()
for allocation_source in allocation_sources.order_by('name'):
celery_logger.debug(
'monitor_allocation_sources - allocation_source: %s',
allocation_source
)
for user in allocation_source.all_users.order_by('username'):
celery_logger.debug('monitor_allocation_sources - user: %s', user)
if usernames and user.username not in usernames:
celery_logger.info(
"Skipping User %s - not in the list" % user.username
)
continue
over_allocation = allocation_source.is_over_allocation(user)
celery_logger.debug(
'monitor_allocation_sources - user: %s, over_allocation: %s',
user, over_allocation
)
enforcement_override_choice = AllocationSourcePluginManager.get_enforcement_override(
user, allocation_source
)
celery_logger.debug(
'monitor_allocation_sources - enforcement_override_choice: %s',
enforcement_override_choice
)
if over_allocation and enforcement_override_choice == EnforcementOverrideChoice.NEVER_ENFORCE:
celery_logger.debug(
'Allocation source is over allocation, but %s + user %s has an override of %s, '
'therefore not enforcing', allocation_source, user,
enforcement_override_choice
)
continue
if not over_allocation and enforcement_override_choice == EnforcementOverrideChoice.ALWAYS_ENFORCE:
celery_logger.debug(
'Allocation source is not over allocation, but %s + user %s has an override of %s, '
'therefore enforcing', allocation_source, user,
enforcement_override_choice
)
# Note: The enforcing happens in the next `if` statement.
if over_allocation or enforcement_override_choice == EnforcementOverrideChoice.ALWAYS_ENFORCE:
assert enforcement_override_choice in (
EnforcementOverrideChoice.NO_OVERRIDE,
EnforcementOverrideChoice.ALWAYS_ENFORCE
)
celery_logger.debug(
'monitor_allocation_sources - Going to enforce on user: %s',
user
)
allocation_source_overage_enforcement_for_user.apply_async(
args=(allocation_source, user)
)
def prune_machines_for(
provider_id, print_logs=False, dry_run=False, forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
import logging
import sys
consolehandler = logging.StreamHandler(sys.stdout)
consolehandler.setLevel(logging.DEBUG)
celery_logger.addHandler(consolehandler)
celery_logger.info("Starting prune_machines for Provider %s @ %s"
% (provider, now))
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(
db_machines, cloud_machines, now=now, dry_run=dry_run)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
celery_logger.info(
"prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned."
% (provider, app_count, ver_count, mach_count))
if print_logs:
celery_logger.removeHandler(consolehandler)
def monitor_instance_allocations():
"""
Update instances for each active provider.
"""
if settings.USE_ALLOCATION_SOURCE:
celery_logger.info("Skipping the old method of monitoring instance allocations")
return False
for p in Provider.get_active():
monitor_instances_for.apply_async(args=[p.id], kwargs={'check_allocations':True})
def prune_machines_for(provider_id,
print_logs=False,
dry_run=False,
forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
console_handler = _init_stdout_logging()
celery_logger.info("Starting prune_machines for Provider %s @ %s" %
(provider, now))
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(db_machines,
cloud_machines,
now=now,
dry_run=dry_run)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
celery_logger.info("prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned." %
(provider, app_count, ver_count, mach_count))
if print_logs:
_exit_stdout_logging(console_handler)
def machine_is_valid(cloud_machine, accounts):
"""
As the criteria for "what makes a glance image an atmosphere ProviderMachine" changes, we can use this function to hook out to external plugins, etc.
Filters out:
- ChromoSnapShot, eri-, eki-
- Private images not shared with atmosphere accounts
- Domain-specific image catalog(?)
"""
provider = accounts.core_provider
# If the name of the machine indicates that it is a Ramdisk, Kernel, or Chromogenic Snapshot, skip it.
if any(
cloud_machine.name.startswith(prefix)
for prefix in ['eri-', 'eki-', 'ChromoSnapShot']):
celery_logger.info("Skipping cloud machine %s" % cloud_machine)
return False
# If the metadata 'skip_atmosphere' is found, do not add the machine.
if cloud_machine.get('skip_atmosphere', False):
celery_logger.info(
"Skipping cloud machine %s - Includes 'skip_atmosphere' metadata" %
cloud_machine)
return False
# If the metadata indicates that the image-type is snapshot -- skip it.
if cloud_machine.get('image_type', 'image') == 'snapshot':
celery_logger.info(
"Skipping cloud machine %s - Image type indicates a snapshot" %
cloud_machine)
return False
owner_project = _get_owner(accounts, cloud_machine)
# If the image is private, ensure that an owner can be found inside the system.
if cloud_machine.get('visibility', '') == 'private':
shared_with_projects = accounts.shared_images_for(cloud_machine.id)
shared_with_projects.append(owner_project)
project_names = [p.name for p in shared_with_projects
if p] # TODO: better error handling here
identity_matches = provider.identity_set.filter(
credential__key='ex_project_name',
credential__value__in=project_names).count() > 0
if not identity_matches:
celery_logger.info(
"Skipping private machine %s - The owner does not exist in Atmosphere"
% cloud_machine)
return False
if accounts.provider_creds.get('ex_force_auth_version',
'2.0_password') != '3.x_password':
return True
# NOTE: Potentially if we wanted to do 'domain-restrictions' *inside* of atmosphere,
# we could do that (based on the domain of the image owner) here.
domain_id = owner_project.domain_id
config_domain = accounts.get_config('user', 'domain', 'default')
owner_domain = accounts.openstack_sdk.identity.get_domain(domain_id)
account_domain = accounts.openstack_sdk.identity.get_domain(config_domain)
if owner_domain.id != account_domain.id: # and if FLAG FOR DOMAIN-SPECIFIC ATMOSPHERE
celery_logger.info(
"Skipping private machine %s - The owner belongs to a different domain (%s)"
% (cloud_machine, owner_domain))
return False
return True
def monitor_volumes_for(provider_id, print_logs=False):
"""
Run the set of tasks related to monitoring sizes for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
"""
from service.driver import get_account_driver
from core.models import Identity
if print_logs:
console_handler = _init_stdout_logging()
provider = Provider.objects.get(id=provider_id)
account_driver = get_account_driver(provider)
# Non-End dated volumes on this provider
db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider)
all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30)
seen_volumes = []
for cloud_volume in all_volumes:
try:
core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid)
seen_volumes.append(core_volume)
except ObjectDoesNotExist:
tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id']
tenant = account_driver.get_project_by_id(tenant_id)
tenant_name = tenant.name if tenant else tenant_id
try:
if not tenant:
celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume)
raise ObjectDoesNotExist()
identity = Identity.objects.filter(
contains_credential('ex_project_name', tenant_name), provider=provider
).first()
if not identity:
raise ObjectDoesNotExist()
core_volume = convert_esh_volume(
cloud_volume,
provider.uuid, identity.uuid,
identity.created_by)
except ObjectDoesNotExist:
celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name))
pass
now_time = timezone.now()
needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes]
for volume in needs_end_date:
celery_logger.debug("End dating inactive volume: %s" % volume)
volume.end_date = now_time
volume.save()
if print_logs:
_exit_stdout_logging(console_handler)
for vol in seen_volumes:
vol.esh = None
return [vol.instance_source.identifier for vol in seen_volumes]
def monitor_volumes_for(provider_id, print_logs=False):
"""
Run the set of tasks related to monitoring sizes for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
"""
from service.driver import get_account_driver
from core.models import Identity
if print_logs:
console_handler = _init_stdout_logging()
provider = Provider.objects.get(id=provider_id)
account_driver = get_account_driver(provider)
# Non-End dated volumes on this provider
db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider)
all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30)
seen_volumes = []
for cloud_volume in all_volumes:
try:
core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid)
seen_volumes.append(core_volume)
except ObjectDoesNotExist:
tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id']
tenant = account_driver.get_project_by_id(tenant_id)
tenant_name = tenant.name if tenant else tenant_id
try:
if not tenant:
celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume)
raise ObjectDoesNotExist()
identity = Identity.objects.filter(
contains_credential('ex_project_name', tenant_name), provider=provider
).first()
if not identity:
raise ObjectDoesNotExist()
core_volume = convert_esh_volume(
cloud_volume,
provider.uuid, identity.uuid,
identity.created_by)
except ObjectDoesNotExist:
celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name))
pass
now_time = timezone.now()
needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes]
for volume in needs_end_date:
celery_logger.debug("End dating inactive volume: %s" % volume)
volume.end_date = now_time
volume.save()
if print_logs:
_exit_stdout_logging(console_handler)
for vol in seen_volumes:
vol.esh = None
return [vol.instance_source.identifier for vol in seen_volumes]
def export_request_error(task_uuid, export_request_id):
celery_logger.info("export_request_id=%s" % export_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback,)
celery_logger.error(err_str)
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = err_str
export_request.save()
def add_application_membership(application, identity, dry_run=False):
for membership_obj in identity.identitymembership_set.all():
# For every 'member' of this identity:
group = membership_obj.member
# Add an application membership if not already there
if application.applicationmembership_set.filter(group=group).count() == 0:
celery_logger.info("Added ApplicationMembership %s for %s" % (group.name, application.name))
if not dry_run:
ApplicationMembership.objects.create(application=application, group=group)
else:
#celery_logger.debug("SKIPPED _ Group %s already ApplicationMember for %s" % (group.name, application.name))
pass
def export_request_task(export_request_id):
celery_logger.info("export_request_task task started at %s." % timezone.now())
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = 'processing'
export_request.save()
(orig_managerCls, orig_creds) = export_request.prepare_manager()
default_kwargs = export_request.get_export_args()
file_loc = export_source(orig_managerCls, orig_creds, default_kwargs)
celery_logger.info("export_request_task task finished at %s." % timezone.now())
return file_loc
def export_request_task(export_request_id):
celery_logger.info("export_request_task task started at %s." % timezone.now())
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = 'processing'
export_request.save()
(orig_managerCls, orig_creds) = export_request.prepare_manager()
default_kwargs = export_request.get_export_args()
file_loc = export_source(orig_managerCls, orig_creds, default_kwargs)
celery_logger.info("export_request_task task finished at %s." % timezone.now())
return file_loc
def add_application_membership(application, identity, dry_run=False):
for membership_obj in identity.identity_memberships.all():
# For every 'member' of this identity:
group = membership_obj.member
# Add an application membership if not already there
if application.applicationmembership_set.filter(group=group).count() == 0:
celery_logger.info("Added ApplicationMembership %s for %s" % (group.name, application.name))
if not dry_run:
ApplicationMembership.objects.create(application=application, group=group)
else:
#celery_logger.debug("SKIPPED _ Group %s already ApplicationMember for %s" % (group.name, application.name))
pass
def export_request_error(task_uuid, export_request_id):
celery_logger.info("export_request_id=%s" % export_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback,)
celery_logger.error(err_str)
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = err_str
export_request.save()
def machine_request_error(task_uuid, machine_request_id):
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = _status_to_error(machine_request.old_status, result.result,
result.traceback)
celery_logger.error(err_str)
send_image_request_failed_email(machine_request, err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
def check_volume_task(driverCls, provider, identity, instance_id, volume_id,
*args, **kwargs):
try:
celery_logger.debug("check_volume task started at %s." %
datetime.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
volume = driver.get_volume(volume_id)
attach_data = volume.extra['attachments'][0]
device = attach_data['device']
private_key = ATMOSPHERE_PRIVATE_KEYFILE
kwargs.update({'ssh_key': private_key})
kwargs.update({'timeout': 120})
# One script to make two checks:
# 1. Voume exists 2. Volume has a filesystem
cv_script = check_volume(device)
# NOTE: non_zero_deploy needed to stop LibcloudDeploymentError from being
# raised
kwargs.update({'deploy': cv_script, 'non_zero_deploy': True})
driver.deploy_to(instance, **kwargs)
kwargs.pop('non_zero_deploy', None)
# Script execute
if cv_script.exit_status != 0:
if 'No such file' in cv_script.stdout:
raise Exception('Volume check failed: %s. '
'Device %s does not exist on instance %s' %
(volume, device, instance))
elif 'Bad magic number' in cv_script.stdout:
# Filesystem needs to be created for this device
celery_logger.info("Mkfs needed")
mkfs_script = mkfs_volume(device)
kwargs.update({'deploy': mkfs_script})
driver.deploy_to(instance, **kwargs)
else:
raise Exception('Volume check failed: Something weird')
celery_logger.debug("check_volume task finished at %s." %
datetime.now())
except LibcloudDeploymentError as exc:
celery_logger.exception(exc)
except Exception as exc:
celery_logger.warn(exc)
check_volume_task.retry(exc=exc)
def update_membership(application, shared_identities):
"""
For machine in application/version:
Get list of current users
For "super-set" list of identities:
if identity exists on provider && identity NOT in current user list:
account_driver.add_user(identity.name)
"""
db_identity_membership = identity.identitymembership_set.all().distinct()
for db_identity_member in db_identity_membership:
# For each group who holds this identity:
# grant them access to the now-private App, Version & Machine
db_group = db_identity_member.member
ApplicationMembership.objects.get_or_create(
application=application, group=db_group)
celery_logger.info("Added Application, Version, and Machine Membership to Group: %s" % (db_group,))
return application
def update_membership(application, shared_identities):
"""
For machine in application/version:
Get list of current users
For "super-set" list of identities:
if identity exists on provider && identity NOT in current user list:
account_driver.add_user(identity.name)
"""
db_identity_membership = identity.identity_memberships.all().distinct()
for db_identity_member in db_identity_membership:
# For each group who holds this identity:
# grant them access to the now-private App, Version & Machine
db_group = db_identity_member.member
ApplicationMembership.objects.get_or_create(
application=application, group=db_group)
celery_logger.info("Added Application, Version, and Machine Membership to Group: %s" % (db_group,))
return application
def machine_request_error(task_uuid, machine_request_id):
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "(%s) ERROR - %r Exception:%r" % (machine_request.old_status,
result.result,
result.traceback,
)
celery_logger.error(err_str)
send_image_request_failed_email(machine_request, err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
def check_volume_task(driverCls, provider, identity,
instance_id, volume_id, *args, **kwargs):
try:
celery_logger.debug("check_volume task started at %s." % datetime.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
volume = driver.get_volume(volume_id)
attach_data = volume.extra['attachments'][0]
device = attach_data['device']
private_key = ATMOSPHERE_PRIVATE_KEYFILE
kwargs.update({'ssh_key': private_key})
kwargs.update({'timeout': 120})
# One script to make two checks:
# 1. Voume exists 2. Volume has a filesystem
cv_script = check_volume(device)
# NOTE: non_zero_deploy needed to stop LibcloudDeploymentError from being
# raised
kwargs.update({'deploy': cv_script,
'non_zero_deploy': True})
driver.deploy_to(instance, **kwargs)
kwargs.pop('non_zero_deploy', None)
# Script execute
if cv_script.exit_status != 0:
if 'No such file' in cv_script.stdout:
raise Exception('Volume check failed: %s. '
'Device %s does not exist on instance %s'
% (volume, device, instance))
elif 'Bad magic number' in cv_script.stdout:
# Filesystem needs to be created for this device
celery_logger.info("Mkfs needed")
mkfs_script = mkfs_volume(device)
kwargs.update({'deploy': mkfs_script})
driver.deploy_to(instance, **kwargs)
else:
raise Exception('Volume check failed: Something weird')
celery_logger.debug("check_volume task finished at %s." % datetime.now())
except LibcloudDeploymentError as exc:
celery_logger.exception(exc)
except Exception as exc:
celery_logger.warn(exc)
check_volume_task.retry(exc=exc)
def check_volume_task(
driverCls,
provider,
identity,
instance_id,
volume_id,
device_type='ext4',
*args,
**kwargs
):
try:
celery_logger.debug("check_volume task started at %s." % timezone.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
volume = driver.get_volume(volume_id)
username = identity.get_username()
attach_data = volume.extra['attachments'][0]
device_location = attach_data['device']
celery_logger.info("device_location: %s" % device_location)
# One playbook to make two checks:
# 1. Voume exists
# 2. Volume has a filesystem
# (If not, create one of type 'device_type')
playbook_results = deploy_check_volume(
instance.ip,
username,
instance.id,
device_location,
device_type=device_type
)
success = not (
execution_has_failures(playbook_results)
or execution_has_unreachable(playbook_results)
)
if not success:
raise Exception(
"Error encountered while checking volume for filesystem: instance_id: {}, volume_id: {}"
.format(instance_id, volume_id)
)
return success
except Exception as exc:
celery_logger.warn(exc)
check_volume_task.retry(exc=exc)
def make_machines_public(application, account_drivers={}, dry_run=False):
"""
This method is called when the DB has marked the Machine/Application as PRIVATE
But the CLOUD states that the machine is really public.
"""
for version in application.active_versions():
for machine in version.active_machines():
provider = machine.instance_source.provider
account_driver = memoized_driver(machine, account_drivers)
image = account_driver.image_manager.get_image(image_id=machine.identifier)
if image and image.is_public == False:
celery_logger.info("Making Machine %s public" % image.id)
if not dry_run:
image.update(is_public=True)
# Set top-level application to public (This will make all versions and PMs public too!)
application.private = False
celery_logger.info("Making Application %s public" % application.name)
if not dry_run:
application.save()
def set_machine_request_metadata(machine_request, image_id):
admin_driver = get_admin_driver(machine_request.new_machine_provider)
machine = admin_driver.get_machine(image_id)
lc_driver = admin_driver._connection
if not machine:
celery_logger.warn("Could not find machine with ID=%s" % image_id)
return
if not hasattr(lc_driver, 'ex_set_image_metadata'):
return
metadata = lc_driver.ex_get_image_metadata(machine)
if machine_request.new_application_description:
metadata['description'] = machine_request.new_application_description
if machine_request.new_version_tags:
metadata['tags'] = machine_request.new_version_tags
celery_logger.info("LC Driver:%s - Machine:%s - Metadata:%s" %
(lc_driver, machine.id, metadata))
lc_driver.ex_set_image_metadata(machine, metadata)
return machine
def set_machine_request_metadata(machine_request, image_id):
admin_driver = get_admin_driver(machine_request.new_machine_provider)
machine = admin_driver.get_machine(image_id)
lc_driver = admin_driver._connection
if not machine:
celery_logger.warn("Could not find machine with ID=%s" % image_id)
return
if not hasattr(lc_driver, 'ex_set_image_metadata'):
return
metadata = lc_driver.ex_get_image_metadata(machine)
if machine_request.new_application_description:
metadata['description'] = machine_request.new_application_description
if machine_request.new_version_tags:
metadata['tags'] = machine_request.new_version_tags
celery_logger.info("LC Driver:%s - Machine:%s - Metadata:%s"
% (lc_driver, machine.id, metadata))
lc_driver.ex_set_image_metadata(machine, metadata)
return machine
def check_volume_task(driverCls,
provider,
identity,
instance_id,
volume_id,
device_type='ext4',
*args,
**kwargs):
try:
celery_logger.debug("check_volume task started at %s." %
datetime.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
volume = driver.get_volume(volume_id)
username = identity.get_username()
attach_data = volume.extra['attachments'][0]
device_location = attach_data['device']
celery_logger.info("device_location: %s" % device_location)
# One playbook to make two checks:
# 1. Voume exists
# 2. Volume has a filesystem
# (If not, create one of type 'device_type')
playbooks = deploy_check_volume(instance.ip,
username,
instance.id,
device_location,
device_type=device_type)
celery_logger.info(playbooks.__dict__)
hostname = build_host_name(instance.id, instance.ip)
result = False if execution_has_failures(playbooks, hostname)\
or execution_has_unreachable(playbooks, hostname) else True
if not result:
raise Exception(
"Error encountered while checking volume for filesystem: %s" %
playbooks.stats.summarize(host=hostname))
return result
except LibcloudDeploymentError as exc:
celery_logger.exception(exc)
except Exception as exc:
celery_logger.warn(exc)
check_volume_task.retry(exc=exc)
def make_machines_public(application, account_drivers={}, dry_run=False):
"""
This method is called when the DB has marked the Machine/Application as PRIVATE
But the CLOUD states that the machine is really public.
"""
for version in application.active_versions():
for machine in version.active_machines():
provider = machine.instance_source.provider
account_driver = memoized_driver(machine, account_drivers)
image = account_driver.image_manager.get_image(
image_id=machine.identifier)
if image and image.is_public == False:
celery_logger.info("Making Machine %s public" % image.id)
if not dry_run:
image.update(is_public=True)
# Set top-level application to public (This will make all versions and PMs public too!)
application.private = False
celery_logger.info("Making Application %s public" % application.name)
if not dry_run:
application.save()
def machine_request_error(task_request, *args, **kwargs):
#Args format: (exception, ?, subtask_args...)
exception = args[0]
machine_request_id = args[2]
task_uuid = task_request.id
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % (task_uuid,) )
celery_logger.info("exception=%s" % (exception,) )
celery_logger.info("task_kwargs=%s" % kwargs)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = _status_to_error(machine_request.old_status, result.result, result.traceback)
celery_logger.info("traceback=%s" % (result.traceback,) )
celery_logger.error(err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
send_image_request_failed_email(machine_request, err_str)
def make_machines_public(application, account_drivers={}, dry_run=False):
"""
This method is called when the DB has marked the Machine/Application as PRIVATE
But the CLOUD states that the machine is really public.
"""
for version in application.active_versions():
for machine in version.active_machines():
provider = machine.instance_source.provider
account_driver = memoized_driver(machine, account_drivers)
try:
image = account_driver.image_manager.get_image(
image_id=machine.identifier)
except: # Image not found
celery_logger.info("Image not found on this provider: %s" %
(machine))
continue
image_is_public = image.is_public if hasattr(
image, 'is_public') else image.get('visibility',
'') == 'public'
if image and image_is_public == False:
celery_logger.info("Making Machine %s public" % image.id)
if not dry_run:
account_driver.image_manager.glance.images.update(
image.id, visibility='public')
# Set top-level application to public (This will make all versions and PMs public too!)
application.private = False
celery_logger.info("Making Application %s:%s public" %
(application.id, application.name))
if not dry_run:
application.save()
def update_snapshot_cyverse(start_date=None, end_date=None):
logger.debug("update_snapshot_cyverse task started at %s." % datetime.now())
end_date = timezone.now().replace(microsecond=0) if not end_date else end_date
for allocation_source in AllocationSource.objects.order_by('name'):
# calculate and save snapshots here
allocation_source_name = allocation_source.name
last_renewal_event = EventTable.objects.filter(
name='allocation_source_created_or_renewed',
payload__allocation_source_name__exact=str(allocation_source_name)).order_by('timestamp')
if not last_renewal_event:
logger.info('Allocation Source %s Create/Renewal event missing', allocation_source_name)
continue
start_date = last_renewal_event.last().timestamp.replace(microsecond=0) if not start_date else start_date
total_compute_used = 0
total_burn_rate = 0
for user in allocation_source.all_users:
compute_used, burn_rate = total_usage(user.username, start_date=start_date,
end_date=end_date, allocation_source_name=allocation_source_name,
burn_rate=True)
UserAllocationSnapshot.objects.update_or_create(allocation_source=allocation_source, user=user,
defaults={'compute_used': compute_used,
'burn_rate': burn_rate})
total_compute_used += compute_used
total_burn_rate += burn_rate
AllocationSourceSnapshot.objects.update_or_create(allocation_source=allocation_source,
defaults={'compute_used': total_compute_used,
'global_burn_rate': total_burn_rate})
run_all(rule_list=cyverse_rules,
defined_variables=CyverseTestRenewalVariables(allocation_source, current_time=end_date,
last_renewal_event_date=start_date),
defined_actions=CyverseTestRenewalActions(allocation_source, current_time=end_date))
# At the end of the task, fire-off an allocation threshold check
logger.debug("update_snapshot_cyverse task finished at %s." % datetime.now())
allocation_threshold_check.apply_async()
def make_machines_public(application, account_drivers={}, dry_run=False):
"""
This method is called when the DB has marked the Machine/Application as PRIVATE
But the CLOUD states that the machine is really public.
"""
for version in application.active_versions():
for machine in version.active_machines():
provider = machine.instance_source.provider
account_driver = memoized_driver(machine, account_drivers)
try:
image = account_driver.image_manager.get_image(image_id=machine.identifier)
except: # Image not found
celery_logger.info("Image not found on this provider: %s" % (machine))
continue
image_is_public = image.is_public if hasattr(image,'is_public') else image.get('visibility','') == 'public'
if image and image_is_public == False:
celery_logger.info("Making Machine %s public" % image.id)
if not dry_run:
account_driver.image_manager.glance.images.update(image.id, visibility='public')
# Set top-level application to public (This will make all versions and PMs public too!)
application.private = False
celery_logger.info("Making Application %s:%s public" % (application.id,application.name))
if not dry_run:
application.save()
def machine_request_error(task_request, *args, **kwargs):
#Args format: (exception, ?, subtask_args...)
exception = args[0]
machine_request_id = args[2]
task_uuid = task_request.id
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % (task_uuid, ))
celery_logger.info("exception=%s" % (exception, ))
celery_logger.info("task_kwargs=%s" % kwargs)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = _status_to_error(machine_request.old_status, result.result,
result.traceback)
celery_logger.info("traceback=%s" % (result.traceback, ))
celery_logger.error(err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
send_image_request_failed_email(machine_request, err_str)
def mount_failed(task_uuid, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args):
from service import volume as volume_service
try:
celery_logger.debug("mount_failed task started at %s." % datetime.now())
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "Mount Error Traceback:%s" % (result.traceback,)
celery_logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = "umount_error"
else:
tmp_status = "mount_error"
return volume_service.update_volume_metadata(driver, volume, metadata={"tmp_status": tmp_status})
celery_logger.debug("mount_failed task finished at %s." % datetime.now())
except Exception as exc:
celery_logger.warn(exc)
mount_failed.retry(exc=exc)
def remove_machine(db_machine, now_time=None, dry_run=False):
"""
End date the DB ProviderMachine
If all PMs are end-dated, End date the ApplicationVersion
if all Versions are end-dated, End date the Application
"""
if not now_time:
now_time = timezone.now()
db_machine.end_date = now_time
celery_logger.info("End dating machine: %s" % db_machine)
if not dry_run:
db_machine.save()
db_version = db_machine.application_version
if db_version.machines.filter(
# Look and see if all machines are end-dated.
Q(instance_source__end_date__isnull=True) |
Q(instance_source__end_date__gt=now_time)
).count() != 0:
# Other machines exist.. No cascade necessary.
return True
# Version also completely end-dated. End date this version.
db_version.end_date = now_time
celery_logger.info("End dating version: %s" % db_version)
if not dry_run:
db_version.save()
db_application = db_version.application
if db_application.versions.filter(
# If all versions are end-dated
only_current(now_time)
).count() != 0:
# Other versions exist.. No cascade necessary..
return True
db_application.end_date = now_time
celery_logger.info("End dating application: %s" % db_application)
if not dry_run:
db_application.save()
return True
def prune_machines_for(
provider_id,
print_logs=False,
dry_run=False,
forced_removal=False,
validate=True
):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
console_handler = _init_stdout_logging()
celery_logger.info(
"Starting prune_machines for Provider %s @ %s" % (provider, now)
)
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider
)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider
)
cloud_machines = []
machine_validator = MachineValidationPluginManager.get_validator(
account_driver
)
cloud_machines = [
cloud_machine for cloud_machine in cloud_machines
if not validate or machine_validator.machine_is_valid(cloud_machine)
]
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(
db_machines, cloud_machines, now=now, dry_run=dry_run
)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
# Clear out application, provider machine, and version memberships
# if the result is >128.
# Additionally, remove all users who are not in the machine request (if one exists).
_clean_memberships(db_machines, account_driver)
celery_logger.info(
"prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned." %
(provider, app_count, ver_count, mach_count)
)
if print_logs:
_exit_stdout_logging(console_handler)
def mount_task(driverCls, provider, identity, instance_id, volume_id,
device=None, mount_location=None, *args, **kwargs):
try:
celery_logger.debug("mount task started at %s." % datetime.now())
celery_logger.debug("mount_location: %s" % (mount_location, ))
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
volume = driver.get_volume(volume_id)
username = identity.get_username()
# DEV NOTE: Set as 'users' because this is a GUARANTEED group
# and we know our 'user' will exist (if atmo_init_full was executed)
# in case the VM does NOT rely on iPlant LDAP
groupname = "users"
celery_logger.debug(volume)
try:
attach_data = volume.extra['attachments'][0]
if not device:
device = attach_data['device']
except KeyError as IndexError:
celery_logger.warn("Volume %s missing attachments in Extra"
% (volume,))
device = None
if not device:
celery_logger.warn("Device never attached. Nothing to mount")
return None
private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa"
kwargs.update({'ssh_key': private_key})
kwargs.update({'timeout': 120})
# Step 2. Check the volume is not already mounted
cm_script = check_mount()
kwargs.update({'deploy': cm_script})
driver.deploy_to(instance, **kwargs)
if device in cm_script.stdout:
mount_location = _parse_mount_location(cm_script.stdout, device)
if not mount_location:
raise Exception("Device already mounted, "
"but mount location could not be determined!"
"Check _parse_mount_location()!")
celery_logger.warn(
"Device already mounted. Mount output:%s" %
cm_script.stdout)
# Device has already been mounted. Move along..
return mount_location
# Step 3. Find a suitable location to mount the volume
celery_logger.info("Original mount location - %s" % mount_location)
if not mount_location:
inc = 1
while True:
if '/vol%s' % inc in cm_script.stdout:
inc += 1
else:
break
mount_location = '/vol%s' % inc
celery_logger.info("Device location - %s" % device)
celery_logger.info("New mount location - %s" % mount_location)
mv_script = mount_volume(device, mount_location, username, groupname)
kwargs.update({'deploy': mv_script})
driver.deploy_to(instance, **kwargs)
celery_logger.debug("mount task finished at %s." % datetime.now())
return mount_location
except Exception as exc:
celery_logger.warn(exc)
mount_task.retry(exc=exc)
def start_machine_imaging(machine_request, delay=False):
"""
Builds up a machine imaging task using core.models.machine_request
delay - If true, wait until task is completed before returning
"""
new_status, _ = StatusType.objects.get_or_create(name="started")
machine_request.status = new_status
machine_request.save()
original_status = machine_request.old_status
last_run_error, original_status = _recover_from_error(original_status)
if last_run_error:
machine_request.old_status = original_status
machine_request.save()
instance_id = machine_request.instance.provider_alias
(orig_managerCls, orig_creds,
dest_managerCls, dest_creds) = machine_request.prepare_manager()
imaging_args = machine_request.get_imaging_args()
admin_driver = get_admin_driver(machine_request.new_machine_provider)
admin_ident = machine_request.new_admin_identity()
imaging_error_task = machine_request_error.s(machine_request.id)
# Task 2 = Imaging w/ Chromogenic
imaging_task = _get_imaging_task(orig_managerCls, orig_creds,
dest_managerCls, dest_creds,
imaging_args)
imaging_task.link_error(imaging_error_task)
# Assume we are starting from the beginning.
init_task = imaging_task
# Task 2 = Process the machine request
if 'processing' in original_status:
# If processing, start here..
image_id = original_status.replace("processing - ", "")
logger.info("Start with processing:%s" % image_id)
process_task = process_request.s(image_id, machine_request.id)
init_task = process_task
else:
# Link from imaging to process..
process_task = process_request.s(machine_request.id)
imaging_task.link(process_task)
process_task.link_error(imaging_error_task)
# Task 3 = Validate the new image by launching an instance
if 'validating' in original_status:
image_id = machine_request.new_machine.identifier
celery_logger.info("Start with validating:%s" % image_id)
# If validating, seed the image_id and start here..
validate_task = validate_new_image.s(image_id, machine_request.id)
init_task = validate_task
else:
validate_task = validate_new_image.s(machine_request.id)
process_task.link(validate_task)
# Task 4 = Wait for new instance to be 'active'
wait_for_task = wait_for_instance.s(
# NOTE: 1st arg, instance_id, passed from last task.
admin_driver.__class__,
admin_driver.provider,
admin_driver.identity,
"active",
return_id=True)
validate_task.link(wait_for_task)
validate_task.link_error(imaging_error_task)
# Task 5 = Terminate the new instance on completion
destroy_task = destroy_instance.s(
admin_ident.uuid)
wait_for_task.link(destroy_task)
wait_for_task.link_error(imaging_error_task)
# Task 6 - Finally, email the user that their image is ready!
# NOTE: si == Ignore the result of the last task.
email_task = imaging_complete.si(machine_request.id)
destroy_task.link_error(imaging_error_task)
destroy_task.link(email_task)
email_task.link_error(imaging_error_task)
# Set status to imaging ONLY if our initial task is the imaging task.
if init_task == imaging_task:
machine_request.old_status = 'imaging'
machine_request.save()
# Start the task.
async = init_task.apply_async()
if delay:
async.get()
return async
