def mount_failed(
context,
exception_msg,
traceback,
driverCls,
provider,
identity,
volume_id,
unmount=False,
**celery_task_args
):
from service import volume as volume_service
try:
celery_logger.debug("mount_failed task started at %s." % timezone.now())
celery_logger.info("task context=%s" % context)
err_str = "%s\nMount Error Traceback:%s" % (exception_msg, traceback)
celery_logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = 'umount_error'
else:
tmp_status = 'mount_error'
return volume_service._update_volume_metadata(
driver, volume, metadata={'tmp_status': tmp_status}
)
except Exception as exc:
celery_logger.warn(exc)
mount_failed.retry(exc=exc)
def mount_failed(task_uuid,
driverCls,
provider,
identity,
volume_id,
unmount=False,
**celery_task_args):
from service import volume as volume_service
try:
celery_logger.debug("mount_failed task started at %s." %
datetime.now())
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "Mount Error Traceback:%s" % (result.traceback, )
celery_logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = 'umount_error'
else:
tmp_status = 'mount_error'
return volume_service._update_volume_metadata(
driver, volume, metadata={'tmp_status': tmp_status})
celery_logger.debug("mount_failed task finished at %s." %
datetime.now())
except Exception as exc:
celery_logger.warn(exc)
mount_failed.retry(exc=exc)
def export_request_error(task_uuid, export_request_id):
celery_logger.info("export_request_id=%s" % export_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback,)
celery_logger.error(err_str)
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = err_str
export_request.save()
def export_request_error(task_uuid, export_request_id):
celery_logger.info("export_request_id=%s" % export_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback,)
celery_logger.error(err_str)
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = err_str
export_request.save()
def machine_request_error(task_uuid, machine_request_id):
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = _status_to_error(machine_request.old_status, result.result,
result.traceback)
celery_logger.error(err_str)
send_image_request_failed_email(machine_request, err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
def machine_request_error(task_uuid, machine_request_id):
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "(%s) ERROR - %r Exception:%r" % (machine_request.old_status,
result.result,
result.traceback,
)
celery_logger.error(err_str)
send_image_request_failed_email(machine_request, err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
def send_email(subject, body, from_email, to, cc=None,
fail_silently=False, html=False):
"""
Use django.core.mail.EmailMessage to send and log an Atmosphere email.
"""
try:
msg = EmailMessage(subject=subject, body=body,
from_email=from_email,
to=to,
cc=cc)
if html:
msg.content_subtype = 'html'
msg.send(fail_silently=fail_silently)
args = (from_email, to, cc, subject, body)
email_logger.info(log_message.format(*args))
return True
except Exception as e:
celery_logger.error(e)
return False
def machine_request_error(task_request, *args, **kwargs):
#Args format: (exception, ?, subtask_args...)
exception = args[0]
machine_request_id = args[2]
task_uuid = task_request.id
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % (task_uuid,) )
celery_logger.info("exception=%s" % (exception,) )
celery_logger.info("task_kwargs=%s" % kwargs)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = _status_to_error(machine_request.old_status, result.result, result.traceback)
celery_logger.error(err_str)
send_image_request_failed_email(machine_request, err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
def machine_request_error(task_request, *args, **kwargs):
#Args format: (exception, ?, subtask_args...)
exception = args[0]
machine_request_id = args[2]
task_uuid = task_request.id
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % (task_uuid,) )
celery_logger.info("exception=%s" % (exception,) )
celery_logger.info("task_kwargs=%s" % kwargs)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = _status_to_error(machine_request.old_status, result.result, result.traceback)
celery_logger.info("traceback=%s" % (result.traceback,) )
celery_logger.error(err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
send_image_request_failed_email(machine_request, err_str)
def mount_failed(task_uuid, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args):
from service import volume as volume_service
try:
celery_logger.debug("mount_failed task started at %s." % datetime.now())
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "Mount Error Traceback:%s" % (result.traceback,)
celery_logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = "umount_error"
else:
tmp_status = "mount_error"
return volume_service.update_volume_metadata(driver, volume, metadata={"tmp_status": tmp_status})
celery_logger.debug("mount_failed task finished at %s." % datetime.now())
except Exception as exc:
celery_logger.warn(exc)
mount_failed.retry(exc=exc)
def monitor_machines_for(provider_id, print_logs=False, dry_run=False):
"""
Run the set of tasks related to monitoring machines for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
NEW LOGIC:
* Membership and Privacy is dictated at the APPLICATION level.
* loop over all machines on the cloud
* * If machine is PUBLIC, ensure the APP is public.
* * If machine is PRIVATE, ensure the APP is private && sync the membership!
* * Ignore the possibility of conflicts, prior schema should be sufficient for ensuring the above two usecases
"""
provider = Provider.objects.get(id=provider_id)
if print_logs:
console_handler = _init_stdout_logging()
#STEP 1: get the apps
new_public_apps, private_apps = get_public_and_private_apps(provider)
#STEP 2: Find conflicts and report them.
intersection = set(private_apps.keys()) & set(new_public_apps)
if intersection:
celery_logger.error("These applications were listed as BOTH public && private apps. Manual conflict correction required: %s" % intersection)
#STEP 3: Apply the changes at app-level
#Memoization at this high of a level will help save time
account_drivers = {} # Provider -> accountDriver
provider_tenant_mapping = {} # Provider -> [{TenantId : TenantName},...]
image_maps = {}
if settings.ENFORCING:
for app in new_public_apps:
if app in intersection:
celery_logger.error("Skipped public app: %s <%s>" % (app, app.id))
continue
make_machines_public(app, account_drivers, dry_run=dry_run)
for app, membership in private_apps.items():
if app in intersection:
celery_logger.error("Skipped private app: %s <%s>" % (app, app.id))
continue
make_machines_private(app, membership, account_drivers, provider_tenant_mapping, image_maps, dry_run=dry_run)
else: # settings.ENFORCING = False
celery_logger.warn("Settings.ENFORCING is set to False -- So we assume this is a development build and *NO* changes should be made to glance as a result of an 'information mismatch'")
if print_logs:
_exit_stdout_logging(console_handler)
return
def monitor_machines_for(provider_id, print_logs=False, dry_run=False):
"""
Run the set of tasks related to monitoring machines for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
NEW LOGIC:
* Membership and Privacy is dictated at the APPLICATION level.
* loop over all machines on the cloud
* * If machine is PUBLIC, ensure the APP is public.
* * If machine is PRIVATE, ensure the APP is private && sync the membership!
* * Ignore the possibility of conflicts, prior schema should be sufficient for ensuring the above two usecases
"""
provider = Provider.objects.get(id=provider_id)
if print_logs:
console_handler = _init_stdout_logging()
#STEP 1: get the apps
new_public_apps, private_apps = get_public_and_private_apps(provider)
#STEP 2: Find conflicts and report them.
intersection = set(private_apps.keys()) & set(new_public_apps)
if intersection:
celery_logger.error(
"These applications were listed as BOTH public && private apps. Manual conflict correction required: %s"
% intersection)
#STEP 3: Apply the changes at app-level
#Memoization at this high of a level will help save time
account_drivers = {} # Provider -> accountDriver
provider_tenant_mapping = {} # Provider -> [{TenantId : TenantName},...]
image_maps = {}
if settings.ENFORCING:
for app in new_public_apps:
if app in intersection:
celery_logger.error("Skipped public app: %s <%s>" %
(app, app.id))
continue
make_machines_public(app, account_drivers, dry_run=dry_run)
for app, membership in private_apps.items():
if app in intersection:
celery_logger.error("Skipped private app: %s <%s>" %
(app, app.id))
continue
make_machines_private(app,
membership,
account_drivers,
provider_tenant_mapping,
image_maps,
dry_run=dry_run)
else: # settings.ENFORCING = False
celery_logger.warn(
"Settings.ENFORCING is set to False -- So we assume this is a development build and *NO* changes should be made to glance as a result of an 'information mismatch'"
)
if print_logs:
_exit_stdout_logging(console_handler)
return