def mount_failed( context, exception_msg, traceback, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args ): from service import volume as volume_service try: celery_logger.debug("mount_failed task started at %s." % timezone.now()) celery_logger.info("task context=%s" % context) err_str = "%s\nMount Error Traceback:%s" % (exception_msg, traceback) celery_logger.error(err_str) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_id) if unmount: tmp_status = 'umount_error' else: tmp_status = 'mount_error' return volume_service._update_volume_metadata( driver, volume, metadata={'tmp_status': tmp_status} ) except Exception as exc: celery_logger.warn(exc) mount_failed.retry(exc=exc)
def mount_failed(task_uuid, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args): from service import volume as volume_service try: celery_logger.debug("mount_failed task started at %s." % datetime.now()) celery_logger.info("task_uuid=%s" % task_uuid) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = "Mount Error Traceback:%s" % (result.traceback, ) celery_logger.error(err_str) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_id) if unmount: tmp_status = 'umount_error' else: tmp_status = 'mount_error' return volume_service._update_volume_metadata( driver, volume, metadata={'tmp_status': tmp_status}) celery_logger.debug("mount_failed task finished at %s." % datetime.now()) except Exception as exc: celery_logger.warn(exc) mount_failed.retry(exc=exc)
def export_request_error(task_uuid, export_request_id): celery_logger.info("export_request_id=%s" % export_request_id) celery_logger.info("task_uuid=%s" % task_uuid) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback,) celery_logger.error(err_str) export_request = ExportRequest.objects.get(id=export_request_id) export_request.status = err_str export_request.save()
def machine_request_error(task_uuid, machine_request_id): celery_logger.info("machine_request_id=%s" % machine_request_id) celery_logger.info("task_uuid=%s" % task_uuid) machine_request = MachineRequest.objects.get(id=machine_request_id) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = _status_to_error(machine_request.old_status, result.result, result.traceback) celery_logger.error(err_str) send_image_request_failed_email(machine_request, err_str) machine_request = MachineRequest.objects.get(id=machine_request_id) machine_request.old_status = err_str machine_request.save()
def machine_request_error(task_uuid, machine_request_id): celery_logger.info("machine_request_id=%s" % machine_request_id) celery_logger.info("task_uuid=%s" % task_uuid) machine_request = MachineRequest.objects.get(id=machine_request_id) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = "(%s) ERROR - %r Exception:%r" % (machine_request.old_status, result.result, result.traceback, ) celery_logger.error(err_str) send_image_request_failed_email(machine_request, err_str) machine_request = MachineRequest.objects.get(id=machine_request_id) machine_request.old_status = err_str machine_request.save()
def send_email(subject, body, from_email, to, cc=None, fail_silently=False, html=False): """ Use django.core.mail.EmailMessage to send and log an Atmosphere email. """ try: msg = EmailMessage(subject=subject, body=body, from_email=from_email, to=to, cc=cc) if html: msg.content_subtype = 'html' msg.send(fail_silently=fail_silently) args = (from_email, to, cc, subject, body) email_logger.info(log_message.format(*args)) return True except Exception as e: celery_logger.error(e) return False
def machine_request_error(task_request, *args, **kwargs): #Args format: (exception, ?, subtask_args...) exception = args[0] machine_request_id = args[2] task_uuid = task_request.id celery_logger.info("machine_request_id=%s" % machine_request_id) celery_logger.info("task_uuid=%s" % (task_uuid,) ) celery_logger.info("exception=%s" % (exception,) ) celery_logger.info("task_kwargs=%s" % kwargs) machine_request = MachineRequest.objects.get(id=machine_request_id) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = _status_to_error(machine_request.old_status, result.result, result.traceback) celery_logger.error(err_str) send_image_request_failed_email(machine_request, err_str) machine_request = MachineRequest.objects.get(id=machine_request_id) machine_request.old_status = err_str machine_request.save()
def machine_request_error(task_request, *args, **kwargs): #Args format: (exception, ?, subtask_args...) exception = args[0] machine_request_id = args[2] task_uuid = task_request.id celery_logger.info("machine_request_id=%s" % machine_request_id) celery_logger.info("task_uuid=%s" % (task_uuid,) ) celery_logger.info("exception=%s" % (exception,) ) celery_logger.info("task_kwargs=%s" % kwargs) machine_request = MachineRequest.objects.get(id=machine_request_id) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = _status_to_error(machine_request.old_status, result.result, result.traceback) celery_logger.info("traceback=%s" % (result.traceback,) ) celery_logger.error(err_str) machine_request = MachineRequest.objects.get(id=machine_request_id) machine_request.old_status = err_str machine_request.save() send_image_request_failed_email(machine_request, err_str)
def mount_failed(task_uuid, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args): from service import volume as volume_service try: celery_logger.debug("mount_failed task started at %s." % datetime.now()) celery_logger.info("task_uuid=%s" % task_uuid) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = "Mount Error Traceback:%s" % (result.traceback,) celery_logger.error(err_str) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_id) if unmount: tmp_status = "umount_error" else: tmp_status = "mount_error" return volume_service.update_volume_metadata(driver, volume, metadata={"tmp_status": tmp_status}) celery_logger.debug("mount_failed task finished at %s." % datetime.now()) except Exception as exc: celery_logger.warn(exc) mount_failed.retry(exc=exc)
def monitor_machines_for(provider_id, print_logs=False, dry_run=False): """ Run the set of tasks related to monitoring machines for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. NEW LOGIC: * Membership and Privacy is dictated at the APPLICATION level. * loop over all machines on the cloud * * If machine is PUBLIC, ensure the APP is public. * * If machine is PRIVATE, ensure the APP is private && sync the membership! * * Ignore the possibility of conflicts, prior schema should be sufficient for ensuring the above two usecases """ provider = Provider.objects.get(id=provider_id) if print_logs: console_handler = _init_stdout_logging() #STEP 1: get the apps new_public_apps, private_apps = get_public_and_private_apps(provider) #STEP 2: Find conflicts and report them. intersection = set(private_apps.keys()) & set(new_public_apps) if intersection: celery_logger.error("These applications were listed as BOTH public && private apps. Manual conflict correction required: %s" % intersection) #STEP 3: Apply the changes at app-level #Memoization at this high of a level will help save time account_drivers = {} # Provider -> accountDriver provider_tenant_mapping = {} # Provider -> [{TenantId : TenantName},...] image_maps = {} if settings.ENFORCING: for app in new_public_apps: if app in intersection: celery_logger.error("Skipped public app: %s <%s>" % (app, app.id)) continue make_machines_public(app, account_drivers, dry_run=dry_run) for app, membership in private_apps.items(): if app in intersection: celery_logger.error("Skipped private app: %s <%s>" % (app, app.id)) continue make_machines_private(app, membership, account_drivers, provider_tenant_mapping, image_maps, dry_run=dry_run) else: # settings.ENFORCING = False celery_logger.warn("Settings.ENFORCING is set to False -- So we assume this is a development build and *NO* changes should be made to glance as a result of an 'information mismatch'") if print_logs: _exit_stdout_logging(console_handler) return
def monitor_machines_for(provider_id, print_logs=False, dry_run=False): """ Run the set of tasks related to monitoring machines for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. NEW LOGIC: * Membership and Privacy is dictated at the APPLICATION level. * loop over all machines on the cloud * * If machine is PUBLIC, ensure the APP is public. * * If machine is PRIVATE, ensure the APP is private && sync the membership! * * Ignore the possibility of conflicts, prior schema should be sufficient for ensuring the above two usecases """ provider = Provider.objects.get(id=provider_id) if print_logs: console_handler = _init_stdout_logging() #STEP 1: get the apps new_public_apps, private_apps = get_public_and_private_apps(provider) #STEP 2: Find conflicts and report them. intersection = set(private_apps.keys()) & set(new_public_apps) if intersection: celery_logger.error( "These applications were listed as BOTH public && private apps. Manual conflict correction required: %s" % intersection) #STEP 3: Apply the changes at app-level #Memoization at this high of a level will help save time account_drivers = {} # Provider -> accountDriver provider_tenant_mapping = {} # Provider -> [{TenantId : TenantName},...] image_maps = {} if settings.ENFORCING: for app in new_public_apps: if app in intersection: celery_logger.error("Skipped public app: %s <%s>" % (app, app.id)) continue make_machines_public(app, account_drivers, dry_run=dry_run) for app, membership in private_apps.items(): if app in intersection: celery_logger.error("Skipped private app: %s <%s>" % (app, app.id)) continue make_machines_private(app, membership, account_drivers, provider_tenant_mapping, image_maps, dry_run=dry_run) else: # settings.ENFORCING = False celery_logger.warn( "Settings.ENFORCING is set to False -- So we assume this is a development build and *NO* changes should be made to glance as a result of an 'information mismatch'" ) if print_logs: _exit_stdout_logging(console_handler) return