def send_email(subject, body, from_email, to, cc=None, fail_silently=False, html=False): """ Use django.core.mail.EmailMessage to send and log an Atmosphere email. """ try: msg = EmailMessage(subject=subject, body=body, from_email=from_email, to=to, cc=cc) if html: msg.content_subtype = 'html' email_logger.info("\n> From:%s\n> To:%s\n> Cc:%s\n> Subject:%s\n> Body:\n%s", from_email, to, cc, subject, body) if getattr(settings, "SEND_EMAILS", True): msg.send(fail_silently=fail_silently) email_logger.info("NOTE: Above message sent successfully") celery_logger.info("NOTE: Above message sent successfully") else: email_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False") celery_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False") return True except Exception as e: celery_logger.exception(e) return False
def remove_empty_networks_for(provider_id): provider = Provider.objects.get(id=provider_id) os_driver = get_account_driver(provider) all_instances = os_driver.admin_driver.list_all_instances() project_map = os_driver.network_manager.project_network_map() projects_with_networks = project_map.keys() for project in projects_with_networks: networks = project_map[project]['network'] if not isinstance(networks, list): networks = [networks] for network in networks: network_name = network['name'] celery_logger.debug("Checking if network %s is in use" % network_name) if running_instances(network_name, all_instances): continue # TODO: MUST change when not using 'usergroups' explicitly. user = project try: celery_logger.debug("Removing project network for User:%s, Project:%s" % (user, project)) os_driver.network_manager.delete_project_network(user, project) except NeutronClientException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project)) except NeutronException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project))
def send_email(subject, body, from_email, to, cc=None, html=False): """ Use django.core.mail.EmailMessage to send and log an Atmosphere email. """ try: msg = EmailMessage(subject=subject, body=body, from_email=from_email, to=to, cc=cc) if html: msg.content_subtype = 'html' email_logger.info( "\n> From:%s\n> To:%s\n> Cc:%s\n> Subject:%s\n> Body:\n%s", from_email, to, cc, subject, body) if getattr(settings, "SEND_EMAILS", True): msg.send() email_logger.info("NOTE: Above message sent successfully") celery_logger.info("NOTE: Above message sent successfully") else: email_logger.info( "NOTE: Above message not sent -- SEND_EMAILS was False") celery_logger.info( "NOTE: Above message not sent -- SEND_EMAILS was False") return True except Exception as e: celery_logger.exception(e) return False
def update_mount_location(new_mount_location, driverCls, provider, identity, volume_alias): """ """ from service import volume as volume_service try: celery_logger.debug( "update_mount_location task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_alias) if not volume: return if not new_mount_location: return volume_metadata = volume.extra['metadata'] return volume_service._update_volume_metadata( driver, volume, metadata={'mount_location': new_mount_location}) celery_logger.debug( "update_mount_location task finished at %s." % datetime.now()) except Exception as exc: celery_logger.exception(exc) update_mount_location.retry(exc=exc)
def update_mount_location(new_mount_location, driverCls, provider, identity, volume_alias): """ """ from service import volume as volume_service try: celery_logger.debug( "update_mount_location task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_alias) if not volume: return if not new_mount_location: return volume_metadata = volume.extra['metadata'] return volume_service._update_volume_metadata( driver, volume, metadata={'mount_location': new_mount_location}) celery_logger.debug( "update_mount_location task finished at %s." % datetime.now()) except Exception as exc: celery_logger.exception(exc) update_mount_location.retry(exc=exc)
def remove_empty_networks_for(provider_id): provider = Provider.objects.get(id=provider_id) os_driver = get_account_driver(provider) all_instances = os_driver.admin_driver.list_all_instances() project_map = os_driver.network_manager.project_network_map() projects_with_networks = project_map.keys() for project in projects_with_networks: networks = project_map[project]['network'] if not isinstance(networks, list): networks = [networks] for network in networks: network_name = network['name'] celery_logger.debug("Checking if network %s is in use" % network_name) if running_instances(network_name, all_instances): continue # TODO: MUST change when not using 'usergroups' explicitly. user = project try: celery_logger.debug( "Removing project network for User:%s, Project:%s" % (user, project)) os_driver.network_manager.delete_project_network(user, project) except NeutronClientException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project)) except NeutronException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project))
def send_email(subject, body, from_email, to, cc=None, fail_silently=False, html=False): """ Use django.core.mail.EmailMessage to send and log an Atmosphere email. """ try: msg = EmailMessage(subject=subject, body=body, from_email=from_email, to=to, cc=cc) if html: msg.content_subtype = 'html' log_message = "\n> From:{0}\n> To:{1}\n> Cc:{2}\n> Subject:{3}\n> Body:\n{4}" args = (from_email, to, cc, subject, body) email_logger.info(log_message.format(*args)) msg.send(fail_silently=fail_silently) return True except Exception as e: celery_logger.exception(e) return False
def send_email(subject, body, from_email, to, cc=None, fail_silently=False, html=False): """ Use django.core.mail.EmailMessage to send and log an Atmosphere email. """ try: msg = EmailMessage(subject=subject, body=body, from_email=from_email, to=to, cc=cc) if html: msg.content_subtype = 'html' log_message = "\n> From:{0}\n> To:{1}\n> Cc:{2}\n> Subject:{3}\n> Body:\n{4}" args = (from_email, to, cc, subject, body) email_logger.info(log_message.format(*args)) if getattr(settings, "SEND_EMAILS", True): msg.send(fail_silently=fail_silently) email_logger.info("NOTE: Above message sent successfully") celery_logger.info("NOTE: Above message sent successfully") else: email_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False") celery_logger.info("NOTE: Above message not sent -- SEND_EMAILS was False") return True except Exception as e: celery_logger.exception(e) return False
def monitor_instances_for(provider_id, users=None, print_logs=False, check_allocations=False, start_date=None, end_date=None): """ Run the set of tasks related to monitoring instances for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. """ provider = Provider.objects.get(id=provider_id) # For now, lets just ignore everything that isn't openstack. if 'openstack' not in provider.type.name.lower(): return instance_map = _get_instance_owner_map(provider, users=users) if print_logs: import logging import sys consolehandler = logging.StreamHandler(sys.stdout) consolehandler.setLevel(logging.DEBUG) celery_logger.addHandler(consolehandler) # DEVNOTE: Potential slowdown running multiple functions # Break this out when instance-caching is enabled running_total = 0 for username in sorted(instance_map.keys()): running_instances = instance_map[username] running_total += len(running_instances) identity = _get_identity_from_tenant_name(provider, username) if identity and running_instances: try: driver = get_cached_driver(identity=identity) core_running_instances = [ convert_esh_instance( driver, inst, identity.provider.uuid, identity.uuid, identity.created_by) for inst in running_instances] except Exception as exc: celery_logger.exception( "Could not convert running instances for %s" % username) continue else: # No running instances. core_running_instances = [] # Using the 'known' list of running instances, cleanup the DB core_instances = _cleanup_missing_instances( identity, core_running_instances) if check_allocations: allocation_result = user_over_allocation_enforcement( provider, username, print_logs, start_date, end_date) if print_logs: celery_logger.removeHandler(consolehandler) return running_total
def set_provider_quota(identity_uuid): try: return spq(identity_uuid) except Exception as exc: celery_logger.exception("Encountered an exception trying to " "'set_provider_quota' for Identity UUID:%s" % identity_uuid) set_provider_quota.retry(exc=exc)
def add_membership_task(image_version, group): celery_logger.debug("add_membership_task task started at %s." % timezone.now()) try: add_membership(image_version, group) celery_logger.debug("add_membership_task task finished at %s." % timezone.now()) except Exception as exc: celery_logger.exception(exc) add_membership_task.retry(exc=exc)
def set_provider_quota(identity_uuid): try: return spq(identity_uuid) except Exception as exc: celery_logger.exception( "Encountered an exception trying to " "'set_provider_quota' for Identity UUID:%s" % identity_uuid) set_provider_quota.retry(exc=exc)
def _req( self, method, url, json_data={}, additional_headers={}, json_resp=True ): """ send a request with given method to the given url Args: method (str): HTTP method url (str): api url to send the request to json_data (dict, optional): JSON payload. Defaults to None. additional_header (dict, optional): additional headers. Defaults to None. json_resp (bool, optional): if response is json. Defaults to True. Raises: ResponseNotJSON: raised when the response is not JSON HTTPError: requert failed Returns: dict: response text as JSON object """ try: headers = {} headers["Host"] = self.host headers["Accept"] = "application/json;q=0.9,*/*;q=0.8" headers["Content-Type"] = "application/json" if self._token: headers["Authorization"] = "Bearer " + self._token if additional_headers: headers.update(additional_headers) full_url = self.base_url + url requests_func = _http_method(method) if json_data: resp = requests_func( full_url, headers=headers, json=json_data, verify=self.verify ) else: resp = requests_func( full_url, headers=headers, verify=self.verify ) resp.raise_for_status() if json_resp: return json.loads(resp.text) return resp.text except JSONDecodeError as exc: msg = "ARGO - REST API, {}, {}".format(type(exc), resp.text) logger.exception(msg) raise ResponseNotJSON("ARGO, Fail to parse response body as JSON") except requests.exceptions.HTTPError as exc: msg = "ARGO - REST API, {}, {}".format(type(exc), resp.text) logger.exception(msg) raise exc
def attach_task(driverCls, provider, identity, instance_id, volume_id, device_choice=None, *args, **kwargs): try: celery_logger.debug("attach_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) # Step 1. Attach the volume # NOTE: device_choice !== device 100% driver.attach_volume(instance, volume, device_choice) # When the reslt returns the volume will be 'attaching' # We can't do anything until the volume is 'available/in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) # Give up if you can't find the volume if not volume: return None if attempts > 6: # After 6 attempts (~1min) break # Openstack Check if isinstance(driver, OSDriver) and\ 'attaching' not in volume.extra.get('status', ''): break if isinstance(driver, EucaDriver) and\ 'attaching' not in volume.extra.get('status', ''): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts celery_logger.debug("Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra.get('status', 'no-status'), sleep_time)) time.sleep(sleep_time) if 'available' in volume.extra.get('status', ''): raise Exception("Volume %s failed to attach to instance %s" % (volume.id, instance.id)) # Device path for euca == openstack try: attach_data = volume.extra['attachments'][0] device = attach_data['device'] except (IndexError, KeyError) as bad_fetch: celery_logger.warn("Could not find 'device' in " "volume.extra['attachments']: " "Volume:%s Extra:%s" % (volume.id, volume.extra)) device = None celery_logger.debug("attach_task finished at %s." % datetime.now()) return device except Exception as exc: celery_logger.exception(exc) attach_task.retry(exc=exc)
def monitor_instances_for(provider_id, users=None, print_logs=False, check_allocations=False, start_date=None, end_date=None): """ Run the set of tasks related to monitoring instances for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. """ provider = Provider.objects.get(id=provider_id) # For now, lets just ignore everything that isn't openstack. if 'openstack' not in provider.type.name.lower(): return instance_map = _get_instance_owner_map(provider, users=users) if print_logs: console_handler = _init_stdout_logging() # DEVNOTE: Potential slowdown running multiple functions # Break this out when instance-caching is enabled running_total = 0 if not settings.ENFORCING: celery_logger.debug('Settings dictate allocations are NOT enforced') for username in sorted(instance_map.keys()): running_instances = instance_map[username] running_total += len(running_instances) identity = _get_identity_from_tenant_name(provider, username) if identity and running_instances: try: driver = get_cached_driver(identity=identity) core_running_instances = [ convert_esh_instance(driver, inst, identity.provider.uuid, identity.uuid, identity.created_by) for inst in running_instances ] except Exception as exc: celery_logger.exception( "Could not convert running instances for %s" % username) continue else: # No running instances. core_running_instances = [] # Using the 'known' list of running instances, cleanup the DB core_instances = _cleanup_missing_instances(identity, core_running_instances) if check_allocations: allocation_result = user_over_allocation_enforcement( provider, username, print_logs, start_date, end_date) if print_logs: _exit_stdout_logging(console_handler) return running_total
def add_membership_task(image_version, group): celery_logger.debug("add_membership_task task started at %s." % timezone.now()) try: add_membership(image_version, group) celery_logger.debug("add_membership_task task finished at %s." % timezone.now()) except Exception as exc: celery_logger.exception(exc) add_membership_task.retry(exc=exc)
def argo_wf_template_exec( wf_template_filename, provider_uuid, workflow_data, config_file_path=None, wait=False ): """ Execute an specified Argo workflow. Find file based on provider. Pass argument to workflow. Args: wf_template_filename (str): filename of the workflow provider_uuid (str): uuid of the provider workflow_data (dict): data to be passed to workflow as arguments config_file_path (str, optional): path to the config file. will use the default one from the setting if None. Defaults to None. wait (bool, optional): wait for workflow to complete. Defaults to False. Returns: (ArgoWorkflow, dict): workflow and status of the workflow, e.g. {"complete": bool, "success": bool, "error": bool} """ try: # read configuration from file config = read_argo_config(config_file_path=config_file_path) # construct workflow context context = ArgoContext(config=config) # find the workflow definition wf_temp_def = argo_lookup_yaml_file( config["workflow_base_dir"], wf_template_filename, provider_uuid ) # submit workflow template wf_temp = ArgoWorkflowTemplate.create(context, wf_temp_def) wf_name = wf_temp.execute(context, wf_param=workflow_data) wf = ArgoWorkflow(wf_name) # polling if needed if wait: status = wf.watch(context, 10, 18) if status.complete: return (wf_name, status) status = wf.watch(context, 60, 1440) return (wf, status) return (wf, {"complete": None, "success": None, "error": None}) except Exception as exc: logger.exception( "ARGO, argo_wf_template_exec(), {} {}".format(type(exc), exc) ) raise exc
def monitor_instances_for(provider_id, users=None, print_logs=False, start_date=None, end_date=None): """ Run the set of tasks related to monitoring instances for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. """ provider = Provider.objects.get(id=provider_id) # For now, lets just ignore everything that isn't openstack. if 'openstack' not in provider.type.name.lower(): return instance_map = _get_instance_owner_map(provider, users=users) if print_logs: console_handler = _init_stdout_logging() seen_instances = [] # DEVNOTE: Potential slowdown running multiple functions # Break this out when instance-caching is enabled if not settings.ENFORCING: celery_logger.debug('Settings dictate allocations are NOT enforced') for tenant_name in sorted(instance_map.keys()): running_instances = instance_map[tenant_name] identity = _get_identity_from_tenant_name(provider, tenant_name) if identity and running_instances: try: driver = get_cached_driver(identity=identity) core_running_instances = [ convert_esh_instance( driver, inst, identity.provider.uuid, identity.uuid, identity.created_by) for inst in running_instances] seen_instances.extend(core_running_instances) except Exception as exc: celery_logger.exception( "Could not convert running instances for %s" % tenant_name) continue else: # No running instances. core_running_instances = [] # Using the 'known' list of running instances, cleanup the DB core_instances = _cleanup_missing_instances( identity, core_running_instances) if print_logs: _exit_stdout_logging(console_handler) # return seen_instances NOTE: this has been commented out to avoid PicklingError! # TODO: Uncomment the above, Determine what _we can return_ and return that instead.... return
def monitor_instances_for( provider_id, users=None, print_logs=False, start_date=None, end_date=None ): """ Run the set of tasks related to monitoring instances for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. """ provider = Provider.objects.get(id=provider_id) # For now, lets just ignore everything that isn't openstack. if 'openstack' not in provider.type.name.lower(): return instance_map = _get_instance_owner_map(provider, users=users) if print_logs: console_handler = _init_stdout_logging() seen_instances = [] # DEVNOTE: Potential slowdown running multiple functions # Break this out when instance-caching is enabled if not settings.ENFORCING: celery_logger.debug('Settings dictate allocations are NOT enforced') for tenant_name in sorted(instance_map.keys()): running_instances = instance_map[tenant_name] identity = _get_identity_from_tenant_name(provider, tenant_name) if identity and running_instances: try: driver = get_cached_driver(identity=identity) core_running_instances = [ convert_esh_instance( driver, inst, identity.provider.uuid, identity.uuid, identity.created_by ) for inst in running_instances ] seen_instances.extend(core_running_instances) except Exception: celery_logger.exception( "Could not convert running instances for %s" % tenant_name ) continue else: # No running instances. core_running_instances = [] # Using the 'known' list of running instances, cleanup the DB _cleanup_missing_instances(identity, core_running_instances) if print_logs: _exit_stdout_logging(console_handler) # return seen_instances NOTE: this has been commented out to avoid PicklingError! # TODO: Uncomment the above, Determine what _we can return_ and return that instead.... return
def detach_task( driverCls, provider, identity, instance_id, volume_id, *args, **kwargs ): try: celery_logger.debug("detach_task started at %s." % timezone.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) driver.detach_volume(volume) # When the reslt returns the volume will be 'detaching' # We will ensure the volume does not return to 'in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) if attempts > 6: # After 6 attempts (~1min) break # The Openstack way if isinstance(driver, OSDriver)\ and 'detaching' not in volume.extra['status']: break # The Eucalyptus way attach_data = volume.extra['attachments'][0] if isinstance(driver, EucaDriver) and attach_data\ and 'detaching' not in attach_data.get('status'): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts celery_logger.debug( "Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra['status'], sleep_time) ) time.sleep(sleep_time) if 'in-use' in volume.extra['status']: raise Exception( "Failed to detach Volume %s to instance %s" % (volume, instance) ) celery_logger.debug("detach_task finished at %s." % timezone.now()) except DeviceBusyException: # We should NOT retry if the device is busy raise except Exception as exc: # If the volume is NOT attached, do not retry. if 'Volume is not attached' in exc.message: return celery_logger.exception(exc) detach_task.retry(exc=exc)
def allocation_source_overage_enforcement_for_user(allocation_source, user): celery_logger.debug('allocation_source_overage_enforcement_for_user - allocation_source: %s, user: %s', allocation_source, user) user_instances = [] for identity in user.current_identities: try: celery_logger.debug('allocation_source_overage_enforcement_for_user - identity: %s', identity) affected_instances = allocation_source_overage_enforcement_for(allocation_source, user, identity) user_instances.extend(affected_instances) except Exception: celery_logger.exception( 'allocation_source_overage_enforcement_for allocation_source: %s, user: %s, and identity: %s', allocation_source, user, identity) return user_instances
def allocation_source_overage_enforcement_for_user(allocation_source, user): celery_logger.debug('allocation_source_overage_enforcement_for_user - allocation_source: %s, user: %s', allocation_source, user) user_instances = [] for identity in user.current_identities: try: celery_logger.debug('allocation_source_overage_enforcement_for_user - identity: %s', identity) affected_instances = allocation_source_overage_enforcement_for(allocation_source, user, identity) user_instances.extend(affected_instances) except Exception: celery_logger.exception( 'allocation_source_overage_enforcement_for allocation_source: %s, user: %s, and identity: %s', allocation_source, user, identity) return user_instances
def argo_workflow_exec( workflow_filename, provider_uuid, workflow_data, config_file_path=None, wait=False ): """ Execute an specified Argo workflow. Find file based on provider. Pass argument to workflow. Args: workflow_filename (str): filename of the workflow provider_uuid (str): uuid of the provider workflow_data (dict): data to be passed to workflow as arguments config_file_path (str, optional): path to the config file. will use the default one from the setting if None. Defaults to None. wait (bool, optional): wait for workflow to complete. Defaults to False. Returns: (ArgoWorkflow, ArgoWorkflowStatus): workflow and status of the workflow """ try: # read configuration from file config = read_argo_config( config_file_path=config_file_path, provider_uuid=provider_uuid ) # find the workflow definition & construct workflow wf_def = argo_lookup_yaml_file( config["workflow_base_dir"], workflow_filename, provider_uuid ) # construct workflow context context = ArgoContext(config=config) # execute if wait: result = ArgoWorkflow.create_n_watch(context, wf_def, workflow_data) return result wf = ArgoWorkflow.create(context, wf_def, workflow_data) return (wf, ArgoWorkflowStatus()) except Exception as exc: logger.exception( "ARGO, argo_workflow_exec(), {} {}".format(type(exc), exc) ) raise exc
def update_volume_metadata(driverCls, provider, identity, volume_alias, metadata): """ """ from service import volume as volume_service try: celery_logger.debug("update_volume_metadata task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_alias) if not volume: return return volume_service.update_volume_metadata(driver, volume, metadata=metadata) celery_logger.debug("volume_metadata task finished at %s." % datetime.now()) except Exception as exc: celery_logger.exception(exc) update_volume_metadata.retry(exc=exc)
def remove_empty_networks_for(provider_id): provider = Provider.objects.get(id=provider_id) os_driver = get_account_driver(provider) if not os_driver: celery_logger.warn( "Cannot remove_empty_networks_for provider %s -- Account Driver not created" % provider) return all_instances = os_driver.admin_driver.list_all_instances() project_map = os_driver.network_manager.project_network_map() known_project_names = Credential.objects.filter( key='ex_project_name').values_list('value', flat=True) projects_with_networks = sorted( [k for k in project_map.keys() if k in known_project_names]) for project in projects_with_networks: networks = project_map[project]['network'] if not isinstance(networks, list): networks = [networks] for network in networks: network_name = network['name'] celery_logger.debug("Checking if network %s is in use" % network_name) if running_instances(network_name, all_instances): continue user = project identity = Identity.objects.filter( provider_id=provider_id, credential__key='ex_project_name', credential__value=project).filter( credential__key='key', credential__value=user).first() if not identity: celery_logger.warn( "NOT Removing project network for User:%s, Project:%s -- No Valid Identity found!" % (user, project)) continue try: celery_logger.debug( "Removing project network for User:%s, Project:%s" % (user, project)) os_driver.delete_user_network(identity) except NeutronClientException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project)) except NeutronException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project))
def check_volume_task(driverCls, provider, identity, instance_id, volume_id, *args, **kwargs): try: celery_logger.debug("check_volume task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) attach_data = volume.extra['attachments'][0] device = attach_data['device'] private_key = ATMOSPHERE_PRIVATE_KEYFILE kwargs.update({'ssh_key': private_key}) kwargs.update({'timeout': 120}) # One script to make two checks: # 1. Voume exists 2. Volume has a filesystem cv_script = check_volume(device) # NOTE: non_zero_deploy needed to stop LibcloudDeploymentError from being # raised kwargs.update({'deploy': cv_script, 'non_zero_deploy': True}) driver.deploy_to(instance, **kwargs) kwargs.pop('non_zero_deploy', None) # Script execute if cv_script.exit_status != 0: if 'No such file' in cv_script.stdout: raise Exception('Volume check failed: %s. ' 'Device %s does not exist on instance %s' % (volume, device, instance)) elif 'Bad magic number' in cv_script.stdout: # Filesystem needs to be created for this device celery_logger.info("Mkfs needed") mkfs_script = mkfs_volume(device) kwargs.update({'deploy': mkfs_script}) driver.deploy_to(instance, **kwargs) else: raise Exception('Volume check failed: Something weird') celery_logger.debug("check_volume task finished at %s." % datetime.now()) except LibcloudDeploymentError as exc: celery_logger.exception(exc) except Exception as exc: celery_logger.warn(exc) check_volume_task.retry(exc=exc)
def check_volume_task(driverCls, provider, identity, instance_id, volume_id, *args, **kwargs): try: celery_logger.debug("check_volume task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) attach_data = volume.extra['attachments'][0] device = attach_data['device'] private_key = ATMOSPHERE_PRIVATE_KEYFILE kwargs.update({'ssh_key': private_key}) kwargs.update({'timeout': 120}) # One script to make two checks: # 1. Voume exists 2. Volume has a filesystem cv_script = check_volume(device) # NOTE: non_zero_deploy needed to stop LibcloudDeploymentError from being # raised kwargs.update({'deploy': cv_script, 'non_zero_deploy': True}) driver.deploy_to(instance, **kwargs) kwargs.pop('non_zero_deploy', None) # Script execute if cv_script.exit_status != 0: if 'No such file' in cv_script.stdout: raise Exception('Volume check failed: %s. ' 'Device %s does not exist on instance %s' % (volume, device, instance)) elif 'Bad magic number' in cv_script.stdout: # Filesystem needs to be created for this device celery_logger.info("Mkfs needed") mkfs_script = mkfs_volume(device) kwargs.update({'deploy': mkfs_script}) driver.deploy_to(instance, **kwargs) else: raise Exception('Volume check failed: Something weird') celery_logger.debug("check_volume task finished at %s." % datetime.now()) except LibcloudDeploymentError as exc: celery_logger.exception(exc) except Exception as exc: celery_logger.warn(exc) check_volume_task.retry(exc=exc)
def check_volume_task(driverCls, provider, identity, instance_id, volume_id, device_type='ext4', *args, **kwargs): try: celery_logger.debug("check_volume task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) username = identity.get_username() attach_data = volume.extra['attachments'][0] device_location = attach_data['device'] celery_logger.info("device_location: %s" % device_location) # One playbook to make two checks: # 1. Voume exists # 2. Volume has a filesystem # (If not, create one of type 'device_type') playbooks = deploy_check_volume(instance.ip, username, instance.id, device_location, device_type=device_type) celery_logger.info(playbooks.__dict__) hostname = build_host_name(instance.id, instance.ip) result = False if execution_has_failures(playbooks, hostname)\ or execution_has_unreachable(playbooks, hostname) else True if not result: raise Exception( "Error encountered while checking volume for filesystem: %s" % playbooks.stats.summarize(host=hostname)) return result except LibcloudDeploymentError as exc: celery_logger.exception(exc) except Exception as exc: celery_logger.warn(exc) check_volume_task.retry(exc=exc)
def update_volume_metadata( driverCls, provider, identity, volume_alias, metadata ): """ """ from service import volume as volume_service try: celery_logger.debug( "update_volume_metadata task started at %s." % timezone.now() ) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_alias) if not volume: return return volume_service._update_volume_metadata( driver, volume, metadata=metadata ) except Exception as exc: celery_logger.exception(exc) update_volume_metadata.retry(exc=exc)
def send_email(subject, body, from_email, to, cc=None, fail_silently=False, html=False): """ Use django.core.mail.EmailMessage to send and log an Atmosphere email. """ try: msg = EmailMessage(subject=subject, body=body, from_email=from_email, to=to, cc=cc) if html: msg.content_subtype = 'html' msg.send(fail_silently=fail_silently) args = (from_email, to, cc, subject, body) email_logger.info(log_message.format(*args)) return True except Exception as e: celery_logger.exception(e) return False
def attach_task(driverCls, provider, identity, instance_id, volume_id, device_choice=None, *args, **kwargs): try: celery_logger.debug("attach_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) from service.volume import attach_volume # TODO: Test pulling this up -- out of band attach_volume(driver, instance_id, volume_id, device_choice=device_choice) # When the reslt returns the volume will be 'attaching' # We can't do anything until the volume is 'available/in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) # Give up if you can't find the volume if not volume: return None if attempts > 6: # After 6 attempts (~1min) break # Openstack Check if isinstance(driver, OSDriver) and\ 'attaching' not in volume.extra.get('status', ''): break if isinstance(driver, EucaDriver) and\ 'attaching' not in volume.extra.get('status', ''): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts celery_logger.debug( "Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra.get('status', 'no-status'), sleep_time)) time.sleep(sleep_time) if 'available' in volume.extra.get('status', ''): raise Exception("Volume %s failed to attach to instance %s" % (volume.id, instance_id)) # Device path for euca == openstack try: attach_data = volume.extra['attachments'][0] device = attach_data['device'] except (IndexError, KeyError) as bad_fetch: celery_logger.warn("Could not find 'device' in " "volume.extra['attachments']: " "Volume:%s Extra:%s" % (volume.id, volume.extra)) device = None celery_logger.debug("attach_task finished at %s." % datetime.now()) return device except Exception as exc: celery_logger.exception(exc) attach_task.retry(exc=exc)