def update_metadata(driverCls, provider, identity, instance_alias, metadata, replace_metadata=False): """ #NOTE: While this looks like a large number (250 ?!) of retries # we expect this task to fail often when the image is building # and large, uncached images can have a build time. """ try: logger.debug("update_metadata task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: return #NOTE: This task will only be executed in TEST mode if app.conf.CELERY_ALWAYS_EAGER: eager_update_metadata(driver, instance, metadata) return update_instance_metadata(driver, instance, data=metadata, replace=replace_metadata) logger.debug("update_metadata task finished at %s." % datetime.now()) except Exception as exc: logger.exception(exc) update_metadata.retry(exc=exc)
def _deploy_init_to(driverCls, provider, identity, instance_id, password=None, **celery_task_args): try: logger.debug("_deploy_init_to task started at %s." % datetime.now()) #Check if instance still exists driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return #NOTE: This is NOT the password passed by argument #Deploy with no password to use ssh keys logger.info(instance.extra) instance._node.extra['password'] = None kwargs = {} private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa" kwargs.update({'ssh_key': private_key}) kwargs.update({'timeout': 120}) msd = init(instance, identity.user.username, password) kwargs.update({'deploy': msd}) driver.deploy_to(instance, **kwargs) logger.debug("_deploy_init_to task finished at %s." % datetime.now()) except DeploymentError as exc: logger.exception(exc) if isinstance(exc.value, NonZeroDeploymentException): #The deployment was successful, but the return code on one or more # steps is bad. Log the exception and do NOT try again! raise exc.value #TODO: Check if all exceptions thrown at this time #fall in this category, and possibly don't retry if #you hit the Exception block below this. _deploy_init_to.retry(exc=exc) except Exception as exc: logger.exception(exc) _deploy_init_to.retry(exc=exc)
def umount_task(driverCls, provider, identity, instance_id, volume_id, *args, **kwargs): try: celery_logger.debug("umount_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) attach_data = volume.extra["attachments"][0] device = attach_data["device"] # Check mount to find the mount_location for device private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa" kwargs.update({"ssh_key": private_key}) kwargs.update({"timeout": 120}) mount_location = None cm_script = check_mount() kwargs.update({"deploy": cm_script}) driver.deploy_to(instance, **kwargs) regex = re.compile("(?P<device>[\w/]+) on (?P<location>.*) type") for line in cm_script.stdout.split("\n"): res = regex.search(line) if not res: continue search_dict = res.groupdict() dev_found = search_dict["device"] if device == dev_found: mount_location = search_dict["location"] break # Volume not mounted, move along.. if not mount_location: return um_script = umount_volume(device) kwargs.update({"deploy": um_script}) driver.deploy_to(instance, **kwargs) if "device is busy" in um_script.stdout: # Show all processes that are making device busy.. lsof_script = lsof_location(mount_location) kwargs.update({"deploy": lsof_script}) driver.deploy_to(instance, **kwargs) regex = re.compile("(?P<name>[\w]+)\s*(?P<pid>[\d]+)") offending_processes = [] for line in lsof_script.stdout.split("\n"): res = regex.search(line) if not res: continue search_dict = res.groupdict() offending_processes.append((search_dict["name"], search_dict["pid"])) raise DeviceBusyException(mount_location, offending_processes) # Return here if no errors occurred.. celery_logger.debug("umount_task finished at %s." % datetime.now()) except DeviceBusyException: raise except Exception as exc: celery_logger.warn(exc) umount_task.retry(exc=exc)
def _send_instance_email(driverCls, provider, identity, instance_id): try: logger.debug("_send_instance_email task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) #Breakout if instance has been deleted at this point if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return username = identity.user.username profile = UserProfile.objects.get(user__username=username) if profile.send_emails: #Only send emails if allowed by profile setting created = datetime.strptime(instance.extra['created'], "%Y-%m-%dT%H:%M:%SZ") send_instance_email(username, instance.id, instance.name, instance.ip, created, username) else: logger.debug("User %s elected NOT to receive new instance emails" % username) logger.debug("_send_instance_email task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) _send_instance_email.retry(exc=exc)
def deploy_script(driverCls, provider, identity, instance_id, script, **celery_task_args): try: logger.debug("deploy_script task started at %s." % datetime.now()) #Check if instance still exists driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return instance._node.extra['password'] = None kwargs = _generate_ssh_kwargs() kwargs.update({'deploy': script}) driver.deploy_to(instance, **kwargs) logger.debug("deploy_script task finished at %s." % datetime.now()) except DeploymentError as exc: logger.exception(exc) if isinstance(exc.value, NonZeroDeploymentException): #The deployment was successful, but the return code on one or more # steps is bad. Log the exception and do NOT try again! raise exc.value #TODO: Check if all exceptions thrown at this time #fall in this category, and possibly don't retry if #you hit the Exception block below this. deploy_script.retry(exc=exc) except Exception as exc: logger.exception(exc) deploy_script.retry(exc=exc)
def update_mount_location(new_mount_location, driverCls, provider, identity, volume_alias): """ """ from service import volume as volume_service try: celery_logger.debug( "update_mount_location task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_alias) if not volume: return if not new_mount_location: return volume_metadata = volume.extra['metadata'] return volume_service._update_volume_metadata( driver, volume, metadata={'mount_location': new_mount_location}) celery_logger.debug( "update_mount_location task finished at %s." % datetime.now()) except Exception as exc: celery_logger.exception(exc) update_mount_location.retry(exc=exc)
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id, **celery_task_args): from core.models.instance import Instance from core.email import send_deploy_failed_email try: logger.debug("deploy_failed task started at %s." % datetime.now()) logger.info("task_uuid=%s" % task_uuid) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = "DEPLOYERROR::%s" % (result.traceback, ) logger.error(err_str) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) update_instance_metadata(driver, instance, data={'tmp_status': 'deploy_error'}, replace=False) #Send deploy email core_instance = Instance.objects.get(provider_alias=instance_id) send_deploy_failed_email(core_instance, err_str) logger.debug("deploy_failed task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) deploy_failed.retry(exc=exc)
def mount_volume_task( driverCls, provider, identity, instance_id, volume_id, device_location, mount_location, device_type, mount_prefix=None, *args, **kwargs ): try: celery_logger.debug("mount task started at %s." % timezone.now()) celery_logger.debug("mount_location: %s" % (mount_location, )) driver = get_driver(driverCls, provider, identity) username = identity.get_username() instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) try: attach_data = volume.extra['attachments'][0] if not device_location: device_location = attach_data['device'] except (KeyError, IndexError): celery_logger.warn( "Volume %s missing attachments in Extra" % (volume, ) ) if not device_location: raise Exception( "No device_location found or inferred by volume %s" % volume ) if not mount_prefix: mount_prefix = "/vol_" last_char = device_location[-1] # /dev/sdb --> b if not mount_location: mount_location = mount_prefix + last_char playbook_results = deploy_mount_volume( instance.ip, username, instance.id, device_location, mount_location=mount_location, device_type=device_type ) celery_logger.info(playbook_results) if execution_has_failures( playbook_results ) or execution_has_unreachable(playbook_results): raise Exception( "Error encountered while mounting volume: instance_id: {}, volume_id: {}" .format(instance_id, volume_id) ) return mount_location except Exception as exc: celery_logger.warn(exc) mount_volume_task.retry(exc=exc)
def deploy_init_to(driverCls, provider, identity, instance_id, username=None, password=None, redeploy=False, *args, **kwargs): try: logger.debug("deploy_init_to task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return image_metadata = driver._connection\ .ex_get_image_metadata(instance.machine) deploy_chain = get_deploy_chain(driverCls, provider, identity, instance, username, password, redeploy) deploy_chain.apply_async() #Can be really useful when testing. #if kwargs.get('delay'): # async.get() logger.debug("deploy_init_to task finished at %s." % datetime.now()) except SystemExit: logger.exception("System Exits are BAD! Find this and get rid of it!") raise Exception("System Exit called") except NonZeroDeploymentException: raise except Exception as exc: logger.warn(exc) deploy_init_to.retry(exc=exc)
def add_fixed_ip(driverCls, provider, identity, instance_id, core_identity_id=None): from service import instance as instance_service try: logger.debug("add_fixed_ip task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return None if instance._node.private_ips: #TODO: Attempt to rescue logger.info("Instance has fixed IP: %s" % instance_id) return instance network_id = instance_service._get_network_id(driver, instance) fixed_ip = driver._connection.ex_add_fixed_ip(instance, network_id) logger.debug("add_fixed_ip task finished at %s." % datetime.now()) return fixed_ip except Exception as exc: if "Not Ready" not in str(exc): # Ignore 'normal' errors. logger.exception(exc) add_fixed_ip.retry(exc=exc)
def _deploy_init_to(driverCls, provider, identity, instance_id, username=None, password=None, redeploy=False, **celery_task_args): try: logger.debug("_deploy_init_to task started at %s." % datetime.now()) #Check if instance still exists driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return #NOTE: This is unrelated to the password argument logger.info(instance.extra) instance._node.extra['password'] = None msd = init(instance, identity.user.username, password, redeploy) kwargs = _generate_ssh_kwargs() kwargs.update({'deploy': msd}) driver.deploy_to(instance, **kwargs) _update_status_log(instance, "Deploy Finished") logger.debug("_deploy_init_to task finished at %s." % datetime.now()) except DeploymentError as exc: logger.exception(exc) if isinstance(exc.value, NonZeroDeploymentException): #The deployment was successful, but the return code on one or more # steps is bad. Log the exception and do NOT try again! raise exc.value #TODO: Check if all exceptions thrown at this time #fall in this category, and possibly don't retry if #you hit the Exception block below this. _deploy_init_to.retry(exc=exc) except Exception as exc: logger.exception(exc) _deploy_init_to.retry(exc=exc)
def deploy_init_to(driverCls, provider, identity, instance_id, username=None, password=None, redeploy=False, *args, **kwargs): try: logger.debug("deploy_init_to task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return image_metadata = driver._connection\ .ex_get_image_metadata(instance.machine) deploy_chain = get_deploy_chain(driverCls, provider, identity, instance, username, password, redeploy) deploy_chain.apply_async() #Can be really useful when testing. #if kwargs.get('delay'): # async.get() logger.debug("deploy_init_to task finished at %s." % datetime.now()) except SystemExit: logger.exception("System Exits are BAD! Find this and get rid of it!") raise Exception("System Exit called") except NonZeroDeploymentException as non_zero: logger.error(str(non_zero)) logger.error(non_zero.__dict__) raise except Exception as exc: logger.warn(exc) deploy_init_to.retry(exc=exc)
def mount_failed( context, exception_msg, traceback, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args ): from service import volume as volume_service try: celery_logger.debug("mount_failed task started at %s." % timezone.now()) celery_logger.info("task context=%s" % context) err_str = "%s\nMount Error Traceback:%s" % (exception_msg, traceback) celery_logger.error(err_str) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_id) if unmount: tmp_status = 'umount_error' else: tmp_status = 'mount_error' return volume_service._update_volume_metadata( driver, volume, metadata={'tmp_status': tmp_status} ) except Exception as exc: celery_logger.warn(exc) mount_failed.retry(exc=exc)
def attach_task(driverCls, provider, identity, instance_id, volume_id, device_choice=None, *args, **kwargs): #TODO: chain task attach THEN mount for more robust-ness try: logger.debug("attach_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) #Step 1. Attach the volume #NOTE: device_choice !== device 100% driver.attach_volume(instance, volume, device_choice) #When the reslt returns the volume will be 'attaching' #We can't do anything until the volume is 'available/in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) # Give up if you can't find the volume if not volume: return None if attempts > 6: # After 6 attempts (~1min) break #Openstack Check if isinstance(driver, OSDriver) and\ 'attaching' not in volume.extra.get('status',''): break attach_set = volume.extra['attachmentSet'][0] if isinstance(driver, EucaDriver) and\ 'attaching' not in attach_set.get('status', ''): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts logger.debug("Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra.get( 'status', 'no-status'), sleep_time)) time.sleep(sleep_time) if 'available' in volume.extra.get('status', ''): raise Exception("Volume %s failed to attach to instance %s" % (volume.id, instance.id)) #Device path for euca == openstack try: device = volume.extra['attachmentSet'][0]['device'] except: device = None logger.debug("attach_task finished at %s." % datetime.now()) return device except Exception as exc: logger.warn(exc) attach_task.retry(exc=exc)
def mount_task(driverCls, provider, identity, instance_id, volume_id, mount_location=None, *args, **kwargs): try: logger.debug("mount task started at %s." % datetime.now()) logger.debug("mount_location: %s" % (mount_location, )) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) logger.debug(volume) try: device = volume.extra['attachmentSet'][0]['device'] except: device = None if not device: #Device was never attached -- Nothing to mount return private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa" kwargs.update({'ssh_key': private_key}) kwargs.update({'timeout': 120}) #Step 2. Check the volume is not already mounted cm_script = check_mount() kwargs.update({'deploy': cm_script}) driver.deploy_to(instance, **kwargs) if device in cm_script.stdout: #Device has already been mounted. Move along.. return #Step 3. Find a suitable location to mount the volume logger.info("Original mount location - %s" % mount_location) if not mount_location: inc = 1 while True: if '/vol%s' % inc in cm_script.stdout: inc += 1 else: break mount_location = '/vol%s' % inc logger.info("Device location - %s" % device) logger.info("New mount location - %s" % mount_location) mv_script = mount_volume(device, mount_location) kwargs.update({'deploy': mv_script}) driver.deploy_to(instance, **kwargs) #TODO: Update metadata to include volume mount # so this data can persist on the UI? logger.debug("mount task finished at %s." % datetime.now()) return mount_location except Exception as exc: logger.warn(exc) mount_task.retry(exc=exc)
def mount_failed(task_uuid, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args): try: logger.debug("mount_failed task started at %s." % datetime.now()) logger.info("task_uuid=%s" % task_uuid) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = "Mount Error Traceback:%s" % (result.traceback, ) logger.error(err_str) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_id) if unmount: tmp_status = 'umount_error' else: tmp_status = 'mount_error' return volume_service.update_volume_metadata( driver, volume, metadata={'tmp_status': tmp_status}) logger.debug("mount_failed task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) mount_failed.retry(exc=exc)
def unmount_volume_task(driverCls, provider, identity, instance_id, volume_id, *args, **kwargs): try: celery_logger.debug("unmount task started at %s." % timezone.now()) driver = get_driver(driverCls, provider, identity) username = identity.get_username() instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) device_location = None try: attach_data = volume.extra['attachments'][0] device_location = attach_data['device'] except (KeyError, IndexError): celery_logger.warn("Volume %s missing attachments in Extra" % (volume, )) if not device_location: raise Exception( "No device_location found or inferred by volume %s" % volume) try: playbooks = deploy_unmount_volume(instance.ip, username, instance.id, device_location) except DeviceBusyException: # Future-Fixme: Update VolumeStatusHistory.extra, set status to 'unmount_failed' raise hostname = build_host_name(instance.id, instance.ip) result = False if execution_has_failures(playbooks, hostname)\ or execution_has_unreachable(playbooks, hostname) else True if not result: raise Exception("Error encountered while unmounting volume: %s" % playbooks.stats.summarize(host=hostname)) return device_location except Exception as exc: celery_logger.warn(exc) unmount_volume_task.retry(exc=exc)
def attach_task(driverCls, provider, identity, instance_id, volume_id, device_choice=None, *args, **kwargs): try: logger.debug("attach_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) #Step 1. Attach the volume #NOTE: device_choice !== device 100% driver.attach_volume(instance, volume, device_choice) #When the reslt returns the volume will be 'attaching' #We can't do anything until the volume is 'available/in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) # Give up if you can't find the volume if not volume: return None if attempts > 6: # After 6 attempts (~1min) break #Openstack Check if isinstance(driver, OSDriver) and\ 'attaching' not in volume.extra.get('status',''): break if isinstance(driver, EucaDriver) and\ 'attaching' not in volume.extra.get('status', ''): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts logger.debug("Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra.get( 'status', 'no-status'), sleep_time)) time.sleep(sleep_time) if 'available' in volume.extra.get('status', ''): raise Exception("Volume %s failed to attach to instance %s" % (volume.id, instance.id)) #Device path for euca == openstack try: attach_data = volume.extra['attachments'][0] device = attach_data['device'] except (IndexError, KeyError), bad_fetch: logger.warn("Could not find 'device' in " "volume.extra['attachments']: " "Volume:%s Extra:%s" % (volume.id, volume.extra)) device = None logger.debug("attach_task finished at %s." % datetime.now()) return device
def attach_task(driverCls, provider, identity, instance_id, volume_id, device_choice=None, *args, **kwargs): try: celery_logger.debug("attach_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) # Step 1. Attach the volume # NOTE: device_choice !== device 100% driver.attach_volume(instance, volume, device_choice) # When the reslt returns the volume will be 'attaching' # We can't do anything until the volume is 'available/in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) # Give up if you can't find the volume if not volume: return None if attempts > 6: # After 6 attempts (~1min) break # Openstack Check if isinstance(driver, OSDriver) and\ 'attaching' not in volume.extra.get('status', ''): break if isinstance(driver, EucaDriver) and\ 'attaching' not in volume.extra.get('status', ''): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts celery_logger.debug("Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra.get('status', 'no-status'), sleep_time)) time.sleep(sleep_time) if 'available' in volume.extra.get('status', ''): raise Exception("Volume %s failed to attach to instance %s" % (volume.id, instance.id)) # Device path for euca == openstack try: attach_data = volume.extra['attachments'][0] device = attach_data['device'] except (IndexError, KeyError) as bad_fetch: celery_logger.warn("Could not find 'device' in " "volume.extra['attachments']: " "Volume:%s Extra:%s" % (volume.id, volume.extra)) device = None celery_logger.debug("attach_task finished at %s." % datetime.now()) return device except Exception as exc: celery_logger.warn(exc) attach_task.retry(exc=exc)
def attach_task(driverCls, provider, identity, instance_id, volume_id, device_choice=None, *args, **kwargs): #TODO: chain task attach THEN mount for more robust-ness try: logger.debug("attach_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) #Step 1. Attach the volume #NOTE: device_choice !== device 100% driver.attach_volume(instance, volume, device_choice) #When the reslt returns the volume will be 'attaching' #We can't do anything until the volume is 'available/in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) # Give up if you can't find the volume if not volume: return None if attempts > 6: # After 6 attempts (~1min) break #Openstack Check if isinstance(driver, OSDriver) and\ 'attaching' not in volume.extra.get('status',''): break attach_set = volume.extra['attachmentSet'][0] if isinstance(driver, EucaDriver) and\ 'attaching' not in attach_set.get('status', ''): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts logger.debug("Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra.get('status', 'no-status'), sleep_time)) time.sleep(sleep_time) if 'available' in volume.extra.get('status',''): raise Exception("Volume %s failed to attach to instance %s" % (volume.id, instance.id)) #Device path for euca == openstack try: device = volume.extra['attachmentSet'][0]['device'] except: device = None logger.debug("attach_task finished at %s." % datetime.now()) return device except Exception as exc: logger.warn(exc) attach_task.retry(exc=exc)
def clean_empty_ips(driverCls, provider, identity, *args, **kwargs): try: logger.debug("remove_floating_ip task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) ips_cleaned = driver._clean_floating_ip() logger.debug("remove_floating_ip task finished at %s." % datetime.now()) return ips_cleaned except Exception as exc: logger.warn(exc) clean_empty_ips.retry(exc=exc)
def deploy_to(driverCls, provider, identity, instance_id, *args, **kwargs): try: logger.debug("deploy_to task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) driver.deploy_to(instance, *args, **kwargs) logger.debug("deploy_to task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) deploy_to.retry(exc=exc)
def add_floating_ip(driverCls, provider, identity, instance_alias, delete_status=True, *args, **kwargs): #For testing ONLY.. Test cases ignore countdown.. if app.conf.CELERY_ALWAYS_EAGER: logger.debug("Eager task waiting 15 seconds") time.sleep(15) try: logger.debug("add_floating_ip task started at %s." % datetime.now()) #Remove unused floating IPs first, so they can be re-used driver = get_driver(driverCls, provider, identity) driver._clean_floating_ip() #assign if instance doesn't already have an IP addr instance = driver.get_instance(instance_alias) if not instance: logger.debug("Instance has been teminated: %s." % instance_alias) return None floating_ips = driver._connection.neutron_list_ips(instance) if floating_ips: floating_ip = floating_ips[0]["floating_ip_address"] else: floating_ip = driver._connection.neutron_associate_ip( instance, *args, **kwargs)["floating_ip_address"] _update_status_log(instance, "Networking Complete") #TODO: Implement this as its own task, with the result from #'floating_ip' passed in. Add it to the deploy_chain before deploy_to hostname = "" if floating_ip.startswith('128.196'): regex = re.compile( "(?P<one>[0-9]+)\.(?P<two>[0-9]+)\." "(?P<three>[0-9]+)\.(?P<four>[0-9]+)") r = regex.search(floating_ip) (one, two, three, four) = r.groups() hostname = "vm%s-%s.iplantcollaborative.org" % (three, four) else: # Find a way to convert new floating IPs to hostnames.. hostname = floating_ip update_instance_metadata(driver, instance, data={ 'public-hostname': hostname, 'public-ip':floating_ip}, replace=False) logger.info("Assigned IP:%s - Hostname:%s" % (floating_ip, hostname)) #End logger.debug("add_floating_ip task finished at %s." % datetime.now()) return {"floating_ip":floating_ip, "hostname":hostname} except Exception as exc: logger.exception("Error occurred while assigning a floating IP") #Networking can take a LONG time when an instance first launches, #it can also be one of those things you 'just miss' by a few seconds.. #So we will retry 30 times using limited exp.backoff #Max Time: 53min countdown = min(2**current.request.retries, 128) add_floating_ip.retry(exc=exc, countdown=countdown)
def add_floating_ip(driverCls, provider, identity, instance_alias, delete_status=True, *args, **kwargs): #For testing ONLY.. Test cases ignore countdown.. if app.conf.CELERY_ALWAYS_EAGER: logger.debug("Eager task waiting 15 seconds") time.sleep(15) try: logger.debug("add_floating_ip task started at %s." % datetime.now()) #Remove unused floating IPs first, so they can be re-used driver = get_driver(driverCls, provider, identity) driver._clean_floating_ip() #assign if instance doesn't already have an IP addr instance = driver.get_instance(instance_alias) if not instance: logger.debug("Instance has been teminated: %s." % instance_alias) return None floating_ips = driver._connection.neutron_list_ips(instance) if floating_ips: floating_ip = floating_ips[0]["floating_ip_address"] else: floating_ip = driver._connection.neutron_associate_ip( instance, *args, **kwargs)["floating_ip_address"] _update_status_log(instance, "Networking Complete") #TODO: Implement this as its own task, with the result from #'floating_ip' passed in. Add it to the deploy_chain before deploy_to hostname = "" if floating_ip.startswith('128.196'): regex = re.compile( "(?P<one>[0-9]+)\.(?P<two>[0-9]+)\." "(?P<three>[0-9]+)\.(?P<four>[0-9]+)") r = regex.search(floating_ip) (one, two, three, four) = r.groups() hostname = "vm%s-%s.iplantcollaborative.org" % (three, four) else: # Find a way to convert new floating IPs to hostnames.. hostname = floating_ip update_instance_metadata(driver, instance, data={ 'public-hostname': hostname, 'public-ip': floating_ip}, replace=False) logger.info("Assigned IP:%s - Hostname:%s" % (floating_ip, hostname)) #End logger.debug("add_floating_ip task finished at %s." % datetime.now()) return {"floating_ip": floating_ip, "hostname": hostname} except Exception as exc: logger.exception("Error occurred while assigning a floating IP") #Networking can take a LONG time when an instance first launches, #it can also be one of those things you 'just miss' by a few seconds.. #So we will retry 30 times using limited exp.backoff #Max Time: 53min countdown = min(2**current.request.retries, 128) add_floating_ip.retry(exc=exc, countdown=countdown)
def _deploy_init_to(driverCls, provider, identity, instance_id, username=None, password=None, token=None, redeploy=False, **celery_task_args): #Note: Splitting preperation (Of the MultiScriptDeployment) and execution # This makes it easier to output scripts for debugging of users. try: logger.debug("_deploy_init_to task started at %s." % datetime.now()) #Check if instance still exists driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return #NOTE: This is required to use ssh to connect. #TODO: Is this still necessary? What about times when we want to use # the adminPass? --Steve logger.info(instance.extra) instance._node.extra['password'] = None msd = init(instance, identity.user.username, password, token, redeploy) except Exception as exc: logger.exception(exc) _deploy_init_to.retry(exc=exc) try: kwargs = _generate_ssh_kwargs() kwargs.update({'deploy': msd}) driver.deploy_to(instance, **kwargs) _update_status_log(instance, "Deploy Finished") logger.debug("_deploy_init_to task finished at %s." % datetime.now()) except DeploymentError as exc: logger.exception(exc) full_deploy_output = _parse_steps_output(msd) if isinstance(exc.value, NonZeroDeploymentException): #The deployment was successful, but the return code on one or more # steps is bad. Log the exception and do NOT try again! raise NonZeroDeploymentException,\ "One or more Script(s) reported a NonZeroDeployment:%s"\ % full_deploy_output,\ sys.exc_info()[2] #TODO: Check if all exceptions thrown at this time #fall in this category, and possibly don't retry if #you hit the Exception block below this. _deploy_init_to.retry(exc=exc) except SystemExit as bad_ssh: logger.exception("ERROR: Someone has raised a SystemExit!") _deploy_init_to.retry(exc=bad_ssh) except Exception as exc: logger.exception(exc) _deploy_init_to.retry(exc=exc)
def deploy_failed(driverCls, provider, identity, instance_id, task_uuid): try: logger.debug("deploy_failed task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) update_instance_metadata(driver, instance, data={'tmp_status': 'deploy_error'}, replace=False) logger.debug("deploy_failed task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) deploy_failed.retry(exc=exc)
def detach_task( driverCls, provider, identity, instance_id, volume_id, *args, **kwargs ): try: celery_logger.debug("detach_task started at %s." % timezone.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) driver.detach_volume(volume) # When the reslt returns the volume will be 'detaching' # We will ensure the volume does not return to 'in-use' attempts = 0 while True: volume = driver.get_volume(volume_id) if attempts > 6: # After 6 attempts (~1min) break # The Openstack way if isinstance(driver, OSDriver)\ and 'detaching' not in volume.extra['status']: break # The Eucalyptus way attach_data = volume.extra['attachments'][0] if isinstance(driver, EucaDriver) and attach_data\ and 'detaching' not in attach_data.get('status'): break # Exponential backoff.. attempts += 1 sleep_time = 2**attempts celery_logger.debug( "Volume %s is not ready (%s). Sleep for %s" % (volume.id, volume.extra['status'], sleep_time) ) time.sleep(sleep_time) if 'in-use' in volume.extra['status']: raise Exception( "Failed to detach Volume %s to instance %s" % (volume, instance) ) celery_logger.debug("detach_task finished at %s." % timezone.now()) except DeviceBusyException: # We should NOT retry if the device is busy raise except Exception as exc: # If the volume is NOT attached, do not retry. if 'Volume is not attached' in exc.message: return celery_logger.exception(exc) detach_task.retry(exc=exc)
def attach_task( driverCls, provider, identity, instance_id, volume_id, device_choice=None, *args, **kwargs ): celery_logger.debug("attach_task started at %s." % timezone.now()) driver = get_driver(driverCls, provider, identity) from service.volume import attach_volume attach_volume(driver, instance_id, volume_id, device_choice=device_choice) attempts = 0 while True: volume = driver.get_volume(volume_id) assert volume, "Volume ({}) does not exist".format(volume_id) volume_status = volume.extra.get('status', '') if volume_status == "in-use": break if attempts > 4: raise Exception( "Attach task timed out for volume {} and instance {}, volume status: {}" .format(volume_id, instance_id, volume_status) ) celery_logger.debug( "Volume {} is not ready. Expected 'in-use', got '{}'".format( volume_id, volume_status ) ) time.sleep(10) attempts += 1 try: attach_data = volume.extra['attachments'][0] device = attach_data['device'] except (IndexError, KeyError): raise Exception( "Could not find 'device' in volume.extra {}".format(volume.extra) ) celery_logger.debug("attach_task finished at %s." % timezone.now()) return device
def update_volume_metadata(driverCls, provider, identity, volume_alias, metadata): """ """ from service import volume as volume_service try: celery_logger.debug("update_volume_metadata task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_alias) if not volume: return return volume_service.update_volume_metadata(driver, volume, metadata=metadata) celery_logger.debug("volume_metadata task finished at %s." % datetime.now()) except Exception as exc: celery_logger.exception(exc) update_volume_metadata.retry(exc=exc)
def remove_empty_network( driverCls, provider, identity, core_identity_id, *args, **kwargs): try: #For testing ONLY.. Test cases ignore countdown.. if app.conf.CELERY_ALWAYS_EAGER: time.sleep(60) logger.debug("remove_empty_network task started at %s." % datetime.now()) logger.debug("CoreIdentity(id=%s)" % core_identity_id) core_identity = Identity.objects.get(id=core_identity_id) driver = get_driver(driverCls, provider, identity) instances = driver.list_instances() active_instances = False for instance in instances: if driver._is_active_instance(instance): active_instances = True break if not active_instances: inactive_instances = False for instance in instances: if driver._is_inactive_instance(instance): inactive_instances = True break #Inactive instances, True: Remove network, False remove_network = not inactive_instances #Check for project network from service.accounts.openstack import AccountDriver as\ OSAccountDriver os_acct_driver = OSAccountDriver(core_identity.provider) logger.info("No active instances. Removing project network" "from %s" % core_identity) os_acct_driver.delete_network(core_identity, remove_network=remove_network) if remove_network: #Sec. group can't be deleted if instances are suspended # when instances are suspended we pass remove_network=False os_acct_driver.delete_security_group(core_identity) return True logger.debug("remove_empty_network task finished at %s." % datetime.now()) return False except Exception as exc: logger.exception("Failed to check if project network is empty") remove_empty_network.retry(exc=exc)
def check_volume_task(driverCls, provider, identity, instance_id, volume_id, *args, **kwargs): try: celery_logger.debug("check_volume task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) attach_data = volume.extra['attachments'][0] device = attach_data['device'] private_key = ATMOSPHERE_PRIVATE_KEYFILE kwargs.update({'ssh_key': private_key}) kwargs.update({'timeout': 120}) # One script to make two checks: # 1. Voume exists 2. Volume has a filesystem cv_script = check_volume(device) # NOTE: non_zero_deploy needed to stop LibcloudDeploymentError from being # raised kwargs.update({'deploy': cv_script, 'non_zero_deploy': True}) driver.deploy_to(instance, **kwargs) kwargs.pop('non_zero_deploy', None) # Script execute if cv_script.exit_status != 0: if 'No such file' in cv_script.stdout: raise Exception('Volume check failed: %s. ' 'Device %s does not exist on instance %s' % (volume, device, instance)) elif 'Bad magic number' in cv_script.stdout: # Filesystem needs to be created for this device celery_logger.info("Mkfs needed") mkfs_script = mkfs_volume(device) kwargs.update({'deploy': mkfs_script}) driver.deploy_to(instance, **kwargs) else: raise Exception('Volume check failed: Something weird') celery_logger.debug("check_volume task finished at %s." % datetime.now()) except LibcloudDeploymentError as exc: celery_logger.exception(exc) except Exception as exc: celery_logger.warn(exc) check_volume_task.retry(exc=exc)
def remove_empty_network( driverCls, provider, identity, core_identity_id, *args, **kwargs): try: #For testing ONLY.. Test cases ignore countdown.. if app.conf.CELERY_ALWAYS_EAGER: logger.debug("Eager task waiting 1 minute") time.sleep(60) logger.debug("remove_empty_network task started at %s." % datetime.now()) logger.debug("CoreIdentity(id=%s)" % core_identity_id) core_identity = Identity.objects.get(id=core_identity_id) driver = get_driver(driverCls, provider, identity) instances = driver.list_instances() active_instances = False for instance in instances: if driver._is_active_instance(instance): active_instances = True break if not active_instances: inactive_instances = all(driver._is_inactive_instance( instance) for instance in instances) #Inactive instances, True: Remove network, False remove_network = not inactive_instances #Check for project network from service.accounts.openstack import AccountDriver as\ OSAccountDriver os_acct_driver = OSAccountDriver(core_identity.provider) logger.info("No active instances. Removing project network" "from %s" % core_identity) os_acct_driver.delete_network(core_identity, remove_network=remove_network) if remove_network: #Sec. group can't be deleted if instances are suspended # when instances are suspended we pass remove_network=False os_acct_driver.delete_security_group(core_identity) return True logger.debug("remove_empty_network task finished at %s." % datetime.now()) return False except Exception as exc: logger.exception("Failed to check if project network is empty") remove_empty_network.retry(exc=exc)
def _is_instance_ready( driverCls, provider, identity, instance_alias, status_query, tasks_allowed=False, return_id=False ): driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) if return_id: return None return False i_status = instance._node.extra["status"].lower() i_task = instance._node.extra["task"] if (i_status not in status_query) or (i_task and not tasks_allowed): raise Exception("Instance: %s: Status: (%s - %s) - Not Ready" % (instance.id, i_status, i_task)) logger.debug("Instance %s: Status: (%s - %s) - Ready" % (instance.id, i_status, i_task)) if return_id: return instance.id return True
def update_metadata(driverCls, provider, identity, instance_alias, metadata): """ #NOTE: While this looks like a large number (250 ?!) of retries # we expect this task to fail often when the image is building # and large, uncached images can have a build time """ try: logger.debug("update_metadata task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: return return update_instance_metadata( driver, instance, data=metadata, replace=False) logger.debug("update_metadata task finished at %s." % datetime.now()) except Exception as exc: logger.exception(exc) update_metadata.retry(exc=exc)
def check_volume_task( driverCls, provider, identity, instance_id, volume_id, device_type='ext4', *args, **kwargs ): try: celery_logger.debug("check_volume task started at %s." % timezone.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) username = identity.get_username() attach_data = volume.extra['attachments'][0] device_location = attach_data['device'] celery_logger.info("device_location: %s" % device_location) # One playbook to make two checks: # 1. Voume exists # 2. Volume has a filesystem # (If not, create one of type 'device_type') playbook_results = deploy_check_volume( instance.ip, username, instance.id, device_location, device_type=device_type ) success = not ( execution_has_failures(playbook_results) or execution_has_unreachable(playbook_results) ) if not success: raise Exception( "Error encountered while checking volume for filesystem: instance_id: {}, volume_id: {}" .format(instance_id, volume_id) ) return success except Exception as exc: celery_logger.warn(exc) check_volume_task.retry(exc=exc)
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id, **celery_task_args): from core.models.instance import Instance from core.email import send_deploy_failed_email try: logger.debug("deploy_failed task started at %s." % datetime.now()) logger.info("task_uuid=%s" % task_uuid) result = app.AsyncResult(task_uuid) with allow_join_result(): exc = result.get(propagate=False) err_str = "DEPLOYERROR::%s" % (result.traceback,) logger.error(err_str) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) update_instance_metadata(driver, instance, data={"tmp_status": "deploy_error"}, replace=False) logger.debug("deploy_failed task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) deploy_failed.retry(exc=exc)
def update_metadata(driverCls, provider, identity, instance_alias, metadata): """ #NOTE: While this looks like a large number (250 ?!) of retries # we expect this task to fail often when the image is building # and large, uncached images can have a build time """ try: logger.debug("update_metadata task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) #NOTE: This task will only be executed in TEST mode if app.conf.CELERY_ALWAYS_EAGER: eager_update_metadata(driver, instance, metadata) return update_instance_metadata( driver, instance, data=metadata, replace=False) logger.debug("update_metadata task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) update_metadata.retry(exc=exc)
def update_volume_metadata(driverCls, provider, identity, volume_alias, metadata): """ """ from service import volume as volume_service try: logger.debug("update_volume_metadata task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) volume = driver.get_volume(volume_alias) if not volume: return return volume_service.update_volume_metadata(driver, volume, metadata=metadata) logger.debug("volume_metadata task finished at %s." % datetime.now()) except Exception as exc: logger.exception(exc) update_volume_metadata.retry(exc=exc)
def complete_resize(driverCls, provider, identity, instance_alias, core_provider_id, core_identity_id, user): """ Confirm the resize of 'instance_alias' """ from service import instance as instance_service try: logger.debug("complete_resize task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return False, None result = instance_service.confirm_resize(driver, instance, core_provider_id, core_identity_id, user) logger.debug("complete_resize task finished at %s." % datetime.now()) return True, result except Exception as exc: logger.exception(exc) complete_resize.retry(exc=exc)
def step_task(driverCls, provider, identity, instance_id, step_id, *args, **kwargs): try: logger.debug("step task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) step = Step.objects.get(id=step_id) private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa" kwargs.update({"ssh_key": private_key}) kwargs.update({"timeout": 120}) script = step_script(step) kwargs.update({"deploy": script}) driver.deploy_to(instance, **kwargs) logger.debug("step task finished at %s." % datetime.now()) except Exception as exc: logger.warn(exc) step_task.retry(exc=exc)
def check_volume_task(driverCls, provider, identity, instance_id, volume_id, device_type='ext4', *args, **kwargs): try: celery_logger.debug("check_volume task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) volume = driver.get_volume(volume_id) username = identity.get_username() attach_data = volume.extra['attachments'][0] device_location = attach_data['device'] celery_logger.info("device_location: %s" % device_location) # One playbook to make two checks: # 1. Voume exists # 2. Volume has a filesystem # (If not, create one of type 'device_type') playbooks = deploy_check_volume(instance.ip, username, instance.id, device_location, device_type=device_type) celery_logger.info(playbooks.__dict__) hostname = build_host_name(instance.id, instance.ip) result = False if execution_has_failures(playbooks, hostname)\ or execution_has_unreachable(playbooks, hostname) else True if not result: raise Exception( "Error encountered while checking volume for filesystem: %s" % playbooks.stats.summarize(host=hostname)) return result except LibcloudDeploymentError as exc: celery_logger.exception(exc) except Exception as exc: celery_logger.warn(exc) check_volume_task.retry(exc=exc)
def check_process_task(driverCls, provider, identity, instance_alias, process_name, *args, **kwargs): """ #NOTE: While this looks like a large number (250 ?!) of retries # we expect this task to fail often when the image is building # and large, uncached images can have a build time. """ from core.models.instance import Instance try: logger.debug("check_process_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: return cp_script = check_process(process_name) kwargs.update({ 'ssh_key': ATMOSPHERE_PRIVATE_KEYFILE, 'timeout': 120, 'deploy': cp_script }) #Execute the script driver.deploy_to(instance, **kwargs) #Parse the output and modify the CORE instance script_out = cp_script.stdout result = True if "1:" in script_out else False #NOTE: Throws Instance.DoesNotExist core_instance = Instance.objects.get(provider_alias=instance_alias) if "vnc" in process_name: core_instance.vnc = result core_instance.save() elif "shellinaboxd" in process_name: core_instance.shell = result core_instance.save() else: return result, script_out logger.debug("check_process_task finished at %s." % datetime.now()) except Instance.DoesNotExist: logger.warn("check_process_task failed: Instance %s no longer exists" % instance_alias) except Exception as exc: logger.exception(exc) check_process_task.retry(exc=exc)
def complete_resize(driverCls, provider, identity, instance_alias, core_provider_id, core_identity_id, user): """ Confirm the resize of 'instance_alias' """ from service import instance as instance_service try: logger.debug("complete_resize task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return False, None result = instance_service.confirm_resize( driver, instance, core_provider_id, core_identity_id, user) logger.debug("complete_resize task finished at %s." % datetime.now()) return True, result except Exception as exc: logger.exception(exc) complete_resize.retry(exc=exc)
def check_process_task(driverCls, provider, identity, instance_alias, process_name, *args, **kwargs): """ #NOTE: While this looks like a large number (250 ?!) of retries # we expect this task to fail often when the image is building # and large, uncached images can have a build time. """ from core.models.instance import Instance try: logger.debug("check_process_task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: return cp_script = check_process(process_name) kwargs.update({ 'ssh_key': ATMOSPHERE_PRIVATE_KEYFILE, 'timeout': 120, 'deploy': cp_script}) #Execute the script driver.deploy_to(instance, **kwargs) #Parse the output and modify the CORE instance script_out = cp_script.stdout result = True if "1:" in script_out else False #NOTE: Throws Instance.DoesNotExist core_instance = Instance.objects.get(provider_alias=instance_alias) if "vnc" in process_name: core_instance.vnc = result core_instance.save() elif "shellinaboxd" in process_name: core_instance.shell = result core_instance.save() else: return result, script_out logger.debug("check_process_task finished at %s." % datetime.now()) except Instance.DoesNotExist: logger.warn("check_process_task failed: Instance %s no longer exists" % instance_alias) except Exception as exc: logger.exception(exc) check_process_task.retry(exc=exc)
def _is_instance_ready(driverCls, provider, identity, instance_alias, status_query, tasks_allowed=False, return_id=False): driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_alias) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) if return_id: return None return False i_status = instance._node.extra['status'].lower() i_task = instance._node.extra['task'] if (i_status not in status_query) or (i_task and not tasks_allowed): raise Exception( "Instance: %s: Status: (%s - %s) - Not Ready" % (instance.id, i_status, i_task)) logger.debug("Instance %s: Status: (%s - %s) - Ready" % (instance.id, i_status, i_task)) if return_id: return instance.id return True
def add_fixed_ip(driverCls, provider, identity, instance_id): from service import instance as instance_service try: logger.debug("add_fixed_ip task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) instance = driver.get_instance(instance_id) if not instance: logger.debug("Instance has been teminated: %s." % instance_id) return None if instance._node.private_ips: return instance network_id = instance_service._convert_network_name( driver, instance) fixed_ip = driver._connection.ex_add_fixed_ip(instance, network_id) logger.debug("add_fixed_ip task finished at %s." % datetime.now()) return fixed_ip except Exception as exc: if "Not Ready" not in str(exc): # Ignore 'normal' errors. logger.exception(exc) add_fixed_ip.retry(exc=exc)
def remove_empty_network(driverCls, provider, identity, core_identity_uuid, *args, **kwargs): try: #For testing ONLY.. Test cases ignore countdown.. if app.conf.CELERY_ALWAYS_EAGER: logger.debug("Eager task waiting 1 minute") time.sleep(60) logger.debug("remove_empty_network task started at %s." % datetime.now()) logger.debug("CoreIdentity(uuid=%s)" % core_identity_uuid) core_identity = Identity.objects.get(uuid=core_identity_uuid) driver = get_driver(driverCls, provider, identity) instances = driver.list_instances() active_instances = any( driver._is_active_instance(instance) for instance in instances) #If instances are active, we are done.. if not active_instances: inactive_instances = all( driver._is_inactive_instance(instance) for instance in instances) #Inactive instances: An instance that is 'stopped' or 'suspended' #Inactive instances, True: Remove network, False remove_network = not inactive_instances #Check for project network os_acct_driver = get_account_driver(core_identity.provider) logger.info("No active instances. Removing project network" "from %s" % core_identity) os_acct_driver.delete_network(core_identity, remove_network=remove_network) if remove_network: #Sec. group can't be deleted if instances are suspended # when instances are suspended we pass remove_network=False os_acct_driver.delete_security_group(core_identity) return True logger.debug("remove_empty_network task finished at %s." % datetime.now()) return False except Exception as exc: logger.exception("Exception occurred project network is empty")