Beispiel #1
0
def update_metadata(driverCls,
                    provider,
                    identity,
                    instance_alias,
                    metadata,
                    replace_metadata=False):
    """
    #NOTE: While this looks like a large number (250 ?!) of retries
    # we expect this task to fail often when the image is building
    # and large, uncached images can have a build time.
    """
    try:
        logger.debug("update_metadata task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_alias)
        if not instance:
            return
        #NOTE: This task will only be executed in TEST mode
        if app.conf.CELERY_ALWAYS_EAGER:
            eager_update_metadata(driver, instance, metadata)
        return update_instance_metadata(driver,
                                        instance,
                                        data=metadata,
                                        replace=replace_metadata)
        logger.debug("update_metadata task finished at %s." % datetime.now())
    except Exception as exc:
        logger.exception(exc)
        update_metadata.retry(exc=exc)
Beispiel #2
0
def _deploy_init_to(driverCls, provider, identity, instance_id, password=None,
                    **celery_task_args):
    try:
        logger.debug("_deploy_init_to task started at %s." % datetime.now())
        #Check if instance still exists
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        #NOTE: This is NOT the password passed by argument
        #Deploy with no password to use ssh keys
        logger.info(instance.extra)
        instance._node.extra['password'] = None
        kwargs = {}
        private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa"
        kwargs.update({'ssh_key': private_key})
        kwargs.update({'timeout': 120})
        msd = init(instance, identity.user.username, password)
        kwargs.update({'deploy': msd})
        driver.deploy_to(instance, **kwargs)
        logger.debug("_deploy_init_to task finished at %s." % datetime.now())
    except DeploymentError as exc:
        logger.exception(exc)
        if isinstance(exc.value, NonZeroDeploymentException):
            #The deployment was successful, but the return code on one or more
            # steps is bad. Log the exception and do NOT try again!
            raise exc.value
        #TODO: Check if all exceptions thrown at this time
        #fall in this category, and possibly don't retry if
        #you hit the Exception block below this.
        _deploy_init_to.retry(exc=exc)
    except Exception as exc:
        logger.exception(exc)
        _deploy_init_to.retry(exc=exc)
Beispiel #3
0
def umount_task(driverCls, provider, identity, instance_id, volume_id, *args, **kwargs):
    try:
        celery_logger.debug("umount_task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        attach_data = volume.extra["attachments"][0]
        device = attach_data["device"]

        # Check mount to find the mount_location for device
        private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa"
        kwargs.update({"ssh_key": private_key})
        kwargs.update({"timeout": 120})

        mount_location = None
        cm_script = check_mount()
        kwargs.update({"deploy": cm_script})
        driver.deploy_to(instance, **kwargs)
        regex = re.compile("(?P<device>[\w/]+) on (?P<location>.*) type")
        for line in cm_script.stdout.split("\n"):
            res = regex.search(line)
            if not res:
                continue
            search_dict = res.groupdict()
            dev_found = search_dict["device"]
            if device == dev_found:
                mount_location = search_dict["location"]
                break

        # Volume not mounted, move along..
        if not mount_location:
            return

        um_script = umount_volume(device)
        kwargs.update({"deploy": um_script})
        driver.deploy_to(instance, **kwargs)

        if "device is busy" in um_script.stdout:
            # Show all processes that are making device busy..
            lsof_script = lsof_location(mount_location)
            kwargs.update({"deploy": lsof_script})
            driver.deploy_to(instance, **kwargs)

            regex = re.compile("(?P<name>[\w]+)\s*(?P<pid>[\d]+)")
            offending_processes = []
            for line in lsof_script.stdout.split("\n"):
                res = regex.search(line)
                if not res:
                    continue
                search_dict = res.groupdict()
                offending_processes.append((search_dict["name"], search_dict["pid"]))

            raise DeviceBusyException(mount_location, offending_processes)
        # Return here if no errors occurred..
        celery_logger.debug("umount_task finished at %s." % datetime.now())
    except DeviceBusyException:
        raise
    except Exception as exc:
        celery_logger.warn(exc)
        umount_task.retry(exc=exc)
Beispiel #4
0
def _send_instance_email(driverCls, provider, identity, instance_id):
    try:
        logger.debug("_send_instance_email task started at %s." %
                     datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        #Breakout if instance has been deleted at this point
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        username = identity.user.username
        profile = UserProfile.objects.get(user__username=username)
        if profile.send_emails:
            #Only send emails if allowed by profile setting
            created = datetime.strptime(instance.extra['created'],
                                        "%Y-%m-%dT%H:%M:%SZ")
            send_instance_email(username,
                                instance.id,
                                instance.name,
                                instance.ip,
                                created,
                                username)
        else:
            logger.debug("User %s elected NOT to receive new instance emails"
                         % username)

        logger.debug("_send_instance_email task finished at %s." %
                     datetime.now())
    except Exception as exc:
        logger.warn(exc)
        _send_instance_email.retry(exc=exc)
Beispiel #5
0
def deploy_script(driverCls, provider, identity, instance_id,
                  script, **celery_task_args):
    try:
        logger.debug("deploy_script task started at %s." % datetime.now())
        #Check if instance still exists
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        instance._node.extra['password'] = None

        kwargs = _generate_ssh_kwargs()
        kwargs.update({'deploy': script})
        driver.deploy_to(instance, **kwargs)
        logger.debug("deploy_script task finished at %s." % datetime.now())
    except DeploymentError as exc:
        logger.exception(exc)
        if isinstance(exc.value, NonZeroDeploymentException):
            #The deployment was successful, but the return code on one or more
            # steps is bad. Log the exception and do NOT try again!
            raise exc.value
        #TODO: Check if all exceptions thrown at this time
        #fall in this category, and possibly don't retry if
        #you hit the Exception block below this.
        deploy_script.retry(exc=exc)
    except Exception as exc:
        logger.exception(exc)
        deploy_script.retry(exc=exc)
Beispiel #6
0
def update_mount_location(new_mount_location,
                          driverCls, provider, identity,
                          volume_alias):
    """
    """
    from service import volume as volume_service
    try:
        celery_logger.debug(
            "update_mount_location task started at %s." %
            datetime.now())
        driver = get_driver(driverCls, provider, identity)
        volume = driver.get_volume(volume_alias)
        if not volume:
            return
        if not new_mount_location:
            return
        volume_metadata = volume.extra['metadata']
        return volume_service._update_volume_metadata(
            driver, volume,
            metadata={'mount_location': new_mount_location})
        celery_logger.debug(
            "update_mount_location task finished at %s." %
            datetime.now())
    except Exception as exc:
        celery_logger.exception(exc)
        update_mount_location.retry(exc=exc)
Beispiel #7
0
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id,
                  **celery_task_args):
    from core.models.instance import Instance
    from core.email import send_deploy_failed_email
    try:
        logger.debug("deploy_failed task started at %s." % datetime.now())
        logger.info("task_uuid=%s" % task_uuid)
        result = app.AsyncResult(task_uuid)
        with allow_join_result():
            exc = result.get(propagate=False)
        err_str = "DEPLOYERROR::%s" % (result.traceback, )
        logger.error(err_str)
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        update_instance_metadata(driver,
                                 instance,
                                 data={'tmp_status': 'deploy_error'},
                                 replace=False)
        #Send deploy email
        core_instance = Instance.objects.get(provider_alias=instance_id)
        send_deploy_failed_email(core_instance, err_str)
        logger.debug("deploy_failed task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        deploy_failed.retry(exc=exc)
Beispiel #8
0
def _send_instance_email(driverCls, provider, identity, instance_id):
    try:
        logger.debug("_send_instance_email task started at %s." %
                     datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        #Breakout if instance has been deleted at this point
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        username = identity.user.username
        profile = UserProfile.objects.get(user__username=username)
        if profile.send_emails:
            #Only send emails if allowed by profile setting
            created = datetime.strptime(instance.extra['created'],
                                        "%Y-%m-%dT%H:%M:%SZ")
            send_instance_email(username,
                                instance.id,
                                instance.name,
                                instance.ip,
                                created,
                                username)
        else:
            logger.debug("User %s elected NOT to receive new instance emails"
                         % username)

        logger.debug("_send_instance_email task finished at %s." %
                     datetime.now())
    except Exception as exc:
        logger.warn(exc)
        _send_instance_email.retry(exc=exc)
Beispiel #9
0
def update_mount_location(new_mount_location,
                          driverCls, provider, identity,
                          volume_alias):
    """
    """
    from service import volume as volume_service
    try:
        celery_logger.debug(
            "update_mount_location task started at %s." %
            datetime.now())
        driver = get_driver(driverCls, provider, identity)
        volume = driver.get_volume(volume_alias)
        if not volume:
            return
        if not new_mount_location:
            return
        volume_metadata = volume.extra['metadata']
        return volume_service._update_volume_metadata(
            driver, volume,
            metadata={'mount_location': new_mount_location})
        celery_logger.debug(
            "update_mount_location task finished at %s." %
            datetime.now())
    except Exception as exc:
        celery_logger.exception(exc)
        update_mount_location.retry(exc=exc)
Beispiel #10
0
def mount_volume_task(
    driverCls,
    provider,
    identity,
    instance_id,
    volume_id,
    device_location,
    mount_location,
    device_type,
    mount_prefix=None,
    *args,
    **kwargs
):
    try:
        celery_logger.debug("mount task started at %s." % timezone.now())
        celery_logger.debug("mount_location: %s" % (mount_location, ))
        driver = get_driver(driverCls, provider, identity)
        username = identity.get_username()
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)

        try:
            attach_data = volume.extra['attachments'][0]
            if not device_location:
                device_location = attach_data['device']
        except (KeyError, IndexError):
            celery_logger.warn(
                "Volume %s missing attachments in Extra" % (volume, )
            )
        if not device_location:
            raise Exception(
                "No device_location found or inferred by volume %s" % volume
            )
        if not mount_prefix:
            mount_prefix = "/vol_"

        last_char = device_location[-1]    # /dev/sdb --> b
        if not mount_location:
            mount_location = mount_prefix + last_char

        playbook_results = deploy_mount_volume(
            instance.ip,
            username,
            instance.id,
            device_location,
            mount_location=mount_location,
            device_type=device_type
        )
        celery_logger.info(playbook_results)
        if execution_has_failures(
            playbook_results
        ) or execution_has_unreachable(playbook_results):
            raise Exception(
                "Error encountered while mounting volume: instance_id: {}, volume_id: {}"
                .format(instance_id, volume_id)
            )
        return mount_location
    except Exception as exc:
        celery_logger.warn(exc)
        mount_volume_task.retry(exc=exc)
Beispiel #11
0
def deploy_init_to(driverCls, provider, identity, instance_id,
                   username=None, password=None, redeploy=False, *args, **kwargs):
    try:
        logger.debug("deploy_init_to task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        image_metadata = driver._connection\
                               .ex_get_image_metadata(instance.machine)
        deploy_chain = get_deploy_chain(driverCls, provider, identity,
                                        instance, username, password, redeploy)
        deploy_chain.apply_async()
        #Can be really useful when testing.
        #if kwargs.get('delay'):
        #    async.get()
        logger.debug("deploy_init_to task finished at %s." % datetime.now())
    except SystemExit:
        logger.exception("System Exits are BAD! Find this and get rid of it!")
        raise Exception("System Exit called")
    except NonZeroDeploymentException:
        raise
    except Exception as exc:
        logger.warn(exc)
        deploy_init_to.retry(exc=exc)
Beispiel #12
0
def deploy_script(driverCls, provider, identity, instance_id,
                   script, **celery_task_args):
    try:
        logger.debug("deploy_script task started at %s." % datetime.now())
        #Check if instance still exists
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        instance._node.extra['password'] = None

        kwargs = _generate_ssh_kwargs()
        kwargs.update({'deploy': script})
        driver.deploy_to(instance, **kwargs)
        logger.debug("deploy_script task finished at %s." % datetime.now())
    except DeploymentError as exc:
        logger.exception(exc)
        if isinstance(exc.value, NonZeroDeploymentException):
            #The deployment was successful, but the return code on one or more
            # steps is bad. Log the exception and do NOT try again!
            raise exc.value
        #TODO: Check if all exceptions thrown at this time
        #fall in this category, and possibly don't retry if
        #you hit the Exception block below this.
        deploy_script.retry(exc=exc)
    except Exception as exc:
        logger.exception(exc)
        deploy_script.retry(exc=exc)
Beispiel #13
0
def add_fixed_ip(driverCls,
                 provider,
                 identity,
                 instance_id,
                 core_identity_id=None):
    from service import instance as instance_service
    try:
        logger.debug("add_fixed_ip task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return None
        if instance._node.private_ips:
            #TODO: Attempt to rescue
            logger.info("Instance has fixed IP: %s" % instance_id)
            return instance

        network_id = instance_service._get_network_id(driver, instance)
        fixed_ip = driver._connection.ex_add_fixed_ip(instance, network_id)
        logger.debug("add_fixed_ip task finished at %s." % datetime.now())
        return fixed_ip
    except Exception as exc:
        if "Not Ready" not in str(exc):
            # Ignore 'normal' errors.
            logger.exception(exc)
        add_fixed_ip.retry(exc=exc)
Beispiel #14
0
def _deploy_init_to(driverCls, provider, identity, instance_id,
                    username=None, password=None, redeploy=False,
                    **celery_task_args):
    try:
        logger.debug("_deploy_init_to task started at %s." % datetime.now())
        #Check if instance still exists
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return

        #NOTE: This is unrelated to the password argument
        logger.info(instance.extra)
        instance._node.extra['password'] = None
        msd = init(instance, identity.user.username, password, redeploy)

        kwargs = _generate_ssh_kwargs()
        kwargs.update({'deploy': msd})
        driver.deploy_to(instance, **kwargs)
        _update_status_log(instance, "Deploy Finished")
        logger.debug("_deploy_init_to task finished at %s." % datetime.now())
    except DeploymentError as exc:
        logger.exception(exc)
        if isinstance(exc.value, NonZeroDeploymentException):
            #The deployment was successful, but the return code on one or more
            # steps is bad. Log the exception and do NOT try again!
            raise exc.value
        #TODO: Check if all exceptions thrown at this time
        #fall in this category, and possibly don't retry if
        #you hit the Exception block below this.
        _deploy_init_to.retry(exc=exc)
    except Exception as exc:
        logger.exception(exc)
        _deploy_init_to.retry(exc=exc)
Beispiel #15
0
def deploy_init_to(driverCls, provider, identity, instance_id,
                   username=None, password=None, redeploy=False,
                   *args, **kwargs):
    try:
        logger.debug("deploy_init_to task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        image_metadata = driver._connection\
                               .ex_get_image_metadata(instance.machine)
        deploy_chain = get_deploy_chain(driverCls, provider, identity,
                                        instance, username, password, redeploy)
        deploy_chain.apply_async()
        #Can be really useful when testing.
        #if kwargs.get('delay'):
        #    async.get()
        logger.debug("deploy_init_to task finished at %s." % datetime.now())
    except SystemExit:
        logger.exception("System Exits are BAD! Find this and get rid of it!")
        raise Exception("System Exit called")
    except NonZeroDeploymentException as non_zero:
        logger.error(str(non_zero))
        logger.error(non_zero.__dict__)
        raise
    except Exception as exc:
        logger.warn(exc)
        deploy_init_to.retry(exc=exc)
Beispiel #16
0
def mount_failed(
    context,
    exception_msg,
    traceback,
    driverCls,
    provider,
    identity,
    volume_id,
    unmount=False,
    **celery_task_args
):
    from service import volume as volume_service
    try:
        celery_logger.debug("mount_failed task started at %s." % timezone.now())
        celery_logger.info("task context=%s" % context)
        err_str = "%s\nMount Error Traceback:%s" % (exception_msg, traceback)
        celery_logger.error(err_str)
        driver = get_driver(driverCls, provider, identity)
        volume = driver.get_volume(volume_id)
        if unmount:
            tmp_status = 'umount_error'
        else:
            tmp_status = 'mount_error'
        return volume_service._update_volume_metadata(
            driver, volume, metadata={'tmp_status': tmp_status}
        )
    except Exception as exc:
        celery_logger.warn(exc)
        mount_failed.retry(exc=exc)
Beispiel #17
0
def attach_task(driverCls,
                provider,
                identity,
                instance_id,
                volume_id,
                device_choice=None,
                *args,
                **kwargs):
    #TODO: chain task attach THEN mount for more robust-ness
    try:
        logger.debug("attach_task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)

        #Step 1. Attach the volume
        #NOTE: device_choice !== device 100%
        driver.attach_volume(instance, volume, device_choice)

        #When the reslt returns the volume will be 'attaching'
        #We can't do anything until the volume is 'available/in-use'
        attempts = 0
        while True:
            volume = driver.get_volume(volume_id)
            # Give up if you can't find the volume
            if not volume:
                return None
            if attempts > 6:  # After 6 attempts (~1min)
                break
            #Openstack Check
            if isinstance(driver, OSDriver) and\
                    'attaching' not in volume.extra.get('status',''):
                break
            attach_set = volume.extra['attachmentSet'][0]
            if isinstance(driver, EucaDriver) and\
                    'attaching' not in attach_set.get('status', ''):
                break
            # Exponential backoff..
            attempts += 1
            sleep_time = 2**attempts
            logger.debug("Volume %s is not ready (%s). Sleep for %s" %
                         (volume.id, volume.extra.get(
                             'status', 'no-status'), sleep_time))
            time.sleep(sleep_time)

        if 'available' in volume.extra.get('status', ''):
            raise Exception("Volume %s failed to attach to instance %s" %
                            (volume.id, instance.id))

        #Device path for euca == openstack
        try:
            device = volume.extra['attachmentSet'][0]['device']
        except:
            device = None

        logger.debug("attach_task finished at %s." % datetime.now())
        return device
    except Exception as exc:
        logger.warn(exc)
        attach_task.retry(exc=exc)
Beispiel #18
0
def mount_task(driverCls,
               provider,
               identity,
               instance_id,
               volume_id,
               mount_location=None,
               *args,
               **kwargs):
    try:
        logger.debug("mount task started at %s." % datetime.now())
        logger.debug("mount_location: %s" % (mount_location, ))
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        logger.debug(volume)
        try:
            device = volume.extra['attachmentSet'][0]['device']
        except:
            device = None
        if not device:
            #Device was never attached -- Nothing to mount
            return

        private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa"
        kwargs.update({'ssh_key': private_key})
        kwargs.update({'timeout': 120})

        #Step 2. Check the volume is not already mounted
        cm_script = check_mount()
        kwargs.update({'deploy': cm_script})
        driver.deploy_to(instance, **kwargs)

        if device in cm_script.stdout:
            #Device has already been mounted. Move along..
            return

        #Step 3. Find a suitable location to mount the volume
        logger.info("Original mount location - %s" % mount_location)
        if not mount_location:
            inc = 1
            while True:
                if '/vol%s' % inc in cm_script.stdout:
                    inc += 1
                else:
                    break
            mount_location = '/vol%s' % inc

        logger.info("Device location - %s" % device)
        logger.info("New mount location - %s" % mount_location)

        mv_script = mount_volume(device, mount_location)
        kwargs.update({'deploy': mv_script})
        driver.deploy_to(instance, **kwargs)
        #TODO: Update metadata to include volume mount
        # so this data can persist on the UI?
        logger.debug("mount task finished at %s." % datetime.now())
        return mount_location
    except Exception as exc:
        logger.warn(exc)
        mount_task.retry(exc=exc)
Beispiel #19
0
def mount_failed(task_uuid,
                 driverCls,
                 provider,
                 identity,
                 volume_id,
                 unmount=False,
                 **celery_task_args):
    try:
        logger.debug("mount_failed task started at %s." % datetime.now())
        logger.info("task_uuid=%s" % task_uuid)
        result = app.AsyncResult(task_uuid)
        with allow_join_result():
            exc = result.get(propagate=False)
        err_str = "Mount Error Traceback:%s" % (result.traceback, )
        logger.error(err_str)
        driver = get_driver(driverCls, provider, identity)
        volume = driver.get_volume(volume_id)
        if unmount:
            tmp_status = 'umount_error'
        else:
            tmp_status = 'mount_error'
        return volume_service.update_volume_metadata(
            driver, volume, metadata={'tmp_status': tmp_status})
        logger.debug("mount_failed task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        mount_failed.retry(exc=exc)
Beispiel #20
0
def unmount_volume_task(driverCls, provider, identity, instance_id, volume_id,
                        *args, **kwargs):
    try:
        celery_logger.debug("unmount task started at %s." % timezone.now())
        driver = get_driver(driverCls, provider, identity)
        username = identity.get_username()
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        device_location = None

        try:
            attach_data = volume.extra['attachments'][0]
            device_location = attach_data['device']
        except (KeyError, IndexError):
            celery_logger.warn("Volume %s missing attachments in Extra" %
                               (volume, ))
        if not device_location:
            raise Exception(
                "No device_location found or inferred by volume %s" % volume)
        try:
            playbooks = deploy_unmount_volume(instance.ip, username,
                                              instance.id, device_location)
        except DeviceBusyException:
            # Future-Fixme: Update VolumeStatusHistory.extra, set status to 'unmount_failed'
            raise
        hostname = build_host_name(instance.id, instance.ip)
        result = False if execution_has_failures(playbooks, hostname)\
            or execution_has_unreachable(playbooks, hostname) else True
        if not result:
            raise Exception("Error encountered while unmounting volume: %s" %
                            playbooks.stats.summarize(host=hostname))
        return device_location
    except Exception as exc:
        celery_logger.warn(exc)
        unmount_volume_task.retry(exc=exc)
Beispiel #21
0
def attach_task(driverCls,
                provider,
                identity,
                instance_id,
                volume_id,
                device_choice=None,
                *args,
                **kwargs):
    try:
        logger.debug("attach_task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        #Step 1. Attach the volume
        #NOTE: device_choice !== device 100%
        driver.attach_volume(instance, volume, device_choice)

        #When the reslt returns the volume will be 'attaching'
        #We can't do anything until the volume is 'available/in-use'
        attempts = 0
        while True:
            volume = driver.get_volume(volume_id)
            # Give up if you can't find the volume
            if not volume:
                return None
            if attempts > 6:  # After 6 attempts (~1min)
                break
            #Openstack Check
            if isinstance(driver, OSDriver) and\
                    'attaching' not in volume.extra.get('status',''):
                break
            if isinstance(driver, EucaDriver) and\
                    'attaching' not in volume.extra.get('status', ''):
                break
            # Exponential backoff..
            attempts += 1
            sleep_time = 2**attempts
            logger.debug("Volume %s is not ready (%s). Sleep for %s" %
                         (volume.id, volume.extra.get(
                             'status', 'no-status'), sleep_time))
            time.sleep(sleep_time)

        if 'available' in volume.extra.get('status', ''):
            raise Exception("Volume %s failed to attach to instance %s" %
                            (volume.id, instance.id))

        #Device path for euca == openstack
        try:
            attach_data = volume.extra['attachments'][0]
            device = attach_data['device']
        except (IndexError, KeyError), bad_fetch:
            logger.warn("Could not find 'device' in "
                        "volume.extra['attachments']: "
                        "Volume:%s Extra:%s" % (volume.id, volume.extra))
            device = None

        logger.debug("attach_task finished at %s." % datetime.now())
        return device
def attach_task(driverCls, provider, identity, instance_id, volume_id,
                device_choice=None, *args, **kwargs):
    try:
        celery_logger.debug("attach_task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        # Step 1. Attach the volume
        # NOTE: device_choice !== device 100%
        driver.attach_volume(instance,
                             volume,
                             device_choice)

        # When the reslt returns the volume will be 'attaching'
        # We can't do anything until the volume is 'available/in-use'
        attempts = 0
        while True:
            volume = driver.get_volume(volume_id)
            # Give up if you can't find the volume
            if not volume:
                return None
            if attempts > 6:  # After 6 attempts (~1min)
                break
            # Openstack Check
            if isinstance(driver, OSDriver) and\
                    'attaching' not in volume.extra.get('status', ''):
                break
            if isinstance(driver, EucaDriver) and\
                    'attaching' not in volume.extra.get('status', ''):
                break
            # Exponential backoff..
            attempts += 1
            sleep_time = 2**attempts
            celery_logger.debug("Volume %s is not ready (%s). Sleep for %s"
                         % (volume.id, volume.extra.get('status', 'no-status'),
                            sleep_time))
            time.sleep(sleep_time)

        if 'available' in volume.extra.get('status', ''):
            raise Exception("Volume %s failed to attach to instance %s"
                            % (volume.id, instance.id))

        # Device path for euca == openstack
        try:
            attach_data = volume.extra['attachments'][0]
            device = attach_data['device']
        except (IndexError, KeyError) as bad_fetch:
            celery_logger.warn("Could not find 'device' in "
                        "volume.extra['attachments']: "
                        "Volume:%s Extra:%s" % (volume.id, volume.extra))
            device = None

        celery_logger.debug("attach_task finished at %s." % datetime.now())
        return device
    except Exception as exc:
        celery_logger.warn(exc)
        attach_task.retry(exc=exc)
Beispiel #23
0
def attach_task(driverCls, provider, identity, instance_id, volume_id,
                device_choice=None, *args, **kwargs):
    #TODO: chain task attach THEN mount for more robust-ness
    try:
        logger.debug("attach_task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)

        #Step 1. Attach the volume
        #NOTE: device_choice !== device 100%
        driver.attach_volume(instance,
                             volume,
                             device_choice)

        #When the reslt returns the volume will be 'attaching'
        #We can't do anything until the volume is 'available/in-use'
        attempts = 0
        while True:
            volume = driver.get_volume(volume_id)
            # Give up if you can't find the volume
            if not volume:
                return None
            if attempts > 6:  # After 6 attempts (~1min)
                break
            #Openstack Check
            if isinstance(driver, OSDriver) and\
                    'attaching' not in volume.extra.get('status',''):
                break
            attach_set = volume.extra['attachmentSet'][0]
            if isinstance(driver, EucaDriver) and\
                    'attaching' not in attach_set.get('status', ''):
                break
            # Exponential backoff..
            attempts += 1
            sleep_time = 2**attempts
            logger.debug("Volume %s is not ready (%s). Sleep for %s"
                         % (volume.id, volume.extra.get('status', 'no-status'),
                            sleep_time))
            time.sleep(sleep_time)

        if 'available' in volume.extra.get('status',''):
            raise Exception("Volume %s failed to attach to instance %s"
                            % (volume.id, instance.id))

        #Device path for euca == openstack
        try:
            device = volume.extra['attachmentSet'][0]['device']
        except:
            device = None

        logger.debug("attach_task finished at %s." % datetime.now())
        return device
    except Exception as exc:
        logger.warn(exc)
        attach_task.retry(exc=exc)
Beispiel #24
0
def clean_empty_ips(driverCls, provider, identity, *args, **kwargs):
    try:
        logger.debug("remove_floating_ip task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        ips_cleaned = driver._clean_floating_ip()
        logger.debug("remove_floating_ip task finished at %s." % datetime.now())
        return ips_cleaned
    except Exception as exc:
        logger.warn(exc)
        clean_empty_ips.retry(exc=exc)
Beispiel #25
0
def deploy_to(driverCls, provider, identity, instance_id, *args, **kwargs):
    try:
        logger.debug("deploy_to task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        driver.deploy_to(instance, *args, **kwargs)
        logger.debug("deploy_to task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        deploy_to.retry(exc=exc)
Beispiel #26
0
def deploy_to(driverCls, provider, identity, instance_id, *args, **kwargs):
    try:
        logger.debug("deploy_to task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        driver.deploy_to(instance, *args, **kwargs)
        logger.debug("deploy_to task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        deploy_to.retry(exc=exc)
Beispiel #27
0
def add_floating_ip(driverCls, provider, identity,
                    instance_alias, delete_status=True,
                    *args, **kwargs):
    #For testing ONLY.. Test cases ignore countdown..
    if app.conf.CELERY_ALWAYS_EAGER:
        logger.debug("Eager task waiting 15 seconds")
        time.sleep(15)
    try:
        logger.debug("add_floating_ip task started at %s." % datetime.now())
        #Remove unused floating IPs first, so they can be re-used
        driver = get_driver(driverCls, provider, identity)
        driver._clean_floating_ip()

        #assign if instance doesn't already have an IP addr
        instance = driver.get_instance(instance_alias)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_alias)
            return None
        floating_ips = driver._connection.neutron_list_ips(instance)
        if floating_ips:
            floating_ip = floating_ips[0]["floating_ip_address"]
        else:
            floating_ip = driver._connection.neutron_associate_ip(
                instance, *args, **kwargs)["floating_ip_address"]
        _update_status_log(instance, "Networking Complete")
        #TODO: Implement this as its own task, with the result from
        #'floating_ip' passed in. Add it to the deploy_chain before deploy_to
        hostname = ""
        if floating_ip.startswith('128.196'):
            regex = re.compile(
                "(?P<one>[0-9]+)\.(?P<two>[0-9]+)\."
                "(?P<three>[0-9]+)\.(?P<four>[0-9]+)")
            r = regex.search(floating_ip)
            (one, two, three, four) = r.groups()
            hostname = "vm%s-%s.iplantcollaborative.org" % (three, four)
        else:
            # Find a way to convert new floating IPs to hostnames..
            hostname = floating_ip
        update_instance_metadata(driver, instance, data={
            'public-hostname': hostname,
            'public-ip':floating_ip}, replace=False)

        logger.info("Assigned IP:%s - Hostname:%s" % (floating_ip, hostname))
        #End
        logger.debug("add_floating_ip task finished at %s." % datetime.now())
        return {"floating_ip":floating_ip, "hostname":hostname}
    except Exception as exc:
        logger.exception("Error occurred while assigning a floating IP")
        #Networking can take a LONG time when an instance first launches,
        #it can also be one of those things you 'just miss' by a few seconds..
        #So we will retry 30 times using limited exp.backoff
        #Max Time: 53min
        countdown = min(2**current.request.retries, 128)
        add_floating_ip.retry(exc=exc,
                              countdown=countdown)
Beispiel #28
0
def add_floating_ip(driverCls, provider, identity,
                    instance_alias, delete_status=True,
                    *args, **kwargs):
    #For testing ONLY.. Test cases ignore countdown..
    if app.conf.CELERY_ALWAYS_EAGER:
        logger.debug("Eager task waiting 15 seconds")
        time.sleep(15)
    try:
        logger.debug("add_floating_ip task started at %s." % datetime.now())
        #Remove unused floating IPs first, so they can be re-used
        driver = get_driver(driverCls, provider, identity)
        driver._clean_floating_ip()

        #assign if instance doesn't already have an IP addr
        instance = driver.get_instance(instance_alias)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_alias)
            return None
        floating_ips = driver._connection.neutron_list_ips(instance)
        if floating_ips:
            floating_ip = floating_ips[0]["floating_ip_address"]
        else:
            floating_ip = driver._connection.neutron_associate_ip(
                instance, *args, **kwargs)["floating_ip_address"]
        _update_status_log(instance, "Networking Complete")
        #TODO: Implement this as its own task, with the result from
        #'floating_ip' passed in. Add it to the deploy_chain before deploy_to
        hostname = ""
        if floating_ip.startswith('128.196'):
            regex = re.compile(
                "(?P<one>[0-9]+)\.(?P<two>[0-9]+)\."
                "(?P<three>[0-9]+)\.(?P<four>[0-9]+)")
            r = regex.search(floating_ip)
            (one, two, three, four) = r.groups()
            hostname = "vm%s-%s.iplantcollaborative.org" % (three, four)
        else:
            # Find a way to convert new floating IPs to hostnames..
            hostname = floating_ip
        update_instance_metadata(driver, instance, data={
            'public-hostname': hostname,
            'public-ip': floating_ip}, replace=False)

        logger.info("Assigned IP:%s - Hostname:%s" % (floating_ip, hostname))
        #End
        logger.debug("add_floating_ip task finished at %s." % datetime.now())
        return {"floating_ip": floating_ip, "hostname": hostname}
    except Exception as exc:
        logger.exception("Error occurred while assigning a floating IP")
        #Networking can take a LONG time when an instance first launches,
        #it can also be one of those things you 'just miss' by a few seconds..
        #So we will retry 30 times using limited exp.backoff
        #Max Time: 53min
        countdown = min(2**current.request.retries, 128)
        add_floating_ip.retry(exc=exc,
                              countdown=countdown)
Beispiel #29
0
def _deploy_init_to(driverCls,
                    provider,
                    identity,
                    instance_id,
                    username=None,
                    password=None,
                    token=None,
                    redeploy=False,
                    **celery_task_args):
    #Note: Splitting preperation (Of the MultiScriptDeployment) and execution
    # This makes it easier to output scripts for debugging of users.
    try:
        logger.debug("_deploy_init_to task started at %s." % datetime.now())
        #Check if instance still exists
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return
        #NOTE: This is required to use ssh to connect.
        #TODO: Is this still necessary? What about times when we want to use
        # the adminPass? --Steve
        logger.info(instance.extra)
        instance._node.extra['password'] = None
        msd = init(instance, identity.user.username, password, token, redeploy)

    except Exception as exc:
        logger.exception(exc)
        _deploy_init_to.retry(exc=exc)
    try:
        kwargs = _generate_ssh_kwargs()
        kwargs.update({'deploy': msd})
        driver.deploy_to(instance, **kwargs)
        _update_status_log(instance, "Deploy Finished")
        logger.debug("_deploy_init_to task finished at %s." % datetime.now())
    except DeploymentError as exc:
        logger.exception(exc)
        full_deploy_output = _parse_steps_output(msd)
        if isinstance(exc.value, NonZeroDeploymentException):
            #The deployment was successful, but the return code on one or more
            # steps is bad. Log the exception and do NOT try again!
            raise NonZeroDeploymentException,\
                  "One or more Script(s) reported a NonZeroDeployment:%s"\
                          % full_deploy_output,\
                  sys.exc_info()[2]
        #TODO: Check if all exceptions thrown at this time
        #fall in this category, and possibly don't retry if
        #you hit the Exception block below this.
        _deploy_init_to.retry(exc=exc)
    except SystemExit as bad_ssh:
        logger.exception("ERROR: Someone has raised a SystemExit!")
        _deploy_init_to.retry(exc=bad_ssh)
    except Exception as exc:
        logger.exception(exc)
        _deploy_init_to.retry(exc=exc)
Beispiel #30
0
def clean_empty_ips(driverCls, provider, identity, *args, **kwargs):
    try:
        logger.debug("remove_floating_ip task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        ips_cleaned = driver._clean_floating_ip()
        logger.debug("remove_floating_ip task finished at %s." %
                     datetime.now())
        return ips_cleaned
    except Exception as exc:
        logger.warn(exc)
        clean_empty_ips.retry(exc=exc)
Beispiel #31
0
def mount_task(driverCls, provider, identity, instance_id, volume_id,
               mount_location=None, *args, **kwargs):
    try:
        logger.debug("mount task started at %s." % datetime.now())
        logger.debug("mount_location: %s" % (mount_location, ))
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        logger.debug(volume)
        try:
            device = volume.extra['attachmentSet'][0]['device']
        except:
            device = None
        if not device:
            #Device was never attached -- Nothing to mount
            return

        private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa"
        kwargs.update({'ssh_key': private_key})
        kwargs.update({'timeout': 120})

        #Step 2. Check the volume is not already mounted
        cm_script = check_mount()
        kwargs.update({'deploy': cm_script})
        driver.deploy_to(instance, **kwargs)

        if device in cm_script.stdout:
            #Device has already been mounted. Move along..
            return

        #Step 3. Find a suitable location to mount the volume
        logger.info("Original mount location - %s" % mount_location)
        if not mount_location:
            inc = 1
            while True:
                if '/vol%s' % inc in cm_script.stdout:
                    inc += 1
                else:
                    break
            mount_location = '/vol%s' % inc

        logger.info("Device location - %s" % device)
        logger.info("New mount location - %s" % mount_location)

        mv_script = mount_volume(device, mount_location)
        kwargs.update({'deploy': mv_script})
        driver.deploy_to(instance, **kwargs)
        #TODO: Update metadata to include volume mount
        # so this data can persist on the UI?
        logger.debug("mount task finished at %s." % datetime.now())
        return mount_location
    except Exception as exc:
        logger.warn(exc)
        mount_task.retry(exc=exc)
Beispiel #32
0
def deploy_failed(driverCls, provider, identity, instance_id, task_uuid):
    try:
        logger.debug("deploy_failed task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        update_instance_metadata(driver, instance,
                                 data={'tmp_status': 'deploy_error'},
                                 replace=False)
        logger.debug("deploy_failed task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        deploy_failed.retry(exc=exc)
Beispiel #33
0
def detach_task(
    driverCls, provider, identity, instance_id, volume_id, *args, **kwargs
):
    try:
        celery_logger.debug("detach_task started at %s." % timezone.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)

        driver.detach_volume(volume)
        # When the reslt returns the volume will be 'detaching'
        # We will ensure the volume does not return to 'in-use'
        attempts = 0
        while True:
            volume = driver.get_volume(volume_id)
            if attempts > 6:    # After 6 attempts (~1min)
                break
            # The Openstack way
            if isinstance(driver, OSDriver)\
                    and 'detaching' not in volume.extra['status']:
                break
            # The Eucalyptus way
            attach_data = volume.extra['attachments'][0]
            if isinstance(driver, EucaDriver) and attach_data\
                    and 'detaching' not in attach_data.get('status'):
                break
            # Exponential backoff..
            attempts += 1
            sleep_time = 2**attempts
            celery_logger.debug(
                "Volume %s is not ready (%s). Sleep for %s" %
                (volume.id, volume.extra['status'], sleep_time)
            )
            time.sleep(sleep_time)

        if 'in-use' in volume.extra['status']:
            raise Exception(
                "Failed to detach Volume %s to instance %s" %
                (volume, instance)
            )

        celery_logger.debug("detach_task finished at %s." % timezone.now())
    except DeviceBusyException:
        # We should NOT retry if the device is busy
        raise
    except Exception as exc:
        # If the volume is NOT attached, do not retry.
        if 'Volume is not attached' in exc.message:
            return
        celery_logger.exception(exc)
        detach_task.retry(exc=exc)
Beispiel #34
0
def attach_task(
    driverCls,
    provider,
    identity,
    instance_id,
    volume_id,
    device_choice=None,
    *args,
    **kwargs
):
    celery_logger.debug("attach_task started at %s." % timezone.now())
    driver = get_driver(driverCls, provider, identity)
    from service.volume import attach_volume
    attach_volume(driver, instance_id, volume_id, device_choice=device_choice)

    attempts = 0
    while True:
        volume = driver.get_volume(volume_id)
        assert volume, "Volume ({}) does not exist".format(volume_id)

        volume_status = volume.extra.get('status', '')
        if volume_status == "in-use":
            break

        if attempts > 4:
            raise Exception(
                "Attach task timed out for volume {} and instance {}, volume status: {}"
                .format(volume_id, instance_id, volume_status)
            )

        celery_logger.debug(
            "Volume {} is not ready. Expected 'in-use', got '{}'".format(
                volume_id, volume_status
            )
        )
        time.sleep(10)
        attempts += 1

    try:
        attach_data = volume.extra['attachments'][0]
        device = attach_data['device']
    except (IndexError, KeyError):
        raise Exception(
            "Could not find 'device' in volume.extra {}".format(volume.extra)
        )

    celery_logger.debug("attach_task finished at %s." % timezone.now())
    return device
Beispiel #35
0
def update_volume_metadata(driverCls, provider, identity, volume_alias, metadata):
    """
    """
    from service import volume as volume_service

    try:
        celery_logger.debug("update_volume_metadata task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        volume = driver.get_volume(volume_alias)
        if not volume:
            return
        return volume_service.update_volume_metadata(driver, volume, metadata=metadata)
        celery_logger.debug("volume_metadata task finished at %s." % datetime.now())
    except Exception as exc:
        celery_logger.exception(exc)
        update_volume_metadata.retry(exc=exc)
Beispiel #36
0
def remove_empty_network(
        driverCls, provider, identity,
        core_identity_id,
        *args, **kwargs):
    try:
        #For testing ONLY.. Test cases ignore countdown..
        if app.conf.CELERY_ALWAYS_EAGER:
            time.sleep(60)
        logger.debug("remove_empty_network task started at %s." %
                     datetime.now())

        logger.debug("CoreIdentity(id=%s)" % core_identity_id)
        core_identity = Identity.objects.get(id=core_identity_id)
        driver = get_driver(driverCls, provider, identity)
        instances = driver.list_instances()
        active_instances = False
        for instance in instances:
            if driver._is_active_instance(instance):
                active_instances = True
                break
        if not active_instances:
            inactive_instances = False
            for instance in instances:
                if driver._is_inactive_instance(instance):
                    inactive_instances = True
                    break
            #Inactive instances, True: Remove network, False
            remove_network = not inactive_instances
            #Check for project network
            from service.accounts.openstack import AccountDriver as\
                OSAccountDriver
            os_acct_driver = OSAccountDriver(core_identity.provider)
            logger.info("No active instances. Removing project network"
                        "from %s" % core_identity)
            os_acct_driver.delete_network(core_identity,
                    remove_network=remove_network)
            if remove_network:
                #Sec. group can't be deleted if instances are suspended
                # when instances are suspended we pass remove_network=False
                os_acct_driver.delete_security_group(core_identity)
            return True
        logger.debug("remove_empty_network task finished at %s." %
                     datetime.now())
        return False
    except Exception as exc:
        logger.exception("Failed to check if project network is empty")
        remove_empty_network.retry(exc=exc)
Beispiel #37
0
def check_volume_task(driverCls, provider, identity, instance_id, volume_id,
                      *args, **kwargs):
    try:
        celery_logger.debug("check_volume task started at %s." %
                            datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        attach_data = volume.extra['attachments'][0]
        device = attach_data['device']

        private_key = ATMOSPHERE_PRIVATE_KEYFILE
        kwargs.update({'ssh_key': private_key})
        kwargs.update({'timeout': 120})

        # One script to make two checks:
        # 1. Voume exists 2. Volume has a filesystem
        cv_script = check_volume(device)
        # NOTE: non_zero_deploy needed to stop LibcloudDeploymentError from being
        # raised
        kwargs.update({'deploy': cv_script, 'non_zero_deploy': True})
        driver.deploy_to(instance, **kwargs)
        kwargs.pop('non_zero_deploy', None)
        # Script execute

        if cv_script.exit_status != 0:
            if 'No such file' in cv_script.stdout:
                raise Exception('Volume check failed: %s. '
                                'Device %s does not exist on instance %s' %
                                (volume, device, instance))
            elif 'Bad magic number' in cv_script.stdout:
                # Filesystem needs to be created for this device
                celery_logger.info("Mkfs needed")
                mkfs_script = mkfs_volume(device)
                kwargs.update({'deploy': mkfs_script})
                driver.deploy_to(instance, **kwargs)
            else:
                raise Exception('Volume check failed: Something weird')

        celery_logger.debug("check_volume task finished at %s." %
                            datetime.now())
    except LibcloudDeploymentError as exc:
        celery_logger.exception(exc)
    except Exception as exc:
        celery_logger.warn(exc)
        check_volume_task.retry(exc=exc)
Beispiel #38
0
def check_volume_task(driverCls, provider, identity,
                      instance_id, volume_id, *args, **kwargs):
    try:
        celery_logger.debug("check_volume task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        attach_data = volume.extra['attachments'][0]
        device = attach_data['device']

        private_key = ATMOSPHERE_PRIVATE_KEYFILE
        kwargs.update({'ssh_key': private_key})
        kwargs.update({'timeout': 120})

        # One script to make two checks:
        # 1. Voume exists 2. Volume has a filesystem
        cv_script = check_volume(device)
        # NOTE: non_zero_deploy needed to stop LibcloudDeploymentError from being
        # raised
        kwargs.update({'deploy': cv_script,
                       'non_zero_deploy': True})
        driver.deploy_to(instance, **kwargs)
        kwargs.pop('non_zero_deploy', None)
        # Script execute

        if cv_script.exit_status != 0:
            if 'No such file' in cv_script.stdout:
                raise Exception('Volume check failed: %s. '
                                'Device %s does not exist on instance %s'
                                % (volume, device, instance))
            elif 'Bad magic number' in cv_script.stdout:
                # Filesystem needs to be created for this device
                celery_logger.info("Mkfs needed")
                mkfs_script = mkfs_volume(device)
                kwargs.update({'deploy': mkfs_script})
                driver.deploy_to(instance, **kwargs)
            else:
                raise Exception('Volume check failed: Something weird')

        celery_logger.debug("check_volume task finished at %s." % datetime.now())
    except LibcloudDeploymentError as exc:
        celery_logger.exception(exc)
    except Exception as exc:
        celery_logger.warn(exc)
        check_volume_task.retry(exc=exc)
Beispiel #39
0
def remove_empty_network(
        driverCls, provider, identity,
        core_identity_id,
        *args, **kwargs):
    try:
        #For testing ONLY.. Test cases ignore countdown..
        if app.conf.CELERY_ALWAYS_EAGER:
            logger.debug("Eager task waiting 1 minute")
            time.sleep(60)
        logger.debug("remove_empty_network task started at %s." %
                     datetime.now())

        logger.debug("CoreIdentity(id=%s)" % core_identity_id)
        core_identity = Identity.objects.get(id=core_identity_id)
        driver = get_driver(driverCls, provider, identity)
        instances = driver.list_instances()
        active_instances = False
        for instance in instances:
            if driver._is_active_instance(instance):
                active_instances = True
                break
        if not active_instances:
            inactive_instances = all(driver._is_inactive_instance(
                instance) for instance in instances)
            #Inactive instances, True: Remove network, False
            remove_network = not inactive_instances
            #Check for project network
            from service.accounts.openstack import AccountDriver as\
                OSAccountDriver
            os_acct_driver = OSAccountDriver(core_identity.provider)
            logger.info("No active instances. Removing project network"
                        "from %s" % core_identity)
            os_acct_driver.delete_network(core_identity,
                                          remove_network=remove_network)
            if remove_network:
                #Sec. group can't be deleted if instances are suspended
                # when instances are suspended we pass remove_network=False
                os_acct_driver.delete_security_group(core_identity)
            return True
        logger.debug("remove_empty_network task finished at %s." %
                     datetime.now())
        return False
    except Exception as exc:
        logger.exception("Failed to check if project network is empty")
        remove_empty_network.retry(exc=exc)
Beispiel #40
0
def _is_instance_ready(
    driverCls, provider, identity, instance_alias, status_query, tasks_allowed=False, return_id=False
):
    driver = get_driver(driverCls, provider, identity)
    instance = driver.get_instance(instance_alias)
    if not instance:
        logger.debug("Instance has been teminated: %s." % instance_id)
        if return_id:
            return None
        return False
    i_status = instance._node.extra["status"].lower()
    i_task = instance._node.extra["task"]
    if (i_status not in status_query) or (i_task and not tasks_allowed):
        raise Exception("Instance: %s: Status: (%s - %s) - Not Ready" % (instance.id, i_status, i_task))
    logger.debug("Instance %s: Status: (%s - %s) - Ready" % (instance.id, i_status, i_task))
    if return_id:
        return instance.id
    return True
Beispiel #41
0
def update_metadata(driverCls, provider, identity, instance_alias, metadata):
    """
    #NOTE: While this looks like a large number (250 ?!) of retries
    # we expect this task to fail often when the image is building
    # and large, uncached images can have a build time 
    """
    try:
        logger.debug("update_metadata task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_alias)
        if not instance:
            return
        return update_instance_metadata(
            driver, instance, data=metadata, replace=False)
        logger.debug("update_metadata task finished at %s." % datetime.now())
    except Exception as exc:
        logger.exception(exc)
        update_metadata.retry(exc=exc)
Beispiel #42
0
def check_volume_task(
    driverCls,
    provider,
    identity,
    instance_id,
    volume_id,
    device_type='ext4',
    *args,
    **kwargs
):
    try:
        celery_logger.debug("check_volume task started at %s." % timezone.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        username = identity.get_username()
        attach_data = volume.extra['attachments'][0]
        device_location = attach_data['device']
        celery_logger.info("device_location: %s" % device_location)

        # One playbook to make two checks:
        # 1. Voume exists
        # 2. Volume has a filesystem
        #    (If not, create one of type 'device_type')
        playbook_results = deploy_check_volume(
            instance.ip,
            username,
            instance.id,
            device_location,
            device_type=device_type
        )
        success = not (
            execution_has_failures(playbook_results)
            or execution_has_unreachable(playbook_results)
        )
        if not success:
            raise Exception(
                "Error encountered while checking volume for filesystem: instance_id: {}, volume_id: {}"
                .format(instance_id, volume_id)
            )
        return success
    except Exception as exc:
        celery_logger.warn(exc)
        check_volume_task.retry(exc=exc)
Beispiel #43
0
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id, **celery_task_args):
    from core.models.instance import Instance
    from core.email import send_deploy_failed_email

    try:
        logger.debug("deploy_failed task started at %s." % datetime.now())
        logger.info("task_uuid=%s" % task_uuid)
        result = app.AsyncResult(task_uuid)
        with allow_join_result():
            exc = result.get(propagate=False)
        err_str = "DEPLOYERROR::%s" % (result.traceback,)
        logger.error(err_str)
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        update_instance_metadata(driver, instance, data={"tmp_status": "deploy_error"}, replace=False)
        logger.debug("deploy_failed task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        deploy_failed.retry(exc=exc)
Beispiel #44
0
def update_metadata(driverCls, provider, identity, instance_alias, metadata):
    """
    #NOTE: While this looks like a large number (250 ?!) of retries
    # we expect this task to fail often when the image is building
    # and large, uncached images can have a build time 
    """
    try:
        logger.debug("update_metadata task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_alias)
        #NOTE: This task will only be executed in TEST mode
        if app.conf.CELERY_ALWAYS_EAGER:
            eager_update_metadata(driver, instance, metadata)
        return update_instance_metadata(
            driver, instance, data=metadata, replace=False)
        logger.debug("update_metadata task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        update_metadata.retry(exc=exc)
Beispiel #45
0
def update_volume_metadata(driverCls, provider, identity, volume_alias,
                           metadata):
    """
    """
    from service import volume as volume_service
    try:
        logger.debug("update_volume_metadata task started at %s." %
                     datetime.now())
        driver = get_driver(driverCls, provider, identity)
        volume = driver.get_volume(volume_alias)
        if not volume:
            return
        return volume_service.update_volume_metadata(driver,
                                                     volume,
                                                     metadata=metadata)
        logger.debug("volume_metadata task finished at %s." % datetime.now())
    except Exception as exc:
        logger.exception(exc)
        update_volume_metadata.retry(exc=exc)
Beispiel #46
0
def complete_resize(driverCls, provider, identity, instance_alias, core_provider_id, core_identity_id, user):
    """
    Confirm the resize of 'instance_alias'
    """
    from service import instance as instance_service

    try:
        logger.debug("complete_resize task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_alias)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return False, None
        result = instance_service.confirm_resize(driver, instance, core_provider_id, core_identity_id, user)
        logger.debug("complete_resize task finished at %s." % datetime.now())
        return True, result
    except Exception as exc:
        logger.exception(exc)
        complete_resize.retry(exc=exc)
Beispiel #47
0
def step_task(driverCls, provider, identity, instance_id, step_id, *args, **kwargs):
    try:
        logger.debug("step task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        step = Step.objects.get(id=step_id)

        private_key = "/opt/dev/atmosphere/extras/ssh/id_rsa"
        kwargs.update({"ssh_key": private_key})
        kwargs.update({"timeout": 120})
        script = step_script(step)
        kwargs.update({"deploy": script})

        driver.deploy_to(instance, **kwargs)

        logger.debug("step task finished at %s." % datetime.now())
    except Exception as exc:
        logger.warn(exc)
        step_task.retry(exc=exc)
Beispiel #48
0
def check_volume_task(driverCls,
                      provider,
                      identity,
                      instance_id,
                      volume_id,
                      device_type='ext4',
                      *args,
                      **kwargs):
    try:
        celery_logger.debug("check_volume task started at %s." %
                            datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        volume = driver.get_volume(volume_id)
        username = identity.get_username()
        attach_data = volume.extra['attachments'][0]
        device_location = attach_data['device']
        celery_logger.info("device_location: %s" % device_location)

        # One playbook to make two checks:
        # 1. Voume exists
        # 2. Volume has a filesystem
        #    (If not, create one of type 'device_type')
        playbooks = deploy_check_volume(instance.ip,
                                        username,
                                        instance.id,
                                        device_location,
                                        device_type=device_type)
        celery_logger.info(playbooks.__dict__)
        hostname = build_host_name(instance.id, instance.ip)
        result = False if execution_has_failures(playbooks, hostname)\
            or execution_has_unreachable(playbooks, hostname) else True
        if not result:
            raise Exception(
                "Error encountered while checking volume for filesystem: %s" %
                playbooks.stats.summarize(host=hostname))
        return result
    except LibcloudDeploymentError as exc:
        celery_logger.exception(exc)
    except Exception as exc:
        celery_logger.warn(exc)
        check_volume_task.retry(exc=exc)
Beispiel #49
0
def check_process_task(driverCls, provider, identity, instance_alias,
                       process_name, *args, **kwargs):
    """
    #NOTE: While this looks like a large number (250 ?!) of retries
    # we expect this task to fail often when the image is building
    # and large, uncached images can have a build time.
    """
    from core.models.instance import Instance
    try:
        logger.debug("check_process_task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_alias)
        if not instance:
            return
        cp_script = check_process(process_name)
        kwargs.update({
            'ssh_key': ATMOSPHERE_PRIVATE_KEYFILE,
            'timeout': 120,
            'deploy': cp_script
        })
        #Execute the script
        driver.deploy_to(instance, **kwargs)
        #Parse the output and modify the CORE instance
        script_out = cp_script.stdout
        result = True if "1:" in script_out else False
        #NOTE: Throws Instance.DoesNotExist
        core_instance = Instance.objects.get(provider_alias=instance_alias)
        if "vnc" in process_name:
            core_instance.vnc = result
            core_instance.save()
        elif "shellinaboxd" in process_name:
            core_instance.shell = result
            core_instance.save()
        else:
            return result, script_out
        logger.debug("check_process_task finished at %s." % datetime.now())
    except Instance.DoesNotExist:
        logger.warn("check_process_task failed: Instance %s no longer exists" %
                    instance_alias)
    except Exception as exc:
        logger.exception(exc)
        check_process_task.retry(exc=exc)
Beispiel #50
0
def complete_resize(driverCls, provider, identity, instance_alias,
                    core_provider_id, core_identity_id, user):
    """
    Confirm the resize of 'instance_alias'
    """
    from service import instance as instance_service
    try:
        logger.debug("complete_resize task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_alias)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return False, None
        result = instance_service.confirm_resize(
            driver, instance, core_provider_id, core_identity_id, user)
        logger.debug("complete_resize task finished at %s." % datetime.now())
        return True, result
    except Exception as exc:
        logger.exception(exc)
        complete_resize.retry(exc=exc)
Beispiel #51
0
def check_process_task(driverCls, provider, identity,
                       instance_alias, process_name, *args, **kwargs):
    """
    #NOTE: While this looks like a large number (250 ?!) of retries
    # we expect this task to fail often when the image is building
    # and large, uncached images can have a build time.
    """
    from core.models.instance import Instance
    try:
        logger.debug("check_process_task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_alias)
        if not instance:
            return
        cp_script = check_process(process_name)
        kwargs.update({
            'ssh_key': ATMOSPHERE_PRIVATE_KEYFILE,
            'timeout': 120,
            'deploy': cp_script})
        #Execute the script
        driver.deploy_to(instance, **kwargs)
        #Parse the output and modify the CORE instance
        script_out = cp_script.stdout
        result = True if "1:" in script_out else False
        #NOTE: Throws Instance.DoesNotExist
        core_instance = Instance.objects.get(provider_alias=instance_alias)
        if "vnc" in process_name:
            core_instance.vnc = result
            core_instance.save()
        elif "shellinaboxd" in process_name:
            core_instance.shell = result
            core_instance.save()
        else:
            return result, script_out
        logger.debug("check_process_task finished at %s." % datetime.now())
    except Instance.DoesNotExist:
        logger.warn("check_process_task failed: Instance %s no longer exists"
                    % instance_alias)
    except Exception as exc:
        logger.exception(exc)
        check_process_task.retry(exc=exc)
Beispiel #52
0
def _is_instance_ready(driverCls, provider, identity,
                       instance_alias, status_query,
                       tasks_allowed=False, return_id=False):
    driver = get_driver(driverCls, provider, identity)
    instance = driver.get_instance(instance_alias)
    if not instance:
        logger.debug("Instance has been teminated: %s." % instance_id)
        if return_id:
            return None
        return False
    i_status = instance._node.extra['status'].lower()
    i_task = instance._node.extra['task']
    if (i_status not in status_query) or (i_task and not tasks_allowed):
        raise Exception(
            "Instance: %s: Status: (%s - %s) - Not Ready"
            % (instance.id, i_status, i_task))
    logger.debug("Instance %s: Status: (%s - %s) - Ready"
                 % (instance.id, i_status, i_task))
    if return_id:
        return instance.id
    return True
Beispiel #53
0
def add_fixed_ip(driverCls, provider, identity, instance_id):
    from service import instance as instance_service
    try:
        logger.debug("add_fixed_ip task started at %s." % datetime.now())
        driver = get_driver(driverCls, provider, identity)
        instance = driver.get_instance(instance_id)
        if not instance:
            logger.debug("Instance has been teminated: %s." % instance_id)
            return None
        if instance._node.private_ips:
            return instance
        network_id = instance_service._convert_network_name(
                driver, instance)
        fixed_ip = driver._connection.ex_add_fixed_ip(instance, network_id)
        logger.debug("add_fixed_ip task finished at %s." % datetime.now())
        return fixed_ip
    except Exception as exc:
        if "Not Ready" not in str(exc):
            # Ignore 'normal' errors.
            logger.exception(exc)
        add_fixed_ip.retry(exc=exc)
Beispiel #54
0
def remove_empty_network(driverCls, provider, identity, core_identity_uuid,
                         *args, **kwargs):
    try:
        #For testing ONLY.. Test cases ignore countdown..
        if app.conf.CELERY_ALWAYS_EAGER:
            logger.debug("Eager task waiting 1 minute")
            time.sleep(60)
        logger.debug("remove_empty_network task started at %s." %
                     datetime.now())

        logger.debug("CoreIdentity(uuid=%s)" % core_identity_uuid)
        core_identity = Identity.objects.get(uuid=core_identity_uuid)
        driver = get_driver(driverCls, provider, identity)
        instances = driver.list_instances()
        active_instances = any(
            driver._is_active_instance(instance) for instance in instances)
        #If instances are active, we are done..
        if not active_instances:
            inactive_instances = all(
                driver._is_inactive_instance(instance)
                for instance in instances)
            #Inactive instances: An instance that is 'stopped' or 'suspended'
            #Inactive instances, True: Remove network, False
            remove_network = not inactive_instances
            #Check for project network
            os_acct_driver = get_account_driver(core_identity.provider)
            logger.info("No active instances. Removing project network"
                        "from %s" % core_identity)
            os_acct_driver.delete_network(core_identity,
                                          remove_network=remove_network)
            if remove_network:
                #Sec. group can't be deleted if instances are suspended
                # when instances are suspended we pass remove_network=False
                os_acct_driver.delete_security_group(core_identity)
            return True
        logger.debug("remove_empty_network task finished at %s." %
                     datetime.now())
        return False
    except Exception as exc:
        logger.exception("Exception occurred project network is empty")