def test_wait_for_log_after_execution_finishes(self):
        """wait_for_execution continues polling logs, after execution status
        is terminated
        """

        # prepare mock executions.get() calls - first return a status='started'
        # then continue returning status='terminated'
        executions = chain(
            [MagicMock(status=Execution.STARTED)],
            repeat(MagicMock(status=Execution.TERMINATED))
        )

        # prepare mock events.get() calls - first return empty events 100 times
        # and only then return a 'workflow_succeeded' event
        events = chain(
            repeat(([], 0), 100),
            [([{'event_type': 'workflow_succeeded'}], 1)],
            repeat(([], 0))
        )

        self.client.executions.get = MagicMock(side_effect=executions)
        self.client.events.get = MagicMock(side_effect=events)

        mock_execution = MagicMock(status=Execution.STARTED)
        wait_for_execution(self.client, mock_execution, timeout=None)

        calls_count = len(self.client.events.get.mock_calls)
        self.assertEqual(calls_count, 101, """wait_for_execution didnt keep
            polling events after execution terminated (expected 101
            calls, got %d)""" % calls_count)
    def test_wait_for_execution_after_log_succeeded(self):
        """wait_for_execution continues polling the execution status,
        even after it received a "workflow succeeded" log
        """

        # prepare mock executions.get() calls - return a status='started'
        # execution the first 100 times, and then return a 'terminated' one
        executions = chain(
            [MagicMock(status=Execution.STARTED)] * 100,
            repeat(MagicMock(status=Execution.TERMINATED))
        )

        # prepare mock events.get() calls - return a 'workflow_succeeded'
        # immediately, and there's no events after that
        events = chain(
            [([{'event_type': 'workflow_succeeded'}], 1)],
            repeat(([], 0))
        )

        self.client.executions.get = MagicMock(side_effect=executions)
        self.client.events.get = MagicMock(side_effect=events)

        mock_execution = MagicMock(status=Execution.STARTED)
        wait_for_execution(self.client, mock_execution, timeout=None)

        calls_count = len(self.client.executions.get.mock_calls)
        self.assertEqual(calls_count, 101, """wait_for_execution didnt keep
            polling the execution status after it received a workflow_succeeded
            event (expected 101 calls, got %d)""" % calls_count)
Example #3
0
    def worker(dep_id):
        try:
            execution = client.executions.start(dep_id, workflow_id)

            execution = wait_for_execution(
                client, execution, events_handler=threadsafe_events_logger, include_logs=include_logs, timeout=900
            )

            if execution.error:
                log_to_summary(
                    "Execution of workflow '{0}' for "
                    "deployment '{1}' failed. [error={2}]".format(workflow_id, dep_id, execution.error)
                )
            else:
                threadsafe_log(
                    "Finished executing workflow " "'{0}' on deployment" " '{1}'".format(workflow_id, dep_id)
                )

        except ExecutionTimeoutError as e:
            log_to_summary(
                "Execution of workflow '{0}' "
                "for deployment '{1}' timed out. "
                "* Run 'cfy executions cancel "
                "--execution-id {2}' to cancel"
                " the running workflow.".format(workflow_id, deployment_id, e.execution_id)
            )
Example #4
0
def update(blueprint_id, include_logs, json_output, logger, client,
           tenant_name, force):
    """Update the plugins of all the deployments of the given blueprint. This
    will update the deployments one by one until all succeeded.

    `BLUEPRINT_ID` the blueprint's ID to perform the plugins update with.
    """
    utils.explicit_tenant_name_message(tenant_name, logger)
    logger.info('Updating the plugins of the deployments of the blueprint '
                '{}'.format(blueprint_id))
    plugins_update = client.plugins_update.update_plugins(blueprint_id, force)
    events_logger = get_events_logger(json_output)
    execution = execution_events_fetcher.wait_for_execution(
        client,
        client.executions.get(plugins_update.execution_id),
        events_handler=events_logger,
        include_logs=include_logs,
        timeout=None  # don't timeout ever
    )

    if execution.error:
        logger.info("Execution of workflow '{0}' for blueprint "
                    "'{1}' failed. [error={2}]".format(execution.workflow_id,
                                                       blueprint_id,
                                                       execution.error))
        logger.info('Failed updating plugins for blueprint {0}. '
                    'Plugins update ID: {1}. Execution id: {2}'.format(
                        blueprint_id, plugins_update.id, execution.id))
        raise SuppressedCloudifyCliError()
    logger.info("Finished executing workflow '{0}'".format(
        execution.workflow_id))
    logger.info('Successfully updated plugins for blueprint {0}. '
                'Plugins update ID: {1}. Execution id: {2}'.format(
                    blueprint_id, plugins_update.id, execution.id))
Example #5
0
def wait_for_and_validate_execution(client, execution):
    execution = wait_for_execution(client,
                                   execution,
                                   get_events_logger(False),
                                   True,
                                   timeout=None,
                                   logger=logger)
    if execution.status != Execution.TERMINATED:
        raise Exception(
            "Unexpected status of execution {}: {} (expected: {})".format(
                execution.id, execution.status, Execution.TERMINATED))
    return execution
Example #6
0
    def worker(dep_id):
        timeout = 900

        try:
            kwargs = {}
            if install_script is not None:
                kwargs = {
                    'parameters': {
                        'install_script': install_script
                    },
                    'allow_custom_parameters': True
                }
            execution = client.executions.start(
                dep_id,
                workflow_id,
                **kwargs
            )

            execution = wait_for_execution(
                client,
                execution,
                events_handler=threadsafe_events_logger,
                include_logs=include_logs,
                timeout=timeout
            )

            if execution.error:
                log_to_summary("Execution of workflow '{0}' for "
                               "deployment '{1}' failed. [error={2}]"
                               .format(workflow_id,
                                       dep_id,
                                       execution.error))
            else:
                threadsafe_log("Finished executing workflow "
                               "'{0}' on deployment"
                               " '{1}'".format(workflow_id, dep_id))

        except ExecutionTimeoutError as e:
            log_to_summary(
                "Timed out waiting for workflow '{0}' of deployment '{1}' to "
                "end. The execution may still be running properly; however, "
                "the command-line utility was instructed to wait up to {3} "
                "seconds for its completion.\n\n"
                "* Run 'cfy executions list' to determine the execution's "
                "status.\n"
                "* Run 'cfy executions cancel --execution-id {2}' to cancel"
                " the running workflow.".format(
                    workflow_id, deployment_id, e.execution_id, timeout))
Example #7
0
def ls(execution_id, include_logs, tail):
    logger = get_logger()
    management_ip = utils.get_management_server_ip()
    logger.info("Getting events from management server {0} for "
                "execution id '{1}' "
                "[include_logs={2}]".format(management_ip,
                                            execution_id,
                                            include_logs))
    client = utils.get_rest_client(management_ip)
    try:
        execution_events = ExecutionEventsFetcher(
            client,
            execution_id,
            include_logs=include_logs)

        events_logger = get_events_logger()

        if tail:
            execution = wait_for_execution(client,
                                           client.executions.get(execution_id),
                                           events_handler=events_logger,
                                           include_logs=include_logs,
                                           timeout=None)   # don't timeout ever
            if execution.error:
                logger.info("Execution of workflow '{0}' for deployment "
                            "'{1}' failed. [error={2}]"
                            .format(execution.workflow_id,
                                    execution.deployment_id,
                                    execution.error))
                raise SuppressedCloudifyCliError()
            else:
                logger.info("Finished executing workflow '{0}' on deployment "
                            "'{1}'".format(execution.workflow_id,
                                           execution.deployment_id))
        else:
            # don't tail, get only the events created until now and return
            events = execution_events.fetch_and_process_events(
                events_handler=events_logger)
            logger.info('\nTotal events: {0}'.format(events))
    except CloudifyClientError, e:
        if e.status_code != 404:
            raise
        msg = ("Execution '{0}' not found on management server"
               .format(execution_id))
        raise CloudifyCliError(msg)
Example #8
0
def ls(execution_id, include_logs, tail, json):
    logger = get_logger()
    management_ip = utils.get_management_server_ip()
    logger.info('Listing events for execution id {0} '
                '[include_logs={1}]'.format(execution_id, include_logs))
    client = utils.get_rest_client(management_ip)
    try:
        execution_events = ExecutionEventsFetcher(
            client,
            execution_id,
            include_logs=include_logs)

        events_logger = get_events_logger(json)

        if tail:
            execution = wait_for_execution(client,
                                           client.executions.get(execution_id),
                                           events_handler=events_logger,
                                           include_logs=include_logs,
                                           timeout=None)   # don't timeout ever
            if execution.error:
                logger.info('Execution of workflow {0} for deployment '
                            '{1} failed. [error={2}]'.format(
                                execution.workflow_id,
                                execution.deployment_id,
                                execution.error))
                raise SuppressedCloudifyCliError()
            else:
                logger.info('Finished executing workflow {0} on deployment '
                            '{1}'.format(
                                execution.workflow_id,
                                execution.deployment_id))
        else:
            # don't tail, get only the events created until now and return
            events = execution_events.fetch_and_process_events(
                events_handler=events_logger)
            logger.info('\nTotal events: {0}'.format(events))
    except CloudifyClientError as e:
        if e.status_code != 404:
            raise
        raise CloudifyCliError('Execution {0} not found'.format(execution_id))
Example #9
0
    def worker(dep_id):
        try:
            execution = client.executions.start(
                dep_id,
                workflow_id,
            )

            execution = wait_for_execution(
                client,
                execution,
                events_handler=threadsafe_events_logger,
                include_logs=include_logs,
                timeout=900
            )

            if execution.error:
                log_to_summary("Execution of workflow '{0}' for "
                               "deployment '{1}' failed. [error={2}]"
                               .format(workflow_id,
                                       dep_id,
                                       execution.error))
            else:
                threadsafe_log("Finished executing workflow "
                               "'{0}' on deployment"
                               " '{1}'".format(workflow_id, dep_id))

        except ExecutionTimeoutError as e:
            log_to_summary("Execution of workflow '{0}' "
                           "for deployment '{1}' timed out. "
                           "* Run 'cfy executions cancel "
                           "--execution-id {2}' to cancel"
                           " the running workflow."
                           .format(
                               workflow_id,
                               deployment_id,
                               e.execution_id
                           ))
Example #10
0
def start(workflow_id, deployment_id, timeout, force, allow_custom_parameters,
          include_logs, parameters):
    logger = get_logger()
    parameters = utils.inputs_to_dict(parameters, 'parameters')
    management_ip = utils.get_management_server_ip()
    logger.info("Executing workflow '{0}' on deployment '{1}' at"
                " management server {2} [timeout={3} seconds]".format(
                    workflow_id, deployment_id, management_ip, timeout))

    events_logger = get_events_logger()

    events_message = "* Run 'cfy events list --include-logs " \
                     "--execution-id {0}' to retrieve the " \
                     "execution's events/logs"
    try:
        client = utils.get_rest_client(management_ip)
        try:
            execution = client.executions.start(
                deployment_id,
                workflow_id,
                parameters=parameters,
                allow_custom_parameters=allow_custom_parameters,
                force=force)
        except (exceptions.DeploymentEnvironmentCreationInProgressError,
                exceptions.DeploymentEnvironmentCreationPendingError) as e:
            # wait for deployment environment creation workflow
            if isinstance(
                    e, exceptions.DeploymentEnvironmentCreationPendingError):
                status = 'pending'
            else:
                status = 'in progress'

            logger.info(
                'Deployment environment creation is {0}!'.format(status))
            logger.info('Waiting for create_deployment_environment '
                        'workflow execution to finish...')
            now = time.time()
            wait_for_execution(client,
                               _get_deployment_environment_creation_execution(
                                   client, deployment_id),
                               events_handler=events_logger,
                               include_logs=include_logs,
                               timeout=timeout)
            remaining_timeout = time.time() - now
            timeout -= remaining_timeout
            # try to execute user specified workflow
            execution = client.executions.start(
                deployment_id,
                workflow_id,
                parameters=parameters,
                allow_custom_parameters=allow_custom_parameters,
                force=force)

        execution = wait_for_execution(client,
                                       execution,
                                       events_handler=events_logger,
                                       include_logs=include_logs,
                                       timeout=timeout)
        if execution.error:
            logger.info("Execution of workflow '{0}' for deployment "
                        "'{1}' failed. [error={2}]".format(
                            workflow_id, deployment_id, execution.error))
            logger.info(events_message.format(execution.id))
            raise SuppressedCloudifyCliError()
        else:
            logger.info("Finished executing workflow '{0}' on deployment"
                        " '{1}'".format(workflow_id, deployment_id))
            logger.info(events_message.format(execution.id))
    except ExecutionTimeoutError, e:
        logger.info("Execution of workflow '{0}' "
                    "for deployment '{1}' timed out. "
                    "* Run 'cfy executions cancel "
                    "--execution-id {2}' to cancel"
                    " the running workflow.".format(workflow_id, deployment_id,
                                                    e.execution_id))
        events_tail_message = "* Run 'cfy events list --tail --include-logs " \
                              "--execution-id {0}' to retrieve the " \
                              "execution's events/logs"
        logger.info(events_tail_message.format(e.execution_id))
        raise SuppressedCloudifyCliError()
Example #11
0
def start(workflow_id, deployment_id, timeout, force,
          allow_custom_parameters, include_logs, parameters):
    logger = get_logger()
    parameters = json_to_dict(parameters, 'parameters')
    management_ip = utils.get_management_server_ip()
    logger.info("Executing workflow '{0}' on deployment '{1}' at"
                " management server {2} [timeout={3} seconds]"
                .format(workflow_id,
                        deployment_id,
                        management_ip,
                        timeout))

    events_logger = get_events_logger()

    events_message = "* Run 'cfy events list --include-logs " \
                     "--execution-id {0}' for retrieving the " \
                     "execution's events/logs"
    try:
        client = utils.get_rest_client(management_ip)
        try:
            execution = client.executions.start(
                deployment_id,
                workflow_id,
                parameters=parameters,
                allow_custom_parameters=allow_custom_parameters,
                force=force)
        except exceptions.DeploymentEnvironmentCreationInProgressError:
            # wait for deployment environment creation workflow to end
            logger.info('Deployment environment creation is in progress!')
            logger.info('Waiting for create_deployment_environment '
                        'workflow execution to finish...')
            now = time.time()
            wait_for_execution(client,
                               deployment_id,
                               _get_deployment_environment_creation_execution(
                                   client, deployment_id),
                               events_handler=events_logger,
                               include_logs=include_logs,
                               timeout=timeout)
            remaining_timeout = time.time() - now
            timeout -= remaining_timeout
            # try to execute user specified workflow
            execution = client.executions.start(
                deployment_id,
                workflow_id,
                parameters=parameters,
                allow_custom_parameters=allow_custom_parameters,
                force=force)

        execution = wait_for_execution(client,
                                       deployment_id,
                                       execution,
                                       events_handler=events_logger,
                                       include_logs=include_logs,
                                       timeout=timeout)
        if execution.error:
            logger.info("Execution of workflow '{0}' for deployment "
                        "'{1}' failed. [error={2}]"
                        .format(workflow_id,
                                deployment_id,
                                execution.error))
            logger.info(events_message.format(execution.id))
            raise SuppressedCloudifyCliError()
        else:
            logger.info("Finished executing workflow '{0}' on deployment"
                        "'{1}'".format(workflow_id, deployment_id))
            logger.info(events_message.format(execution.id))
    except ExecutionTimeoutError, e:
        logger.info("Execution of workflow '{0}' "
                    "for deployment '{1}' timed out. "
                    "* Run 'cfy executions cancel "
                    "--execution-id {2}' to cancel"
                    " the running workflow."
                    .format(workflow_id, deployment_id, e.execution_id))
        logger.info(events_message.format(e.execution_id))
        raise SuppressedCloudifyCliError()
def _wait(client, execution):
    wait_for_execution(client,
                       execution,
                       events_handler=_events_logger,
                       include_logs=True,
                       logger=logger)
Example #13
0
def update(deployment_id,
           blueprint_path,
           inputs,
           blueprint_filename,
           archive_location,
           skip_install,
           skip_uninstall,
           workflow_id,
           force,
           include_logs,
           json):
    logger = get_logger()
    rest_host = utils.get_rest_host()
    client = utils.get_rest_client(rest_host)

    processed_inputs = utils.inputs_to_dict(inputs, 'inputs')

    blueprint_or_archive_path = blueprint_path.name \
        if blueprint_path else archive_location
    logger.info('Updating deployment {dep_id} using blueprint {path}'.format(
        dep_id=deployment_id, path=blueprint_or_archive_path))

    deployment_update = client.deployment_updates.update(
        deployment_id,
        blueprint_or_archive_path,
        application_file_name=blueprint_filename,
        inputs=processed_inputs,
        workflow_id=workflow_id,
        skip_install=skip_install,
        skip_uninstall=skip_uninstall,
        force=force)

    events_logger = get_events_logger(json)

    execution = wait_for_execution(
        client,
        client.executions.get(deployment_update.execution_id),
        events_handler=events_logger,
        include_logs=include_logs,
        timeout=None)  # don't timeout ever
    if execution.error:
        logger.info("Execution of workflow '{0}' for deployment "
                    "'{1}' failed. [error={2}]"
                    .format(execution.workflow_id,
                            execution.deployment_id,
                            execution.error))
        logger.info('Failed updating deployment {dep_id}. Deployment update '
                    'id: {depup_id}. Execution id: {exec_id}'
                    .format(depup_id=deployment_update.id,
                            dep_id=deployment_id,
                            exec_id=execution.id))
        raise SuppressedCloudifyCliError()
    else:
        logger.info("Finished executing workflow '{0}' on deployment "
                    "'{1}'".format(execution.workflow_id,
                                   execution.deployment_id))
        logger.info('Successfully updated deployment {dep_id}. '
                    'Deployment update id: {depup_id}. Execution id: {exec_id}'
                    .format(depup_id=deployment_update.id,
                            dep_id=deployment_id,
                            exec_id=execution.id))
Example #14
0
def start(workflow_id, deployment_id, timeout, force,
          allow_custom_parameters, include_logs, parameters, json):
    logger = get_logger()
    parameters = utils.inputs_to_dict(parameters, 'parameters')
    rest_host = utils.get_rest_host()
    logger.info("Executing workflow '{0}' on deployment '{1}' at"
                " management server {2} [timeout={3} seconds]"
                .format(workflow_id,
                        deployment_id,
                        rest_host,
                        timeout))

    events_logger = get_events_logger(json)

    events_message = "* Run 'cfy events list --include-logs " \
                     "--execution-id {0}' to retrieve the " \
                     "execution's events/logs"
    original_timeout = timeout

    try:
        client = utils.get_rest_client(rest_host)
        try:
            execution = client.executions.start(
                deployment_id,
                workflow_id,
                parameters=parameters,
                allow_custom_parameters=allow_custom_parameters,
                force=force)
        except (exceptions.DeploymentEnvironmentCreationInProgressError,
                exceptions.DeploymentEnvironmentCreationPendingError) as e:
            # wait for deployment environment creation workflow
            if isinstance(
                    e,
                    exceptions.DeploymentEnvironmentCreationPendingError):
                status = 'pending'
            else:
                status = 'in progress'

            logger.info('Deployment environment creation is {0}...'.format(
                status))
            logger.debug('Waiting for create_deployment_environment '
                         'workflow execution to finish...')
            now = time.time()
            wait_for_execution(client,
                               _get_deployment_environment_creation_execution(
                                   client, deployment_id),
                               events_handler=events_logger,
                               include_logs=include_logs,
                               timeout=timeout)
            remaining_timeout = time.time() - now
            timeout -= remaining_timeout
            # try to execute user specified workflow
            execution = client.executions.start(
                deployment_id,
                workflow_id,
                parameters=parameters,
                allow_custom_parameters=allow_custom_parameters,
                force=force)

        execution = wait_for_execution(client,
                                       execution,
                                       events_handler=events_logger,
                                       include_logs=include_logs,
                                       timeout=timeout)
        if execution.error:
            logger.info('Execution of workflow {0} for deployment '
                        '{1} failed. [error={2}]'.format(
                            workflow_id,
                            deployment_id,
                            execution.error))
            logger.info(events_message.format(execution.id))
            raise SuppressedCloudifyCliError()
        else:
            logger.info('Finished executing workflow {0} on deployment '
                        '{1}'.format(workflow_id, deployment_id))
            logger.info(events_message.format(execution.id))
    except ExecutionTimeoutError as e:
        logger.info(
            "Timed out waiting for workflow '{0}' of deployment '{1}' to "
            "end. The execution may still be running properly; however, "
            "the command-line utility was instructed to wait up to {3} "
            "seconds for its completion.\n\n"
            "* Run 'cfy executions list' to determine the execution's "
            "status.\n"
            "* Run 'cfy executions cancel --execution-id {2}' to cancel"
            " the running workflow.".format(
                workflow_id, deployment_id, e.execution_id, original_timeout))

        events_tail_message = "* Run 'cfy events list --tail --include-logs " \
                              "--execution-id {0}' to retrieve the " \
                              "execution's events/logs"
        logger.info(events_tail_message.format(e.execution_id))
        raise SuppressedCloudifyCliError()