def _set_step_failed(self, task, msg, exc): log_msg = ("RAID configuration job failed for node %(node)s. " "Message: '%(message)s'." % { 'node': task.node.uuid, 'message': msg }) if task.node.provision_state == states.DEPLOYING: manager_utils.deploying_error_handler(task, log_msg, errmsg=msg) else: manager_utils.cleaning_error_handler(task, log_msg, errmsg=msg)
def _set_failed(self, task, error_message): """Set the node in failed state by invoking 'fail' event. :param task: a TaskManager instance with node to act on :param error_message: Error message """ log_msg = ("BIOS configuration failed for node %(node)s. %(error)s " % { 'node': task.node.uuid, 'error': error_message }) if task.node.clean_step: manager_utils.cleaning_error_handler(task, log_msg, error_message) else: manager_utils.deploying_error_handler(task, log_msg, error_message)
def continue_node_deploy(task): """Continue deployment after finishing an async deploy step. This function calculates which step has to run next and passes control into do_next_deploy_step. On the first run, deploy steps and templates are also validated. :param task: a TaskManager instance with an exclusive lock """ node = task.node # Agent is now running, we're ready to validate the remaining steps if not node.driver_internal_info.get('steps_validated'): try: conductor_steps.validate_deploy_templates(task) conductor_steps.set_node_deployment_steps(task, reset_current=False) except exception.IronicException as exc: msg = _('Failed to validate the final deploy steps list ' 'for node %(node)s: %(exc)s') % { 'node': node.uuid, 'exc': exc } return utils.deploying_error_handler(task, msg) info = node.driver_internal_info info['steps_validated'] = True node.driver_internal_info = info node.save() next_step_index = utils.update_next_step_index(task, 'deploy') do_next_deploy_step(task, next_step_index)
def _set_step_failed(self, task, attrs_not_updated): """Fail the cleaning or deployment step and log the error. :param task: a TaskManager instance containing the node to act on. :param attrs_not_updated: the BIOS attributes that were not updated. """ error_msg = (_('Redfish BIOS apply_configuration step failed for node ' '%(node)s. Attributes %(attrs)s are not updated.') % {'node': task.node.uuid, 'attrs': attrs_not_updated}) last_error = (_('Redfish BIOS apply_configuration step failed. ' 'Attributes %(attrs)s are not updated.') % {'attrs': attrs_not_updated}) if task.node.provision_state in [states.CLEANING, states.CLEANWAIT]: manager_utils.cleaning_error_handler(task, last_error) if task.node.provision_state in [states.DEPLOYING, states.DEPLOYWAIT]: manager_utils.deploying_error_handler(task, error_msg, last_error)
def _post_step_reboot(task, step_type): """Reboots a node out of band after a clean/deploy step that requires it. If an agent step has 'reboot_requested': True, reboots the node when the step is completed. Will put the node in CLEANFAIL/DEPLOYFAIL if the node cannot be rebooted. :param task: a TaskManager instance :param step_type: 'clean' or 'deploy' """ current_step = (task.node.clean_step if step_type == 'clean' else task.node.deploy_step) try: # NOTE(fellypefca): Call prepare_ramdisk on ensure that the # baremetal node boots back into the ramdisk after reboot. deploy_opts = deploy_utils.build_agent_options(task.node) task.driver.boot.prepare_ramdisk(task, deploy_opts) manager_utils.node_power_action(task, states.REBOOT) except Exception as e: msg = (_('Reboot requested by %(type)s step %(step)s failed for ' 'node %(node)s: %(err)s') % {'step': current_step, 'node': task.node.uuid, 'err': e, 'type': step_type}) LOG.error(msg, exc_info=not isinstance(e, exception.IronicException)) # do not set cleaning_reboot if we didn't reboot if step_type == 'clean': manager_utils.cleaning_error_handler(task, msg) else: manager_utils.deploying_error_handler(task, msg) return # Signify that we've rebooted driver_internal_info = task.node.driver_internal_info field = ('cleaning_reboot' if step_type == 'clean' else 'deployment_reboot') driver_internal_info[field] = True if not driver_internal_info.get('agent_secret_token_pregenerated', False): # Wipes out the existing recorded token because the machine will # need to re-establish the token. driver_internal_info.pop('agent_secret_token', None) task.node.driver_internal_info = driver_internal_info task.node.save()
def set_failed_state(task, msg, collect_logs=True): """Sets the deploy status as failed with relevant messages. This method sets the deployment as fail with the given message. It sets node's provision_state to DEPLOYFAIL and updates last_error with the given error message. It also powers off the baremetal node. :param task: a TaskManager instance containing the node to act on. :param msg: the message to set in logs and last_error of the node. :param collect_logs: Boolean indicating whether to attempt to collect logs from IPA-based ramdisk. Defaults to True. Actual log collection is also affected by CONF.agent.deploy_logs_collect config option. """ node = task.node if (collect_logs and CONF.agent.deploy_logs_collect in ('on_failure', 'always')): driver_utils.collect_ramdisk_logs(node) try: manager_utils.deploying_error_handler(task, msg, msg, clean_up=False) except exception.InvalidState: msg2 = ('Internal error. Node %(node)s in provision state ' '"%(state)s" could not transition to a failed state.' % { 'node': node.uuid, 'state': node.provision_state }) LOG.exception(msg2) if CONF.deploy.power_off_after_deploy_failure: try: manager_utils.node_power_action(task, states.POWER_OFF) except Exception: msg2 = ('Node %s failed to power off while handling deploy ' 'failure. This may be a serious condition. Node ' 'should be removed from Ironic or put in maintenance ' 'mode until the problem is resolved.' % node.uuid) LOG.exception(msg2) # NOTE(tenbrae): node_power_action() erases node.last_error # so we need to set it here. node.last_error = msg node.save()
def do_next_deploy_step(task, step_index, conductor_id): """Do deployment, starting from the specified deploy step. :param task: a TaskManager instance with an exclusive lock :param step_index: The first deploy step in the list to execute. This is the index (from 0) into the list of deploy steps in the node's driver_internal_info['deploy_steps']. Is None if there are no steps to execute. """ node = task.node if step_index is None: steps = [] else: steps = node.driver_internal_info['deploy_steps'][step_index:] LOG.info( 'Executing %(state)s on node %(node)s, remaining steps: ' '%(steps)s', { 'node': node.uuid, 'steps': steps, 'state': node.provision_state }) # Execute each step until we hit an async step or run out of steps for ind, step in enumerate(steps): # Save which step we're about to start so we can restart # if necessary node.deploy_step = step driver_internal_info = node.driver_internal_info driver_internal_info['deploy_step_index'] = step_index + ind node.driver_internal_info = driver_internal_info node.save() interface = getattr(task.driver, step.get('interface')) LOG.info('Executing %(step)s on node %(node)s', { 'step': step, 'node': node.uuid }) try: result = interface.execute_deploy_step(task, step) except exception.IronicException as e: if isinstance(e, exception.AgentConnectionFailed): if task.node.driver_internal_info.get('deployment_reboot'): LOG.info( 'Agent is not yet running on node %(node)s after ' 'deployment reboot, waiting for agent to come up ' 'to run next deploy step %(step)s.', { 'node': node.uuid, 'step': step }) driver_internal_info['skip_current_deploy_step'] = False node.driver_internal_info = driver_internal_info task.process_event('wait') return log_msg = ('Node %(node)s failed deploy step %(step)s. Error: ' '%(err)s' % { 'node': node.uuid, 'step': node.deploy_step, 'err': e }) utils.deploying_error_handler( task, log_msg, _("Failed to deploy: %s") % node.deploy_step) return except Exception as e: log_msg = ('Node %(node)s failed deploy step %(step)s with ' 'unexpected error: %(err)s' % { 'node': node.uuid, 'step': node.deploy_step, 'err': e }) utils.deploying_error_handler( task, log_msg, _("Failed to deploy. Exception: %s") % e, traceback=True) return if ind == 0: # We've done the very first deploy step. # Update conductor_affinity to reference this conductor's ID # since there may be local persistent state node.conductor_affinity = conductor_id node.save() # Check if the step is done or not. The step should return # states.DEPLOYWAIT if the step is still being executed, or # None if the step is done. # NOTE(deva): Some drivers may return states.DEPLOYWAIT # eg. if they are waiting for a callback if result == states.DEPLOYWAIT: # Kill this worker, the async step will make an RPC call to # continue_node_deploy() to continue deploying LOG.info( 'Deploy step %(step)s on node %(node)s being ' 'executed asynchronously, waiting for driver.', { 'node': node.uuid, 'step': step }) task.process_event('wait') return elif result is not None: # NOTE(rloo): This is an internal/dev error; shouldn't happen. log_msg = (_('While executing deploy step %(step)s on node ' '%(node)s, step returned unexpected state: %(val)s') % { 'step': step, 'node': node.uuid, 'val': result }) utils.deploying_error_handler( task, log_msg, _("Failed to deploy: %s") % node.deploy_step) return LOG.info('Node %(node)s finished deploy step %(step)s', { 'node': node.uuid, 'step': step }) # Finished executing the steps. Clear deploy_step. node.deploy_step = None driver_internal_info = node.driver_internal_info driver_internal_info.pop('agent_secret_token', None) driver_internal_info.pop('agent_secret_token_pregenerated', None) driver_internal_info['deploy_steps'] = None driver_internal_info.pop('deploy_step_index', None) driver_internal_info.pop('deployment_reboot', None) driver_internal_info.pop('deployment_polling', None) # Remove the agent_url cached from the deployment. driver_internal_info.pop('agent_url', None) node.driver_internal_info = driver_internal_info node.save() _start_console_in_deploy(task) task.process_event('done') LOG.info( 'Successfully deployed node %(node)s with ' 'instance %(instance)s.', { 'node': node.uuid, 'instance': node.instance_uuid })
def do_node_deploy(task, conductor_id=None, configdrive=None): """Prepare the environment and deploy a node.""" node = task.node utils.del_secret_token(node) try: if configdrive: if isinstance(configdrive, dict): configdrive = utils.build_configdrive(node, configdrive) _store_configdrive(node, configdrive) except (exception.SwiftOperationError, exception.ConfigInvalid) as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Error while uploading the configdrive for %(node)s ' 'to Swift') % {'node': node.uuid}, _('Failed to upload the configdrive to Swift. ' 'Error: %s') % e, clean_up=False) except db_exception.DBDataError as e: with excutils.save_and_reraise_exception(): # NOTE(hshiina): This error happens when the configdrive is # too large. Remove the configdrive from the # object to update DB successfully in handling # the failure. node.obj_reset_changes() utils.deploying_error_handler( task, ('Error while storing the configdrive for %(node)s into ' 'the database: %(err)s') % { 'node': node.uuid, 'err': e }, _("Failed to store the configdrive in the database. " "%s") % e, clean_up=False) except Exception as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Unexpected error while preparing the configdrive for ' 'node %(node)s') % {'node': node.uuid}, _("Failed to prepare the configdrive. Exception: %s") % e, traceback=True, clean_up=False) try: task.driver.deploy.prepare(task) except exception.IronicException as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Error while preparing to deploy to node %(node)s: ' '%(err)s') % { 'node': node.uuid, 'err': e }, _("Failed to prepare to deploy: %s") % e, clean_up=False) except Exception as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Unexpected error while preparing to deploy to node ' '%(node)s') % {'node': node.uuid}, _("Failed to prepare to deploy. Exception: %s") % e, traceback=True, clean_up=False) try: # This gets the deploy steps and puts them in the node's # driver_internal_info['deploy_steps']. conductor_steps.set_node_deployment_steps(task) except exception.InstanceDeployFailure as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, 'Error while getting deploy steps; cannot deploy to node ' '%(node)s. Error: %(err)s' % { 'node': node.uuid, 'err': e }, _("Cannot get deploy steps; failed to deploy: %s") % e) if not node.driver_internal_info.get('deploy_steps'): msg = _('Error while getting deploy steps: no steps returned for ' 'node %s') % node.uuid utils.deploying_error_handler( task, msg, _("No deploy steps returned by the driver")) raise exception.InstanceDeployFailure(msg) do_next_deploy_step(task, 0, conductor_id)
def do_next_deploy_step(task, step_index): """Do deployment, starting from the specified deploy step. :param task: a TaskManager instance with an exclusive lock :param step_index: The first deploy step in the list to execute. This is the index (from 0) into the list of deploy steps in the node's driver_internal_info['deploy_steps']. Is None if there are no steps to execute. """ node = task.node if step_index is None: steps = [] else: steps = node.driver_internal_info['deploy_steps'][step_index:] LOG.info( 'Executing %(state)s on node %(node)s, remaining steps: ' '%(steps)s', { 'node': node.uuid, 'steps': steps, 'state': node.provision_state }) # Execute each step until we hit an async step or run out of steps for ind, step in enumerate(steps): # Save which step we're about to start so we can restart # if necessary node.deploy_step = step driver_internal_info = node.driver_internal_info driver_internal_info['deploy_step_index'] = step_index + ind node.driver_internal_info = driver_internal_info node.save() interface = getattr(task.driver, step.get('interface')) LOG.info('Executing %(step)s on node %(node)s', { 'step': step, 'node': node.uuid }) try: result = interface.execute_deploy_step(task, step) except exception.IronicException as e: if isinstance(e, exception.AgentConnectionFailed): if task.node.driver_internal_info.get('deployment_reboot'): LOG.info( 'Agent is not yet running on node %(node)s after ' 'deployment reboot, waiting for agent to come up ' 'to run next deploy step %(step)s.', { 'node': node.uuid, 'step': step }) driver_internal_info['skip_current_deploy_step'] = False node.driver_internal_info = driver_internal_info task.process_event('wait') return if isinstance(e, exception.AgentInProgress): LOG.info( 'Conductor attempted to process deploy step for ' 'node %(node)s. Agent indicated it is presently ' 'executing a command. Error: %(error)s', { 'node': task.node.uuid, 'error': e }) driver_internal_info['skip_current_deploy_step'] = False node.driver_internal_info = driver_internal_info task.process_event('wait') return log_msg = ('Node %(node)s failed deploy step %(step)s. Error: ' '%(err)s' % { 'node': node.uuid, 'step': node.deploy_step, 'err': e }) utils.deploying_error_handler( task, log_msg, _("Failed to deploy: Deploy step %(step)s, " "error: %(err)s.") % { 'step': node.deploy_step, 'err': e }) return except Exception as e: log_msg = ('Node %(node)s failed deploy step %(step)s with ' 'unexpected error: %(err)s' % { 'node': node.uuid, 'step': node.deploy_step, 'err': e }) utils.deploying_error_handler( task, log_msg, _("Failed to deploy. Exception: %s") % e, traceback=True) return if task.node.provision_state == states.DEPLOYFAIL: # NOTE(dtantsur): some deploy steps do not raise but rather update # the node and return. Take them into account. LOG.debug( 'Node %s is in error state, not processing ' 'the remaining deploy steps', task.node) return # Check if the step is done or not. The step should return # states.DEPLOYWAIT if the step is still being executed, or # None if the step is done. # NOTE(tenbrae): Some drivers may return states.DEPLOYWAIT # eg. if they are waiting for a callback if result == states.DEPLOYWAIT: # Kill this worker, the async step will make an RPC call to # continue_node_deploy() to continue deploying LOG.info( 'Deploy step %(step)s on node %(node)s being ' 'executed asynchronously, waiting for driver.', { 'node': node.uuid, 'step': step }) if task.node.provision_state != states.DEPLOYWAIT: task.process_event('wait') return elif result is not None: # NOTE(rloo): This is an internal/dev error; shouldn't happen. log_msg = (_('While executing deploy step %(step)s on node ' '%(node)s, step returned unexpected state: %(val)s') % { 'step': step, 'node': node.uuid, 'val': result }) utils.deploying_error_handler( task, log_msg, _("Failed to deploy: %s") % node.deploy_step) return LOG.info('Node %(node)s finished deploy step %(step)s', { 'node': node.uuid, 'step': step }) # Finished executing the steps. Clear deploy_step. node.deploy_step = None utils.wipe_deploy_internal_info(task) node.save() _start_console_in_deploy(task) task.process_event('done') LOG.info( 'Successfully deployed node %(node)s with ' 'instance %(instance)s.', { 'node': node.uuid, 'instance': node.instance_uuid })
def do_node_deploy(task, conductor_id=None, configdrive=None, deploy_steps=None): """Prepare the environment and deploy a node.""" node = task.node utils.wipe_deploy_internal_info(task) try: if configdrive: if isinstance(configdrive, dict): configdrive = utils.build_configdrive(node, configdrive) _store_configdrive(node, configdrive) except (exception.SwiftOperationError, exception.ConfigInvalid) as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Error while uploading the configdrive for %(node)s ' 'to Swift') % {'node': node.uuid}, _('Failed to upload the configdrive to Swift. ' 'Error: %s') % e, clean_up=False) except db_exception.DBDataError as e: with excutils.save_and_reraise_exception(): # NOTE(hshiina): This error happens when the configdrive is # too large. Remove the configdrive from the # object to update DB successfully in handling # the failure. node.obj_reset_changes() utils.deploying_error_handler( task, ('Error while storing the configdrive for %(node)s into ' 'the database: %(err)s') % { 'node': node.uuid, 'err': e }, _("Failed to store the configdrive in the database. " "%s") % e, clean_up=False) except Exception as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Unexpected error while preparing the configdrive for ' 'node %(node)s') % {'node': node.uuid}, _("Failed to prepare the configdrive. Exception: %s") % e, traceback=True, clean_up=False) try: task.driver.deploy.prepare(task) except exception.IronicException as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Error while preparing to deploy to node %(node)s: ' '%(err)s') % { 'node': node.uuid, 'err': e }, _("Failed to prepare to deploy: %s") % e, clean_up=False) except Exception as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, ('Unexpected error while preparing to deploy to node ' '%(node)s') % {'node': node.uuid}, _("Failed to prepare to deploy. Exception: %s") % e, traceback=True, clean_up=False) try: # If any deploy steps provided by user, save them to node. They will be # validated & processed later together with driver and deploy template # steps. if deploy_steps: info = node.driver_internal_info info['user_deploy_steps'] = deploy_steps node.driver_internal_info = info node.save() # This gets the deploy steps (if any) from driver, deploy template and # deploy_steps argument and updates them in the node's # driver_internal_info['deploy_steps']. In-band steps are skipped since # we know that an agent is not running yet. conductor_steps.set_node_deployment_steps(task, skip_missing=True) except exception.InstanceDeployFailure as e: with excutils.save_and_reraise_exception(): utils.deploying_error_handler( task, 'Error while getting deploy steps; cannot deploy to node ' '%(node)s. Error: %(err)s' % { 'node': node.uuid, 'err': e }, _("Cannot get deploy steps; failed to deploy: %s") % e) if not node.driver_internal_info.get('deploy_steps'): msg = _('Error while getting deploy steps: no steps returned for ' 'node %s') % node.uuid utils.deploying_error_handler( task, msg, _("No deploy steps returned by the driver")) raise exception.InstanceDeployFailure(msg) if conductor_id is not None: # Update conductor_affinity to reference this conductor's ID # since there may be local persistent state node.conductor_affinity = conductor_id node.save() do_next_deploy_step(task, 0)
def process_next_step(self, task, step_type, **kwargs): """Start the next clean/deploy step if the previous one is complete. In order to avoid errors and make agent upgrades painless, the agent compares the version of all hardware managers at the start of the process (the agent's get_clean|deploy_steps() call) and before executing each step. If the version has changed between steps, the agent is unable to tell if an ordering change will cause an issue so it returns CLEAN_VERSION_MISMATCH. For automated cleaning, we restart the entire cleaning cycle. For manual cleaning or deploy, we don't. Additionally, if a step includes the reboot_requested property set to True, this method will coordinate the reboot once the step is completed. """ assert step_type in ('clean', 'deploy') node = task.node # For manual clean, the target provision state is MANAGEABLE, whereas # for automated cleaning, it is (the default) AVAILABLE. manual_clean = node.target_provision_state == states.MANAGEABLE agent_commands = self._client.get_commands_status(task.node) if not agent_commands: field = ('cleaning_reboot' if step_type == 'clean' else 'deployment_reboot') if task.node.driver_internal_info.get(field): # Node finished a cleaning step that requested a reboot, and # this is the first heartbeat after booting. Continue cleaning. info = task.node.driver_internal_info info.pop(field, None) task.node.driver_internal_info = info task.node.save() manager_utils.notify_conductor_resume_operation(task, step_type) return else: # Agent has no commands whatsoever return current_step = (node.clean_step if step_type == 'clean' else node.deploy_step) command = _get_completed_command(task, agent_commands, step_type) LOG.debug('%(type)s command status for node %(node)s on step %(step)s:' ' %(command)s', {'node': node.uuid, 'step': current_step, 'command': command, 'type': step_type}) if not command: # Agent command in progress return if command.get('command_status') == 'FAILED': msg = (_('Agent returned error for %(type)s step %(step)s on node ' '%(node)s : %(err)s.') % {'node': node.uuid, 'err': command.get('command_error'), 'step': current_step, 'type': step_type}) LOG.error(msg) return manager_utils.cleaning_error_handler(task, msg) elif command.get('command_status') in ('CLEAN_VERSION_MISMATCH', 'DEPLOY_VERSION_MISMATCH'): # Cache the new clean steps (and 'hardware_manager_version') try: self.refresh_steps(task, step_type) except exception.NodeCleaningFailure as e: msg = (_('Could not continue cleaning on node ' '%(node)s: %(err)s.') % {'node': node.uuid, 'err': e}) LOG.exception(msg) return manager_utils.cleaning_error_handler(task, msg) except exception.InstanceDeployFailure as e: msg = (_('Could not continue deployment on node ' '%(node)s: %(err)s.') % {'node': node.uuid, 'err': e}) LOG.exception(msg) return manager_utils.deploying_error_handler(task, msg) if manual_clean: # Don't restart manual cleaning if agent reboots to a new # version. Both are operator actions, unlike automated # cleaning. Manual clean steps are not necessarily idempotent # like automated clean steps and can be even longer running. LOG.info('During manual cleaning, node %(node)s detected ' 'a clean version mismatch. Re-executing and ' 'continuing from current step %(step)s.', {'node': node.uuid, 'step': node.clean_step}) driver_internal_info = node.driver_internal_info driver_internal_info['skip_current_clean_step'] = False node.driver_internal_info = driver_internal_info node.save() else: # Restart the process, agent must have rebooted to new version LOG.info('During %(type)s, node %(node)s detected a ' '%(type)s version mismatch. Resetting %(type)s steps ' 'and rebooting the node.', {'type': step_type, 'node': node.uuid}) try: conductor_steps.set_node_cleaning_steps(task) except exception.NodeCleaningFailure as e: msg = (_('Could not restart automated cleaning on node ' '%(node)s after step %(step)s: %(err)s.') % {'node': node.uuid, 'err': e, 'step': node.clean_step}) LOG.exception(msg) return manager_utils.cleaning_error_handler(task, msg) except exception.InstanceDeployFailure as e: msg = (_('Could not restart deployment on node ' '%(node)s after step %(step)s: %(err)s.') % {'node': node.uuid, 'err': e, 'step': node.deploy_step}) LOG.exception(msg) return manager_utils.deploying_error_handler(task, msg) manager_utils.notify_conductor_resume_operation(task, step_type) elif command.get('command_status') == 'SUCCEEDED': step_hook = _get_post_step_hook(node, step_type) if step_hook is not None: LOG.debug('For node %(node)s, executing post %(type)s step ' 'hook %(method)s for %(type)s step %(step)s', {'method': step_hook.__name__, 'node': node.uuid, 'step': current_step, 'type': step_type}) try: step_hook(task, command) except Exception as e: msg = (_('For node %(node)s, post %(type)s step hook ' '%(method)s failed for %(type)s step %(step)s.' '%(cls)s: %(error)s') % {'method': step_hook.__name__, 'node': node.uuid, 'error': e, 'cls': e.__class__.__name__, 'step': current_step, 'type': step_type}) LOG.exception(msg) if step_type == 'clean': return manager_utils.cleaning_error_handler(task, msg) else: return manager_utils.deploying_error_handler(task, msg) if current_step.get('reboot_requested'): _post_step_reboot(task, step_type) return LOG.info('Agent on node %(node)s returned %(type)s command ' 'success, moving to next step', {'node': node.uuid, 'type': step_type}) manager_utils.notify_conductor_resume_operation(task, step_type) else: msg = (_('Agent returned unknown status for %(type)s step %(step)s' ' on node %(node)s : %(err)s.') % {'node': node.uuid, 'err': command.get('command_status'), 'step': current_step, 'type': step_type}) LOG.error(msg) if step_type == 'clean': return manager_utils.cleaning_error_handler(task, msg) else: return manager_utils.deploying_error_handler(task, msg)
def _set_failed(self, task, log_msg, error_msg): if task.node.clean_step: manager_utils.cleaning_error_handler(task, log_msg, error_msg) else: manager_utils.deploying_error_handler(task, log_msg, error_msg)