def do_next_deploy_step(task, step_index):
"""Do deployment, starting from the specified deploy step.
:param task: a TaskManager instance with an exclusive lock
:param step_index: The first deploy step in the list to execute. This
is the index (from 0) into the list of deploy steps in the node's
driver_internal_info['deploy_steps']. Is None if there are no steps
to execute.
"""
node = task.node
if step_index is None:
steps = []
else:
steps = node.driver_internal_info['deploy_steps'][step_index:]
LOG.info(
'Executing %(state)s on node %(node)s, remaining steps: '
'%(steps)s', {
'node': node.uuid,
'steps': steps,
'state': node.provision_state
})
# Execute each step until we hit an async step or run out of steps
for ind, step in enumerate(steps):
# Save which step we're about to start so we can restart
# if necessary
node.deploy_step = step
driver_internal_info = node.driver_internal_info
driver_internal_info['deploy_step_index'] = step_index + ind
node.driver_internal_info = driver_internal_info
node.save()
interface = getattr(task.driver, step.get('interface'))
LOG.info('Executing %(step)s on node %(node)s', {
'step': step,
'node': node.uuid
})
try:
result = interface.execute_deploy_step(task, step)
except exception.IronicException as e:
if isinstance(e, exception.AgentConnectionFailed):
if task.node.driver_internal_info.get('deployment_reboot'):
LOG.info(
'Agent is not yet running on node %(node)s after '
'deployment reboot, waiting for agent to come up '
'to run next deploy step %(step)s.', {
'node': node.uuid,
'step': step
})
driver_internal_info['skip_current_deploy_step'] = False
node.driver_internal_info = driver_internal_info
task.process_event('wait')
return
log_msg = ('Node %(node)s failed deploy step %(step)s. Error: '
'%(err)s' % {
'node': node.uuid,
'step': node.deploy_step,
'err': e
})
utils.deploying_error_handler(
task, log_msg,
_("Failed to deploy: Deploy step %(step)s, "
"error: %(err)s.") % {
'step': node.deploy_step,
'err': e
})
return
except Exception as e:
log_msg = ('Node %(node)s failed deploy step %(step)s with '
'unexpected error: %(err)s' % {
'node': node.uuid,
'step': node.deploy_step,
'err': e
})
utils.deploying_error_handler(
task,
log_msg,
_("Failed to deploy. Exception: %s") % e,
traceback=True)
return
if task.node.provision_state == states.DEPLOYFAIL:
# NOTE(dtantsur): some deploy steps do not raise but rather update
# the node and return. Take them into account.
LOG.debug(
'Node %s is in error state, not processing '
'the remaining deploy steps', task.node)
return
# Check if the step is done or not. The step should return
# states.DEPLOYWAIT if the step is still being executed, or
# None if the step is done.
# NOTE(tenbrae): Some drivers may return states.DEPLOYWAIT
# eg. if they are waiting for a callback
if result == states.DEPLOYWAIT:
# Kill this worker, the async step will make an RPC call to
# continue_node_deploy() to continue deploying
LOG.info(
'Deploy step %(step)s on node %(node)s being '
'executed asynchronously, waiting for driver.', {
'node': node.uuid,
'step': step
})
if task.node.provision_state != states.DEPLOYWAIT:
task.process_event('wait')
return
elif result is not None:
# NOTE(rloo): This is an internal/dev error; shouldn't happen.
log_msg = (_('While executing deploy step %(step)s on node '
'%(node)s, step returned unexpected state: %(val)s') %
{
'step': step,
'node': node.uuid,
'val': result
})
utils.deploying_error_handler(
task, log_msg,
_("Failed to deploy: %s") % node.deploy_step)
return
LOG.info('Node %(node)s finished deploy step %(step)s', {
'node': node.uuid,
'step': step
})
# Finished executing the steps. Clear deploy_step.
node.deploy_step = None
utils.wipe_deploy_internal_info(task)
node.save()
_start_console_in_deploy(task)
task.process_event('done')
LOG.info(
'Successfully deployed node %(node)s with '
'instance %(instance)s.', {
'node': node.uuid,
'instance': node.instance_uuid
})
def do_node_deploy(task, conductor_id=None, configdrive=None):
"""Prepare the environment and deploy a node."""
node = task.node
utils.wipe_deploy_internal_info(task)
try:
if configdrive:
if isinstance(configdrive, dict):
configdrive = utils.build_configdrive(node, configdrive)
_store_configdrive(node, configdrive)
except (exception.SwiftOperationError, exception.ConfigInvalid) as e:
with excutils.save_and_reraise_exception():
utils.deploying_error_handler(
task,
('Error while uploading the configdrive for %(node)s '
'to Swift') % {'node': node.uuid},
_('Failed to upload the configdrive to Swift. '
'Error: %s') % e,
clean_up=False)
except db_exception.DBDataError as e:
with excutils.save_and_reraise_exception():
# NOTE(hshiina): This error happens when the configdrive is
# too large. Remove the configdrive from the
# object to update DB successfully in handling
# the failure.
node.obj_reset_changes()
utils.deploying_error_handler(
task,
('Error while storing the configdrive for %(node)s into '
'the database: %(err)s') % {
'node': node.uuid,
'err': e
},
_("Failed to store the configdrive in the database. "
"%s") % e,
clean_up=False)
except Exception as e:
with excutils.save_and_reraise_exception():
utils.deploying_error_handler(
task,
('Unexpected error while preparing the configdrive for '
'node %(node)s') % {'node': node.uuid},
_("Failed to prepare the configdrive. Exception: %s") % e,
traceback=True,
clean_up=False)
try:
task.driver.deploy.prepare(task)
except exception.IronicException as e:
with excutils.save_and_reraise_exception():
utils.deploying_error_handler(
task, ('Error while preparing to deploy to node %(node)s: '
'%(err)s') % {
'node': node.uuid,
'err': e
},
_("Failed to prepare to deploy: %s") % e,
clean_up=False)
except Exception as e:
with excutils.save_and_reraise_exception():
utils.deploying_error_handler(
task,
('Unexpected error while preparing to deploy to node '
'%(node)s') % {'node': node.uuid},
_("Failed to prepare to deploy. Exception: %s") % e,
traceback=True,
clean_up=False)
try:
# This gets the deploy steps (if any) and puts them in the node's
# driver_internal_info['deploy_steps']. In-band steps are skipped since
# we know that an agent is not running yet.
conductor_steps.set_node_deployment_steps(task, skip_missing=True)
except exception.InstanceDeployFailure as e:
with excutils.save_and_reraise_exception():
utils.deploying_error_handler(
task,
'Error while getting deploy steps; cannot deploy to node '
'%(node)s. Error: %(err)s' % {
'node': node.uuid,
'err': e
},
_("Cannot get deploy steps; failed to deploy: %s") % e)
if not node.driver_internal_info.get('deploy_steps'):
msg = _('Error while getting deploy steps: no steps returned for '
'node %s') % node.uuid
utils.deploying_error_handler(
task, msg, _("No deploy steps returned by the driver"))
raise exception.InstanceDeployFailure(msg)
if conductor_id is not None:
# Update conductor_affinity to reference this conductor's ID
# since there may be local persistent state
node.conductor_affinity = conductor_id
node.save()
do_next_deploy_step(task, 0)
