def _wait_for_deploy():
"""Called at an interval until the deployment completes."""
try:
row = db.bm_node_get(context, node['id'])
if instance['uuid'] != row.get('instance_uuid'):
locals['error'] = _("Node associated with another instance"
" while waiting for deploy of %s")
raise loopingcall.LoopingCallDone()
status = row.get('task_state')
if (status == baremetal_states.DEPLOYING
and locals['started'] == False):
LOG.info(
_("PXE deploy started for instance %s") %
instance['uuid'])
locals['started'] = True
elif status in (baremetal_states.DEPLOYDONE,
baremetal_states.ACTIVE):
LOG.info(
_("PXE deploy completed for instance %s") %
instance['uuid'])
raise loopingcall.LoopingCallDone()
elif status == baremetal_states.DEPLOYFAIL:
locals['error'] = _("PXE deploy failed for instance %s")
except exception.NodeNotFound:
locals['error'] = _("Baremetal node deleted while waiting "
"for deployment of instance %s")
if (CONF.baremetal.pxe_deploy_timeout
and timeutils.utcnow() > expiration):
locals['error'] = _("Timeout reached while waiting for "
"PXE deploy of instance %s")
if locals['error']:
raise loopingcall.LoopingCallDone()
def _cleanup_vol(ioctx, volume, retryctx):
try:
rbd.RBD().remove(client.ioctx, volume)
raise loopingcall.LoopingCallDone(retvalue=False)
except (rbd.ImageBusy, rbd.ImageHasSnapshots):
LOG.warn(_LW('rbd remove %(volume)s in pool %(pool)s '
'failed'),
{'volume': volume, 'pool': self.pool})
retryctx['retries'] -= 1
if retryctx['retries'] <= 0:
raise loopingcall.LoopingCallDone()
def _wait_for_power_off():
"""Called at an interval until the node's power is off."""
if self.is_power_on() is False:
self.state = baremetal_states.DELETED
raise loopingcall.LoopingCallDone()
if self.retries > CONF.baremetal.ipmi_power_retry:
self.state = baremetal_states.ERROR
raise loopingcall.LoopingCallDone()
try:
self.retries += 1
self._exec_ipmitool("power off")
except Exception:
LOG.exception(_("IPMI power off failed"))
def _wait_for_boot():
"""Called at an interval until the VM is running."""
statue = FC_MGR.get_vm_by_uuid(instance).status
if statue == constant.VM_STATUS.RUNNING:
LOG.debug(_("vm %s create success."), instance['name'])
boot_result['result'] = True
raise loopingcall.LoopingCallDone()
elif statue == constant.VM_STATUS.STOPPED:
LOG.debug(_("create vm %s success, but start failed."),
instance['name'])
raise loopingcall.LoopingCallDone()
else:
LOG.debug(_("vm %s is still in creating state."),
instance['name'])
def _wait_for_provision_state():
node = _validate_instance_and_node(ironicclient, instance)
if node.provision_state in (ironic_states.NOSTATE,
ironic_states.CLEANING,
ironic_states.CLEANFAIL,
ironic_states.AVAILABLE):
# From a user standpoint, the node is unprovisioned. If a node
# gets into CLEANFAIL state, it must be fixed in Ironic, but we
# can consider the instance unprovisioned.
LOG.debug(
"Ironic node %(node)s is in state %(state)s, "
"instance is now unprovisioned.",
dict(node=node.uuid, state=node.provision_state),
instance=instance)
raise loopingcall.LoopingCallDone()
if data['tries'] >= CONF.ironic.api_max_retries:
msg = (_("Error destroying the instance on node %(node)s. "
"Provision state still '%(state)s'.") % {
'state': node.provision_state,
'node': node.uuid
})
LOG.error(msg)
raise exception.NovaException(msg)
else:
data['tries'] += 1
_log_ironic_polling('unprovision', node, instance)
def _wait_for_active(self, icli, instance):
""" Wait for the node to be marked as ACTIVE in Ironic """
try:
node = icli.call("node.get_by_instance_uuid", instance['uuid'])
except ironic_exception.NotFound:
raise exception.InstanceNotFound(instance_id=instance['uuid'])
if node.provision_state == ironic_states.ACTIVE:
# job is done
raise loopingcall.LoopingCallDone()
if node.target_provision_state == ironic_states.DELETED:
# ironic is trying to delete it now
raise exception.InstanceNotFound(instance_id=instance['uuid'])
if node.provision_state == ironic_states.NOSTATE:
# ironic already deleted it
raise exception.InstanceNotFound(instance_id=instance['uuid'])
if node.provision_state == ironic_states.DEPLOYFAIL:
# ironic failed to deploy
msg = (_("Failed to provision instance %(inst)s: %(reason)s") % {
'inst': instance['uuid'],
'reason': node.last_error
})
raise exception.InstanceDeployFailure(msg)
def _wait_for_active(self, ironicclient, instance):
"""Wait for the node to be marked as ACTIVE in Ironic."""
instance.refresh()
if (instance.task_state == task_states.DELETING or
instance.vm_state in (vm_states.ERROR, vm_states.DELETED)):
raise exception.InstanceDeployFailure(
_("Instance %s provisioning was aborted") % instance.uuid)
node = _validate_instance_and_node(ironicclient, instance)
if node.provision_state == ironic_states.ACTIVE:
# job is done
LOG.debug("Ironic node %(node)s is now ACTIVE",
dict(node=node.uuid), instance=instance)
raise loopingcall.LoopingCallDone()
if node.target_provision_state in (ironic_states.DELETED,
ironic_states.AVAILABLE):
# ironic is trying to delete it now
raise exception.InstanceNotFound(instance_id=instance.uuid)
if node.provision_state in (ironic_states.NOSTATE,
ironic_states.AVAILABLE):
# ironic already deleted it
raise exception.InstanceNotFound(instance_id=instance.uuid)
if node.provision_state == ironic_states.DEPLOYFAIL:
# ironic failed to deploy
msg = (_("Failed to provision instance %(inst)s: %(reason)s")
% {'inst': instance.uuid, 'reason': node.last_error})
raise exception.InstanceDeployFailure(msg)
_log_ironic_polling('become ACTIVE', node, instance)
def _wait_for_active(self, icli, instance):
"""Wait for the node to be marked as ACTIVE in Ironic."""
node = _validate_instance_and_node(icli, instance)
if node.provision_state == ironic_states.ACTIVE:
# job is done
LOG.debug("Ironic node %(node)s is now ACTIVE",
dict(node=node.uuid),
instance=instance)
raise loopingcall.LoopingCallDone()
if node.target_provision_state == ironic_states.DELETED:
# ironic is trying to delete it now
raise exception.InstanceNotFound(instance_id=instance['uuid'])
if node.provision_state == ironic_states.NOSTATE:
# ironic already deleted it
raise exception.InstanceNotFound(instance_id=instance['uuid'])
if node.provision_state == ironic_states.DEPLOYFAIL:
# ironic failed to deploy
msg = (_("Failed to provision instance %(inst)s: %(reason)s") % {
'inst': instance['uuid'],
'reason': node.last_error
})
raise exception.InstanceDeployFailure(msg)
_log_ironic_polling('become ACTIVE', node, instance)
def _wait_for_device_discovery(host_devices, mount_device):
tries = self.tries
for device in host_devices:
LOG.debug(_("Looking for Fibre Channel dev %(device)s"),
{'device': device})
if os.path.exists(device):
self.host_device = device
# get the /dev/sdX device. This is used
# to find the multipath device.
self.device_name = os.path.realpath(device)
raise loopingcall.LoopingCallDone()
if self.tries >= CONF.num_iscsi_scan_tries:
msg = _("Fibre Channel device not found.")
raise exception.NovaException(msg)
LOG.warn(
_("Fibre volume not yet found at: %(mount_device)s. "
"Will rescan & retry. Try number: %(tries)s"), {
'mount_device': mount_device,
'tries': tries
})
linuxscsi.rescan_hosts(hbas)
self.tries = self.tries + 1
def _wait_for_power_on():
"""Called at an interval until the node's power is on."""
if self.is_power_on():
self.state = baremetal_states.ACTIVE
raise loopingcall.LoopingCallDone()
if self.retries > CONF.baremetal.ipmi_power_retry:
LOG.error(
_("IPMI power on failed after %d tries") %
(CONF.baremetal.ipmi_power_retry))
self.state = baremetal_states.ERROR
raise loopingcall.LoopingCallDone()
try:
self.retries += 1
self._exec_ipmitool("power on")
except Exception:
LOG.exception(_("IPMI power on failed"))
def _loop_check():
if alive_dict['fault']:
raise loopingcall.LoopingCallDone()
if alive_dict['alive']:
LOG.debug('compute service alive, host %s',
host_state,
instance=instance)
raise loopingcall.LoopingCallDone()
if alive_dict['count'] == 120:
LOG.debug('check alive timeout, host %s',
host_state,
instance=instance)
raise loopingcall.LoopingCallDone()
alive_dict['count'] += 1
def _wait_for_status_check():
"""Power state of a machine might be ON, but status check is the one which gives the real
"""
ec2_instance = self.ec2_conn.get_all_instance_status(instance_ids=[ec2_id])[0]
if ec2_instance.system_status.status == 'ok':
LOG.info("Instance status check is %s / %s" %
(ec2_instance.system_status.status, ec2_instance.instance_status.status))
raise loopingcall.LoopingCallDone()
def _wait_for_power_state(self, icli, instance, message):
"""Wait for the node to complete a power state change."""
node = _validate_instance_and_node(icli, instance)
if node.target_power_state == ironic_states.NOSTATE:
raise loopingcall.LoopingCallDone()
_log_ironic_polling(message, node, instance)
def _wait_for_power_state():
"""Called at an interval until the VM is running again.
"""
ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])
state = ec2_instance[0].state
if state == desired_state:
LOG.info("Instance has changed state to %s." % desired_state)
raise loopingcall.LoopingCallDone()
def _cleanup_vol(ioctx, volume, retryctx):
deletion_marker = '_to_be_deleted_by_glance'
try:
rbd.RBD().remove(client.ioctx, volume)
raise loopingcall.LoopingCallDone(retvalue=False)
except rbd.ImageHasSnapshots:
new_name = volume + deletion_marker
rbd.RBD().rename(client.ioctx, volume, new_name)
raise loopingcall.LoopingCallDone(retvalue=False)
except rbd.ImageBusy:
LOG.warn(
_LW('rbd remove %(volume)s in pool %(pool)s '
'failed'), {
'volume': volume,
'pool': self.pool
})
retryctx['retries'] -= 1
if retryctx['retries'] <= 0:
raise loopingcall.LoopingCallDone()
def _wait_done():
"""
wait task result
"""
task = self.get_task(task_uri)
if task['status'] == "success":
LOG.info(_("Task [%s] is successfully." % task_uri))
ret['success'] = True
raise loopingcall.LoopingCallDone()
elif task['status'] == "failed":
LOG.info(_("Task [%s] is failed, the reason is %s."), task_uri,
task['reasonDes'])
ret['reason'] = task['reasonDes']
raise loopingcall.LoopingCallDone()
else:
LOG.info(_("Task [%s] is running, the progress is %s."),
task_uri, task['progress'])
def _request_api():
try:
func(*args)
raise loopingcall.LoopingCallDone()
except ironic_exception.HTTPServiceUnavailable:
pass
if self.tries >= CONF.ironic.api_max_retries:
raise MaximumRetriesReached()
else:
self.tries += 1
def _wait_for_state():
"""Called at an interval until the AMI image is available."""
try:
images = self.ec2_conn.get_all_images(image_ids=[ami_id], owners=None,
executable_by=None, filters=None, dry_run=None)
state = images[0].state
# LOG.info("\n\n\nImage id = %s" % ami_id + ", state = %s\n\n\n" % state)
if state == desired_state:
LOG.info("Image has changed state to %s." % desired_state)
raise loopingcall.LoopingCallDone()
except boto_exc.EC2ResponseError:
pass
def _wait_for_reboot():
out,err = self._exec_xcat_command("nodestat %s" % nodename)
if err:
locals['errstr'] = _("Error returned when quering node status"
" for node %s:%s") % (nodename, err)
LOG.warning(locals['errstr'])
raise loopingcall.LoopingCallDone()
if out:
node,status = out.split(": ")
status = status.strip()
if status == "sshd":
LOG.info(_("Rebooting node %s completed.")
% nodename)
raise loopingcall.LoopingCallDone()
if (CONF.xcat.reboot_timeout and
timeutils.utcnow() > expiration):
locals['errstr'] = _("Timeout while waiting for"
" rebooting node %s.") % nodename
LOG.warning(locals['errstr'])
raise loopingcall.LoopingCallDone()
def _wait_for_remove(device, tries):
tries = tries + 1
LOG.debug("Trying (%(tries)s) to remove device %(device)s",
{'tries': tries, 'device': device["device"]})
path = "/sys/bus/scsi/drivers/sd/%s:%s:%s:%s/delete"
echo_scsi_command(path % (device["host"], device["channel"],
device["id"], device["lun"]),
"1")
devices = get_device_list()
if device["device"] not in devices:
raise loopingcall.LoopingCallDone()
def _wait_for_device_discovery(aoedevpath, mount_device):
tries = self.tries
if os.path.exists(aoedevpath):
raise loopingcall.LoopingCallDone()
if self.tries >= CONF.libvirt.num_aoe_discover_tries:
raise exception.NovaException(_("AoE device not found at %s") %
(aoedevpath))
LOG.warn(_LW("AoE volume not yet found at: %(aoedevpath)s. "
"Try number: %(tries)s"),
{'aoedevpath': aoedevpath, 'tries': tries})
self._aoe_discover()
self.tries = self.tries + 1
def _wait_for_handoff_send(print_log=False):
"""Called at an interval until VM synthesis finishes."""
returncode = proc.poll()
if returncode is None:
# keep record stdout
LOG.debug("waiting for finishing handoff send")
in_ready, _, _ = select.select([proc.stdout], [], [])
try:
buf = os.read(proc.stdout.fileno(), 1024 * 100)
if print_log:
LOG.debug(buf)
except OSError as e:
if e.errno == errno.EAGAIN or e.errno == errno.EWOULDBLOCK:
return
else:
raise loopingcall.LoopingCallDone()
def _wait_for_provision_state():
node = _validate_instance_and_node(icli, instance)
if not node.provision_state:
LOG.debug("Ironic node %(node)s is now unprovisioned",
dict(node=node.uuid), instance=instance)
raise loopingcall.LoopingCallDone()
if data['tries'] >= CONF.ironic.api_max_retries:
msg = (_("Error destroying the instance on node %(node)s. "
"Provision state still '%(state)s'.")
% {'state': node.provision_state,
'node': node.uuid})
LOG.error(msg)
raise exception.NovaException(msg)
else:
data['tries'] += 1
_log_ironic_polling('unprovision', node, instance)
def _wait_for_provision_state():
try:
node = icli.call("node.get_by_instance_uuid", instance['uuid'])
except ironic_exception.NotFound:
raise exception.InstanceNotFound(instance_id=instance['uuid'])
if not node.provision_state:
raise loopingcall.LoopingCallDone()
if data['tries'] >= CONF.ironic.api_max_retries:
msg = (_("Error destroying the instance on node %(node)s. "
"Provision state still '%(state)s'.") % {
'state': node.provision_state,
'node': node.uuid
})
LOG.error(msg)
raise exception.NovaException(msg)
else:
data['tries'] += 1
def _wait_for_lpar_status(instance_name, status):
"""Called at an interval until the status is reached."""
lpar_obj = self.get_lpar(instance_name)
if lpar_obj['state'] == status:
raise loopingcall.LoopingCallDone()
def _check_vm_status(instance_name):
if self._get_vm_state(instance_name) in desired_vm_states:
raise loopingcall.LoopingCallDone()
def _wait_for_boot():
"""Called at an interval until the VM is running."""
state = self.get_info(instance).state
if state == power_state.RUNNING:
raise loopingcall.LoopingCallDone()
def _check_power_state(instance):
current_state = get_power_state(instance)
LOG.debug("Wait for soft shutdown: (%s, %s)", current_state,
power_state)
if current_state == power_state:
raise loopingcall.LoopingCallDone()
