def _wait_for_deploy(): """Called at an interval until the deployment completes.""" try: row = db.bm_node_get(context, node['id']) if instance['uuid'] != row.get('instance_uuid'): locals['error'] = _("Node associated with another instance" " while waiting for deploy of %s") raise loopingcall.LoopingCallDone() status = row.get('task_state') if (status == baremetal_states.DEPLOYING and locals['started'] == False): LOG.info( _("PXE deploy started for instance %s") % instance['uuid']) locals['started'] = True elif status in (baremetal_states.DEPLOYDONE, baremetal_states.ACTIVE): LOG.info( _("PXE deploy completed for instance %s") % instance['uuid']) raise loopingcall.LoopingCallDone() elif status == baremetal_states.DEPLOYFAIL: locals['error'] = _("PXE deploy failed for instance %s") except exception.NodeNotFound: locals['error'] = _("Baremetal node deleted while waiting " "for deployment of instance %s") if (CONF.baremetal.pxe_deploy_timeout and timeutils.utcnow() > expiration): locals['error'] = _("Timeout reached while waiting for " "PXE deploy of instance %s") if locals['error']: raise loopingcall.LoopingCallDone()
def _cleanup_vol(ioctx, volume, retryctx): try: rbd.RBD().remove(client.ioctx, volume) raise loopingcall.LoopingCallDone(retvalue=False) except (rbd.ImageBusy, rbd.ImageHasSnapshots): LOG.warn(_LW('rbd remove %(volume)s in pool %(pool)s ' 'failed'), {'volume': volume, 'pool': self.pool}) retryctx['retries'] -= 1 if retryctx['retries'] <= 0: raise loopingcall.LoopingCallDone()
def _wait_for_power_off(): """Called at an interval until the node's power is off.""" if self.is_power_on() is False: self.state = baremetal_states.DELETED raise loopingcall.LoopingCallDone() if self.retries > CONF.baremetal.ipmi_power_retry: self.state = baremetal_states.ERROR raise loopingcall.LoopingCallDone() try: self.retries += 1 self._exec_ipmitool("power off") except Exception: LOG.exception(_("IPMI power off failed"))
def _wait_for_boot(): """Called at an interval until the VM is running.""" statue = FC_MGR.get_vm_by_uuid(instance).status if statue == constant.VM_STATUS.RUNNING: LOG.debug(_("vm %s create success."), instance['name']) boot_result['result'] = True raise loopingcall.LoopingCallDone() elif statue == constant.VM_STATUS.STOPPED: LOG.debug(_("create vm %s success, but start failed."), instance['name']) raise loopingcall.LoopingCallDone() else: LOG.debug(_("vm %s is still in creating state."), instance['name'])
def _wait_for_provision_state(): node = _validate_instance_and_node(ironicclient, instance) if node.provision_state in (ironic_states.NOSTATE, ironic_states.CLEANING, ironic_states.CLEANFAIL, ironic_states.AVAILABLE): # From a user standpoint, the node is unprovisioned. If a node # gets into CLEANFAIL state, it must be fixed in Ironic, but we # can consider the instance unprovisioned. LOG.debug( "Ironic node %(node)s is in state %(state)s, " "instance is now unprovisioned.", dict(node=node.uuid, state=node.provision_state), instance=instance) raise loopingcall.LoopingCallDone() if data['tries'] >= CONF.ironic.api_max_retries: msg = (_("Error destroying the instance on node %(node)s. " "Provision state still '%(state)s'.") % { 'state': node.provision_state, 'node': node.uuid }) LOG.error(msg) raise exception.NovaException(msg) else: data['tries'] += 1 _log_ironic_polling('unprovision', node, instance)
def _wait_for_active(self, icli, instance): """ Wait for the node to be marked as ACTIVE in Ironic """ try: node = icli.call("node.get_by_instance_uuid", instance['uuid']) except ironic_exception.NotFound: raise exception.InstanceNotFound(instance_id=instance['uuid']) if node.provision_state == ironic_states.ACTIVE: # job is done raise loopingcall.LoopingCallDone() if node.target_provision_state == ironic_states.DELETED: # ironic is trying to delete it now raise exception.InstanceNotFound(instance_id=instance['uuid']) if node.provision_state == ironic_states.NOSTATE: # ironic already deleted it raise exception.InstanceNotFound(instance_id=instance['uuid']) if node.provision_state == ironic_states.DEPLOYFAIL: # ironic failed to deploy msg = (_("Failed to provision instance %(inst)s: %(reason)s") % { 'inst': instance['uuid'], 'reason': node.last_error }) raise exception.InstanceDeployFailure(msg)
def _wait_for_active(self, ironicclient, instance): """Wait for the node to be marked as ACTIVE in Ironic.""" instance.refresh() if (instance.task_state == task_states.DELETING or instance.vm_state in (vm_states.ERROR, vm_states.DELETED)): raise exception.InstanceDeployFailure( _("Instance %s provisioning was aborted") % instance.uuid) node = _validate_instance_and_node(ironicclient, instance) if node.provision_state == ironic_states.ACTIVE: # job is done LOG.debug("Ironic node %(node)s is now ACTIVE", dict(node=node.uuid), instance=instance) raise loopingcall.LoopingCallDone() if node.target_provision_state in (ironic_states.DELETED, ironic_states.AVAILABLE): # ironic is trying to delete it now raise exception.InstanceNotFound(instance_id=instance.uuid) if node.provision_state in (ironic_states.NOSTATE, ironic_states.AVAILABLE): # ironic already deleted it raise exception.InstanceNotFound(instance_id=instance.uuid) if node.provision_state == ironic_states.DEPLOYFAIL: # ironic failed to deploy msg = (_("Failed to provision instance %(inst)s: %(reason)s") % {'inst': instance.uuid, 'reason': node.last_error}) raise exception.InstanceDeployFailure(msg) _log_ironic_polling('become ACTIVE', node, instance)
def _wait_for_active(self, icli, instance): """Wait for the node to be marked as ACTIVE in Ironic.""" node = _validate_instance_and_node(icli, instance) if node.provision_state == ironic_states.ACTIVE: # job is done LOG.debug("Ironic node %(node)s is now ACTIVE", dict(node=node.uuid), instance=instance) raise loopingcall.LoopingCallDone() if node.target_provision_state == ironic_states.DELETED: # ironic is trying to delete it now raise exception.InstanceNotFound(instance_id=instance['uuid']) if node.provision_state == ironic_states.NOSTATE: # ironic already deleted it raise exception.InstanceNotFound(instance_id=instance['uuid']) if node.provision_state == ironic_states.DEPLOYFAIL: # ironic failed to deploy msg = (_("Failed to provision instance %(inst)s: %(reason)s") % { 'inst': instance['uuid'], 'reason': node.last_error }) raise exception.InstanceDeployFailure(msg) _log_ironic_polling('become ACTIVE', node, instance)
def _wait_for_device_discovery(host_devices, mount_device): tries = self.tries for device in host_devices: LOG.debug(_("Looking for Fibre Channel dev %(device)s"), {'device': device}) if os.path.exists(device): self.host_device = device # get the /dev/sdX device. This is used # to find the multipath device. self.device_name = os.path.realpath(device) raise loopingcall.LoopingCallDone() if self.tries >= CONF.num_iscsi_scan_tries: msg = _("Fibre Channel device not found.") raise exception.NovaException(msg) LOG.warn( _("Fibre volume not yet found at: %(mount_device)s. " "Will rescan & retry. Try number: %(tries)s"), { 'mount_device': mount_device, 'tries': tries }) linuxscsi.rescan_hosts(hbas) self.tries = self.tries + 1
def _wait_for_power_on(): """Called at an interval until the node's power is on.""" if self.is_power_on(): self.state = baremetal_states.ACTIVE raise loopingcall.LoopingCallDone() if self.retries > CONF.baremetal.ipmi_power_retry: LOG.error( _("IPMI power on failed after %d tries") % (CONF.baremetal.ipmi_power_retry)) self.state = baremetal_states.ERROR raise loopingcall.LoopingCallDone() try: self.retries += 1 self._exec_ipmitool("power on") except Exception: LOG.exception(_("IPMI power on failed"))
def _loop_check(): if alive_dict['fault']: raise loopingcall.LoopingCallDone() if alive_dict['alive']: LOG.debug('compute service alive, host %s', host_state, instance=instance) raise loopingcall.LoopingCallDone() if alive_dict['count'] == 120: LOG.debug('check alive timeout, host %s', host_state, instance=instance) raise loopingcall.LoopingCallDone() alive_dict['count'] += 1
def _wait_for_status_check(): """Power state of a machine might be ON, but status check is the one which gives the real """ ec2_instance = self.ec2_conn.get_all_instance_status(instance_ids=[ec2_id])[0] if ec2_instance.system_status.status == 'ok': LOG.info("Instance status check is %s / %s" % (ec2_instance.system_status.status, ec2_instance.instance_status.status)) raise loopingcall.LoopingCallDone()
def _wait_for_power_state(self, icli, instance, message): """Wait for the node to complete a power state change.""" node = _validate_instance_and_node(icli, instance) if node.target_power_state == ironic_states.NOSTATE: raise loopingcall.LoopingCallDone() _log_ironic_polling(message, node, instance)
def _wait_for_power_state(): """Called at an interval until the VM is running again. """ ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id]) state = ec2_instance[0].state if state == desired_state: LOG.info("Instance has changed state to %s." % desired_state) raise loopingcall.LoopingCallDone()
def _cleanup_vol(ioctx, volume, retryctx): deletion_marker = '_to_be_deleted_by_glance' try: rbd.RBD().remove(client.ioctx, volume) raise loopingcall.LoopingCallDone(retvalue=False) except rbd.ImageHasSnapshots: new_name = volume + deletion_marker rbd.RBD().rename(client.ioctx, volume, new_name) raise loopingcall.LoopingCallDone(retvalue=False) except rbd.ImageBusy: LOG.warn( _LW('rbd remove %(volume)s in pool %(pool)s ' 'failed'), { 'volume': volume, 'pool': self.pool }) retryctx['retries'] -= 1 if retryctx['retries'] <= 0: raise loopingcall.LoopingCallDone()
def _wait_done(): """ wait task result """ task = self.get_task(task_uri) if task['status'] == "success": LOG.info(_("Task [%s] is successfully." % task_uri)) ret['success'] = True raise loopingcall.LoopingCallDone() elif task['status'] == "failed": LOG.info(_("Task [%s] is failed, the reason is %s."), task_uri, task['reasonDes']) ret['reason'] = task['reasonDes'] raise loopingcall.LoopingCallDone() else: LOG.info(_("Task [%s] is running, the progress is %s."), task_uri, task['progress'])
def _request_api(): try: func(*args) raise loopingcall.LoopingCallDone() except ironic_exception.HTTPServiceUnavailable: pass if self.tries >= CONF.ironic.api_max_retries: raise MaximumRetriesReached() else: self.tries += 1
def _wait_for_state(): """Called at an interval until the AMI image is available.""" try: images = self.ec2_conn.get_all_images(image_ids=[ami_id], owners=None, executable_by=None, filters=None, dry_run=None) state = images[0].state # LOG.info("\n\n\nImage id = %s" % ami_id + ", state = %s\n\n\n" % state) if state == desired_state: LOG.info("Image has changed state to %s." % desired_state) raise loopingcall.LoopingCallDone() except boto_exc.EC2ResponseError: pass
def _wait_for_reboot(): out,err = self._exec_xcat_command("nodestat %s" % nodename) if err: locals['errstr'] = _("Error returned when quering node status" " for node %s:%s") % (nodename, err) LOG.warning(locals['errstr']) raise loopingcall.LoopingCallDone() if out: node,status = out.split(": ") status = status.strip() if status == "sshd": LOG.info(_("Rebooting node %s completed.") % nodename) raise loopingcall.LoopingCallDone() if (CONF.xcat.reboot_timeout and timeutils.utcnow() > expiration): locals['errstr'] = _("Timeout while waiting for" " rebooting node %s.") % nodename LOG.warning(locals['errstr']) raise loopingcall.LoopingCallDone()
def _wait_for_remove(device, tries): tries = tries + 1 LOG.debug("Trying (%(tries)s) to remove device %(device)s", {'tries': tries, 'device': device["device"]}) path = "/sys/bus/scsi/drivers/sd/%s:%s:%s:%s/delete" echo_scsi_command(path % (device["host"], device["channel"], device["id"], device["lun"]), "1") devices = get_device_list() if device["device"] not in devices: raise loopingcall.LoopingCallDone()
def _wait_for_device_discovery(aoedevpath, mount_device): tries = self.tries if os.path.exists(aoedevpath): raise loopingcall.LoopingCallDone() if self.tries >= CONF.libvirt.num_aoe_discover_tries: raise exception.NovaException(_("AoE device not found at %s") % (aoedevpath)) LOG.warn(_LW("AoE volume not yet found at: %(aoedevpath)s. " "Try number: %(tries)s"), {'aoedevpath': aoedevpath, 'tries': tries}) self._aoe_discover() self.tries = self.tries + 1
def _wait_for_handoff_send(print_log=False): """Called at an interval until VM synthesis finishes.""" returncode = proc.poll() if returncode is None: # keep record stdout LOG.debug("waiting for finishing handoff send") in_ready, _, _ = select.select([proc.stdout], [], []) try: buf = os.read(proc.stdout.fileno(), 1024 * 100) if print_log: LOG.debug(buf) except OSError as e: if e.errno == errno.EAGAIN or e.errno == errno.EWOULDBLOCK: return else: raise loopingcall.LoopingCallDone()
def _wait_for_provision_state(): node = _validate_instance_and_node(icli, instance) if not node.provision_state: LOG.debug("Ironic node %(node)s is now unprovisioned", dict(node=node.uuid), instance=instance) raise loopingcall.LoopingCallDone() if data['tries'] >= CONF.ironic.api_max_retries: msg = (_("Error destroying the instance on node %(node)s. " "Provision state still '%(state)s'.") % {'state': node.provision_state, 'node': node.uuid}) LOG.error(msg) raise exception.NovaException(msg) else: data['tries'] += 1 _log_ironic_polling('unprovision', node, instance)
def _wait_for_provision_state(): try: node = icli.call("node.get_by_instance_uuid", instance['uuid']) except ironic_exception.NotFound: raise exception.InstanceNotFound(instance_id=instance['uuid']) if not node.provision_state: raise loopingcall.LoopingCallDone() if data['tries'] >= CONF.ironic.api_max_retries: msg = (_("Error destroying the instance on node %(node)s. " "Provision state still '%(state)s'.") % { 'state': node.provision_state, 'node': node.uuid }) LOG.error(msg) raise exception.NovaException(msg) else: data['tries'] += 1
def _wait_for_lpar_status(instance_name, status): """Called at an interval until the status is reached.""" lpar_obj = self.get_lpar(instance_name) if lpar_obj['state'] == status: raise loopingcall.LoopingCallDone()
def _check_vm_status(instance_name): if self._get_vm_state(instance_name) in desired_vm_states: raise loopingcall.LoopingCallDone()
def _wait_for_boot(): """Called at an interval until the VM is running.""" state = self.get_info(instance).state if state == power_state.RUNNING: raise loopingcall.LoopingCallDone()
def _check_power_state(instance): current_state = get_power_state(instance) LOG.debug("Wait for soft shutdown: (%s, %s)", current_state, power_state) if current_state == power_state: raise loopingcall.LoopingCallDone()