def _wait_for_switch(mutable):
if mutable['retries'] > CONF.iboot.max_retry:
LOG.warning(_LW(
'Reached maximum number of attempts (%(attempts)d) to set '
'power state for node %(node)s to "%(op)s"'),
{'attempts': mutable['retries'], 'node': driver_info['uuid'],
'op': states.POWER_ON if enabled else states.POWER_OFF})
raise loopingcall.LoopingCallDone()
try:
mutable['retries'] += 1
mutable['response'] = conn.switch(relay_id, enabled)
if mutable['response']:
raise loopingcall.LoopingCallDone()
except (TypeError, IndexError):
LOG.warning(_LW("Cannot call set power state for node '%(node)s' "
"at relay '%(relay)s'. iBoot switch() failed."),
{'node': driver_info['uuid'], 'relay': relay_id})
def update_health_status(self):
LOG.debug("Updating health status for cluster %s", self.cluster.id)
self._update_health_status()
LOG.debug("Status for cluster %s updated to %s (%s)", self.cluster.id,
self.cluster.health_status,
self.cluster.health_status_reason)
# TODO(flwang): Health status update notifications?
# end the "loop"
raise loopingcall.LoopingCallDone()
def _cleanup_vol(ioctx, volume, retryctx):
try:
RbdProxy().remove(ioctx, volume)
raise loopingcall.LoopingCallDone(retvalue=False)
except rbd.ImageHasSnapshots:
self.remove_snap(volume,
libvirt_utils.RESIZE_SNAPSHOT_NAME,
ignore_errors=True)
except (rbd.ImageBusy, rbd.ImageHasSnapshots):
LOG.warn(
_LW('rbd remove %(volume)s in pool %(pool)s '
'failed'), {
'volume': volume,
'pool': self.pool
})
retryctx['retries'] -= 1
if retryctx['retries'] <= 0:
raise loopingcall.LoopingCallDone()
def _check_create_cloned_volume_finish(self, new_volume_name):
LOG.debug(
'Loopcall: _check_create_cloned_volume_finish(), '
'volume-name: %s.', new_volume_name)
current_volume = self.dsware_client.query_volume(new_volume_name)
if current_volume:
status = current_volume['status']
LOG.debug('Wait clone volume %(volume_name)s, status: %(status)s.',
{
"volume_name": new_volume_name,
"status": status
})
if int(status) == self.DSWARE_VOLUME_CREATING_STATUS or int(
status) == self.DSWARE_VOLUME_DUPLICATE_VOLUME:
self.count += 1
elif int(status) == self.DSWARE_VOLUME_CREATE_SUCCESS_STATUS:
raise loopingcall.LoopingCallDone(retvalue=True)
else:
msg = _('Clone volume %(new_volume_name)s failed, '
'volume status is: %(status)s.')
LOG.error(msg, {
'new_volume_name': new_volume_name,
'status': status
})
raise loopingcall.LoopingCallDone(retvalue=False)
if self.count > self.configuration.clone_volume_timeout:
msg = _('Dsware clone volume time out. '
'Volume: %(new_volume_name)s, status: %(status)s')
LOG.error(
msg, {
'new_volume_name': new_volume_name,
'status': current_volume['status']
})
raise loopingcall.LoopingCallDone(retvalue=False)
else:
LOG.warning(_LW('Can not find volume %s from Dsware.'),
new_volume_name)
self.count += 1
if self.count > 10:
msg = _("Dsware clone volume failed: volume "
"can not be found from Dsware.")
LOG.error(msg)
raise loopingcall.LoopingCallDone(retvalue=False)
def poll_and_check(self):
# TODO(yuanying): temporary implementation to update api_address,
# node_addresses and bay status
stack = self.openstack_client.heat().stacks.get(self.bay.stack_id)
self.attempts += 1
# poll_and_check is detached and polling long time to check status,
# so another user/client can call delete bay/stack.
if stack.stack_status == bay_status.DELETE_COMPLETE:
self._delete_complete()
raise loopingcall.LoopingCallDone()
if stack.stack_status in (bay_status.CREATE_COMPLETE,
bay_status.UPDATE_COMPLETE):
self._sync_bay_and_template_status(stack)
raise loopingcall.LoopingCallDone()
elif stack.stack_status != self.bay.status:
self._sync_bay_status(stack)
if stack.stack_status in (bay_status.CREATE_FAILED,
bay_status.DELETE_FAILED,
bay_status.UPDATE_FAILED):
self._sync_bay_and_template_status(stack)
self._bay_failed(stack)
raise loopingcall.LoopingCallDone()
# only check max attempts when the stack is being created when
# the timeout hasn't been set. If the timeout has been set then
# the loop will end when the stack completes or the timeout occurs
if stack.stack_status == bay_status.CREATE_IN_PROGRESS:
if (stack.timeout_mins is None and
self.attempts > cfg.CONF.bay_heat.max_attempts):
LOG.error(_LE('Bay check exit after %(attempts)s attempts,'
'stack_id: %(id)s, stack_status: %(status)s') %
{'attempts': cfg.CONF.bay_heat.max_attempts,
'id': self.bay.stack_id,
'status': stack.stack_status})
raise loopingcall.LoopingCallDone()
else:
if self.attempts > cfg.CONF.bay_heat.max_attempts:
LOG.error(_LE('Bay check exit after %(attempts)s attempts,'
'stack_id: %(id)s, stack_status: %(status)s') %
{'attempts': cfg.CONF.bay_heat.max_attempts,
'id': self.bay.stack_id,
'status': stack.stack_status})
raise loopingcall.LoopingCallDone()
def _wait_for_disk(device, retries, max_retries, pids, stderr):
# A regex is likely overkill here, but variations in fuser
# means we should likely use it.
fuser_pids_re = re.compile(r'\d+')
retries[0] += 1
if retries[0] > max_retries:
raise loopingcall.LoopingCallDone()
# There are 'psmisc' and 'busybox' versions of the 'fuser' program. The
# 'fuser' programs differ in how they output data to stderr. The
# busybox version does not output the filename to stderr, while the
# standard 'psmisc' version does output the filename to stderr. How
# they output to stdout is almost identical in that only the PIDs are
# output to stdout, with the 'psmisc' version adding a leading space
# character to the list of PIDs.
try:
# NOTE(ifarkas): fuser returns a non-zero return code if none of
# the specified files is accessed.
# NOTE(TheJulia): fuser does not report LVM devices as in use
# unless the LVM device-mapper device is the
# device that is directly polled.
# NOTE(TheJulia): The -m flag allows fuser to reveal data about
# mounted filesystems, which should be considered
# busy/locked. That being said, it is not used
# because busybox fuser has a different behavior.
# NOTE(TheJuia): fuser outputs a list of found PIDs to stdout.
# All other text is returned via stderr, and the
# output to a terminal is merged as a result.
out, err = execute('fuser', device, check_exit_code=[0, 1],
run_as_root=True)
if not out and not err:
raise loopingcall.LoopingCallDone()
stderr[0] = err
# NOTE: findall() returns a list of matches, or an empty list if no
# matches
pids[0] = fuser_pids_re.findall(out)
except processutils.ProcessExecutionError as exc:
LOG.warning('Failed to check the device %(device)s with fuser:'******' %(err)s', {'device': device, 'err': exc})
def _wait_for_status_check():
"""Power state of a machine might be ON, but status check is the one which gives the real
"""
ec2_instance = self.ec2_conn.get_all_instance_status(
instance_ids=[ec2_id])[0]
if ec2_instance.system_status.status == 'ok':
LOG.info("Instance status check is %s / %s" %
(ec2_instance.system_status.status,
ec2_instance.instance_status.status))
raise loopingcall.LoopingCallDone()
def test_no_sleep(self, sleep_mock, random_mock):
# Any call that executes properly the first time shouldn't sleep
random_mock.return_value = 1
func = mock.Mock()
# func.side_effect
func.side_effect = loopingcall.LoopingCallDone(retvalue='return value')
retvalue = loopingcall.BackOffLoopingCall(func).start().wait()
self.assertFalse(sleep_mock.called)
self.assertTrue(retvalue, 'return value')
def _inner():
result = func()
if result:
raise loopingcall.LoopingCallDone()
if int(time.time()) - start_time > timeout:
msg = (_('wait_for_condition: %s timed out.') % func.__name__)
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
def _wait_for_power_state():
"""Called at an interval until the VM is running again.
"""
ec2_instance = self.ec2_conn.get_only_instances(
instance_ids=[ec2_id])
state = ec2_instance[0].state
if state == desired_state:
LOG.info("Instance has changed state to %s." % desired_state)
raise loopingcall.LoopingCallDone()
def _wait(mutable):
mutable['boot_status_value'] = snmp_client.get(BOOT_STATUS_OID)
LOG.debug(
"iRMC SNMP agent of %(node_id)s returned "
"boot status value %(bootstatus)s on attempt %(times)s.", {
'node_id': node.uuid,
'bootstatus': BOOT_STATUS[mutable['boot_status_value']],
'times': mutable['times']
})
if _is_expected_power_state(target_state,
mutable['boot_status_value']):
mutable['state'] = target_state
raise loopingcall.LoopingCallDone()
mutable['times'] += 1
if mutable['times'] > max_retry:
mutable['state'] = states.ERROR
raise loopingcall.LoopingCallDone()
def _wait_state(state):
"""Wait for the expected console mode to be set on node."""
console = _get_console()
if console['console_enabled'] == state:
raise loopingcall.LoopingCallDone(retvalue=console)
_log_ironic_polling('set console mode', node, server)
# Return False to start backing off
return False
def _wait_for_pair_status(self, pvol, svol, is_vvol, status, timeout,
start):
if self._comm_pairevtwait(pvol, svol, is_vvol) in status:
raise loopingcall.LoopingCallDone()
if time.time() - start >= timeout:
msg = basic_lib.output_err(637,
method='_wait_for_pair_status',
timeout=timeout)
raise exception.HBSDError(message=msg)
def _wait_for_msi_irqs(self, inst):
"""Check if each pci device has the expected number of msi irqs."""
_prev = self._msi_irq_count.copy()
addrs = set()
for pci_dev in inst.pci_devices:
addr = pci_dev.address
addrs.update([addr])
try:
irqs, msi_irqs = pci_utils.get_irqs_by_pci_address(addr)
except Exception as e:
msi_irqs = set()
LOG.error(
'_wait_for_msi_irqs: pci_addr=%(A)s, error=%(E)s' % {
'A': addr,
'E': e
})
self._msi_irq_count[addr] = len(msi_irqs)
self._msi_irq_elapsed[
addr] += CONF.parameters.msi_irq_check_interval
if _prev[addr] == self._msi_irq_count[addr]:
self._msi_irq_since[
addr] += CONF.parameters.msi_irq_check_interval
else:
self._msi_irq_since[addr] = 0
# Done when msi irq counts have not changed for some time
if all((self._msi_irq_count[k] > 0) and (
self._msi_irq_since[k] >= CONF.parameters.msi_irq_since)
for k in addrs):
raise loopingcall.LoopingCallDone()
# Abort due to timeout
if all(self._msi_irq_elapsed[k] >= CONF.parameters.msi_irq_timeout
for k in addrs):
msg = ("reached %(timeout)s seconds timeout, waiting for "
"msi irqs of pci_addrs: %(addrs)s") % {
'timeout': CONF.parameters.msi_irq_timeout,
'addrs': list(addrs)
}
LOG.warning(msg)
raise loopingcall.LoopingCallDone()
def _wait(store):
current_power_state = None
with common.cimc_handle(task) as handle:
try:
rack_unit = handle.get_imc_managedobject(
None, None, params={"Dn": "sys/rack-unit-1"})
except imcsdk.ImcException as e:
raise exception.CIMCException(node=task.node.uuid, error=e)
else:
current_power_state = rack_unit[0].get_attr("OperPower")
store['state'] = CIMC_TO_IRONIC_POWER_STATE.get(current_power_state)
if store['state'] == target_state:
raise loopingcall.LoopingCallDone()
store['retries'] -= 1
if store['retries'] <= 0:
store['state'] = states.ERROR
raise loopingcall.LoopingCallDone()
def _sync_status(self, checkpoint, stack_id):
try:
stack = self._heat_client.stacks.get(stack_id)
stack_status = getattr(stack, 'stack_status')
if stack_status == 'CREATE_IN_PROGRESS':
return
raise loopingcall.LoopingCallDone()
except Exception as err:
LOG.info(_("stop sync stack status, stack_id:%s"), stack_id)
raise err
def _is_vdisk_copy_in_progress(self, vdisk_name):
LOG.debug(
'_is_vdisk_copy_in_progress: %(vdisk)s: %(vdisk_in_progress)s.', {
'vdisk': vdisk_name,
'vdisk_in_progress': six.text_type(
self._vdisk_copy_in_progress)
})
if vdisk_name not in self._vdisk_copy_in_progress:
LOG.debug('_is_vdisk_copy_in_progress: '
'vdisk copy is not in progress.')
raise loopingcall.LoopingCallDone(retvalue=True)
def _wait_for_evacuation():
new_instance = self.novaclient.get_server(context, instance.id)
instance_host = getattr(new_instance,
"OS-EXT-SRV-ATTR:hypervisor_hostname")
old_vm_state = getattr(instance, "OS-EXT-STS:vm_state")
new_vm_state = getattr(new_instance, "OS-EXT-STS:vm_state")
if instance_host != host_name:
if ((old_vm_state == 'error' and new_vm_state == 'active')
or old_vm_state == new_vm_state):
raise loopingcall.LoopingCallDone()
def _wait():
if ilo_object.has_disk_erase_completed():
raise loopingcall.LoopingCallDone()
time_elps[0] += interval
LOG.debug(
"%(tim)s secs elapsed while waiting for out-of-band "
"sanitize disk erase to complete for node %(node)s.", {
'tim': time_elps[0],
'node': node.uuid
})
def waiting():
result = self.driver.zoneOperations().get(
project=config['project'],
zone=config['zone'],
operation=operation['name']).execute()
LOG.info('Operation state {}'.format(result['status']))
if result['status'] == 'DONE':
print("done.")
if 'error' in result:
raise Exception(result['error'])
raise loopingcall.LoopingCallDone()
def test_lookup_node(self, lookup_mock, sleep_mock):
content = {
'node': {
'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c'
},
'heartbeat_timeout': 300
}
lookup_mock.side_effect = loopingcall.LoopingCallDone(retvalue=content)
returned_content = self.api_client.lookup_node(
hardware_info=self.hardware_info, timeout=300, starting_interval=1)
self.assertEqual(content, returned_content)
def _wait_for_user_list():
try:
all_users = self.get_user_names(client, instance_id)
self.report.log("Users in the db instance %s: %s" %
(instance_id, all_users))
except Exception as e:
self.report.log(
"Failed to list users in db instance %s(will continue), "
"error: %s" % (instance_id, str(e))
)
else:
raise loopingcall.LoopingCallDone()
def test_no_backoff(self, sleep_mock, random_mock):
random_mock.return_value = 1
func = mock.Mock()
# func.side_effect
func.side_effect = [True, True, True, loopingcall.LoopingCallDone(
retvalue='return value')]
retvalue = loopingcall.BackOffLoopingCall(func).start().wait()
expected_times = [mock.call(1), mock.call(1), mock.call(1)]
self.assertEqual(expected_times, sleep_mock.call_args_list)
self.assertTrue(retvalue, 'return value')
def _loop_func():
LOG.debug(
"Entering _wait_for_lun_or_snap_copy loop: "
"vdev=%s, objid=%s", dest_vdev_id, dest_obj_id)
target_id, mb_copied, percent = wait_func(src_vol_id)
if target_id is None:
# pre-copy transient result
LOG.debug("lun or snap copy prepping.")
pass
elif target_id != wait_id:
# the copy is complete, another lun is being copied
LOG.debug("lun or snap copy complete.")
raise loopingcall.LoopingCallDone(retvalue=True)
elif mb_copied is not None:
# copy is in progress
LOG.debug("MB copied: %(copied)d, percent done: %(percent)d.",
{
'copied': mb_copied,
'percent': percent
})
pass
elif percent == 0:
# copy has just started
LOG.debug("lun or snap copy started.")
pass
elif percent == 100:
# copy is complete
LOG.debug("lun or snap copy complete.")
raise loopingcall.LoopingCallDone(retvalue=True)
else:
# unexpected case
LOG.debug(
"unexpected case (%(id)s, %(bytes)s, %(percent)s)", {
'id': target_id,
'bytes': mb_copied,
'percent': six.text_type(percent)
})
raise loopingcall.LoopingCallDone(retvalue=False)
def test_create(self, mock_openstack_client_class, mock_driver, mock_cm,
mock_trust_manager, mock_heat_poller_class):
timeout = 15
mock_poller = mock.MagicMock()
mock_poller.poll_and_check.return_value = loopingcall.LoopingCallDone()
mock_heat_poller_class.return_value = mock_poller
osc = mock.sentinel.osc
def return_keystone():
return self.keystone_client
osc.keystone = return_keystone
mock_openstack_client_class.return_value = osc
mock_dr = mock.MagicMock()
mock_driver.return_value = mock_dr
def create_stack_side_effect(context, osc, cluster, timeout):
return {'stack': {'id': 'stack-id'}}
mock_dr.create_stack.side_effect = create_stack_side_effect
# Just create a new cluster, since the one in setUp is already
# created and the previous solution seems kind of hacky.
cluster_dict = utils.get_test_cluster(node_count=1)
cluster = objects.Cluster(self.context, **cluster_dict)
node_count = 1
master_count = 1
del cluster_dict['id']
del cluster_dict['uuid']
cluster_obj = objects.Cluster(self.context, **cluster_dict)
cluster = self.handler.cluster_create(self.context, cluster_obj,
master_count, node_count,
timeout)
notifications = fake_notifier.NOTIFICATIONS
self.assertEqual(1, len(notifications))
self.assertEqual('magnum.cluster.create', notifications[0].event_type)
self.assertEqual(taxonomy.OUTCOME_PENDING,
notifications[0].payload['outcome'])
mock_dr.create_cluster.assert_called_once_with(self.context, cluster,
timeout)
mock_cm.generate_certificates_to_cluster.assert_called_once_with(
cluster, context=self.context)
self.assertEqual(cluster_status.CREATE_IN_PROGRESS, cluster.status)
mock_trust_manager.create_trustee_and_trust.assert_called_once_with(
osc, cluster)
self.assertEqual(2, len(cluster.nodegroups))
self.assertEqual(node_count, cluster.node_count)
self.assertEqual(master_count, cluster.master_count)
self.assertEqual(node_count, cluster.default_ng_worker.node_count)
self.assertEqual(master_count, cluster.default_ng_master.node_count)
def _wait():
try:
res = cls.client.get_resource("backups", id)
cur_status = res["backup"]["status"]
except exceptions.NotFound:
if need_delete or "DELETED" in expected_status:
LOG.info('Backup %s is deleted', id)
raise loopingcall.LoopingCallDone()
return
if cur_status in expected_status:
LOG.info('Backup %s becomes %s', id, cur_status)
raise loopingcall.LoopingCallDone()
elif "FAILED" not in expected_status and cur_status == "FAILED":
# If backup status goes to FAILED but is not expected, stop
# waiting
message = "Backup status is FAILED."
caller = test_utils.find_test_caller()
if caller:
message = '({caller}) {message}'.format(caller=caller,
message=message)
raise exceptions.UnexpectedResponseCode(message)
def _wait_for_state():
"""Called at an interval until the AMI image is available."""
try:
# LOG.info("\n\n\nImage id = %s" % ami_id + ", state = %s\n\n\n" % state)
image = self.driver.Image(ami.id)
if image.state == desired_state:
LOG.info("Image has changed state to %s." % desired_state)
raise loopingcall.LoopingCallDone()
else:
LOG.info("Image state %s." % image.state)
except BotoCoreError as e:
LOG.info("BotoCoreError: {}".format(e))
pass
def _wait_for_power_status(mutable):
if mutable['retries'] > CONF.iboot.max_retry:
LOG.warning(_LW(
'Reached maximum number of attempts (%(attempts)d) to get '
'power state for node %(node)s'),
{'attempts': mutable['retries'], 'node': driver_info['uuid']})
raise loopingcall.LoopingCallDone()
try:
mutable['retries'] += 1
response = conn.get_relays()
status = response[relay_id - 1]
if status:
mutable['state'] = states.POWER_ON
else:
mutable['state'] = states.POWER_OFF
raise loopingcall.LoopingCallDone()
except (TypeError, IndexError):
LOG.warning(_LW("Cannot get power state for node '%(node)s' at "
"relay '%(relay)s'. iBoot get_relays() failed."),
{'node': driver_info['uuid'], 'relay': relay_id})
def _unlink_grp():
"""Called at an interval until the synchronization is finished.
:raises: loopingcall.LoopingCallDone
"""
retries = kwargs['retries']
try:
kwargs['retries'] = retries + 1
if not kwargs['modify_grp_snap_success']:
self.rest.modify_storagegroup_snap(
array, source_group_name, target_group_name,
snap_name, extra_specs, unlink=True)
kwargs['modify_grp_snap_success'] = True
except exception.VolumeBackendAPIException:
pass
if kwargs['retries'] > UNLINK_RETRIES:
LOG.error("_unlink_grp failed after %(retries)d "
"tries.", {'retries': retries})
raise loopingcall.LoopingCallDone(retvalue=30)
if kwargs['modify_grp_snap_success']:
raise loopingcall.LoopingCallDone()
def _wait_for_evacuation_confirmation():
old_vm_state, new_vm_state, instance_host = (
self._get_state_and_host_of_instance(context, instance))
if (new_vm_state == 'error' and
new_vm_state != old_vm_state):
raise exception.InstanceEvacuateFailed(
instance_uuid=instance.id)
if instance_host != host_name:
if ((old_vm_state == 'error' and
new_vm_state == 'active') or
old_vm_state == new_vm_state):
raise loopingcall.LoopingCallDone()
