def wait(self, session, timeout=None, ignore_error=False):
"""Wait for the allocation to become active.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param timeout: How much (in seconds) to wait for the allocation.
The value of ``None`` (the default) means no client-side timeout.
:param ignore_error: If ``True``, this call will raise an exception
if the allocation reaches the ``error`` state. Otherwise the error
state is considered successful and the call returns.
:return: This :class:`Allocation` instance.
"""
if self.state == 'active':
return self
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the allocation %s" % self.id):
self.fetch(session)
if self.state == 'error' and not ignore_error:
raise exceptions.SDKException(
"Allocation %(allocation)s failed: %(error)s" %
{'allocation': self.id, 'error': self.last_error})
elif self.state != 'allocating':
return self
session.log.debug(
'Still waiting for the allocation %(allocation)s '
'to become active, the current state is %(state)s',
{'allocation': self.id, 'state': self.state})
def wait_for_power_state(self, session, expected_state, timeout=None):
"""Wait for the node to reach the expected power state.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param expected_state: The expected power state to reach.
:param timeout: If ``wait`` is set to ``True``, specifies how much (in
seconds) to wait for the expected state to be reached. The value of
``None`` (the default) means no client-side timeout.
:return: This :class:`Node` instance.
:raises: :class:`~openstack.exceptions.ResourceTimeout` on timeout.
"""
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for node %(node)s to reach "
"power state '%(state)s'" % {'node': self.id,
'state': expected_state}):
self.fetch(session)
if self.power_state == expected_state:
return self
session.log.debug(
'Still waiting for node %(node)s to reach power state '
'"%(target)s", the current state is "%(state)s"',
{'node': self.id, 'target': expected_state,
'state': self.power_state})
def test_iterate_timeout_int_wait(self, mock_sleep):
iter = utils.iterate_timeout(10,
"test_iterate_timeout_int_wait",
wait=1)
next(iter)
next(iter)
mock_sleep.assert_called_with(1.0)
def wait_for_delete_instance(self, instance, interval=2, wait=180):
"""Wait for the instance to be deleted.
:param instance:
The :class:`~otcextensions.sdk.auto_scaling.v1.instance.Instance`
or instance ID to wait on to be deleted.
:param int interval:
Number of seconds to wait before to consecutive checks.
Default to 2.
:param int wait:
Maximum number of seconds to wait for the delete.
Default to 180.
:return: Method returns self on success.
:raises: :class:`~openstack.exceptions.ResourceTimeout` transition
to status failed to occur in wait seconds.
"""
instance = self._get_resource(_instance.Instance, instance)
for count in utils.iterate_timeout(
timeout=wait,
message="Timeout waiting for instance to delete",
wait=interval):
instance = self._find(_instance.Instance,
name_or_id=instance.id,
group_id=instance.scaling_group_id,
ignore_missing=True)
if instance is None:
return
def wait_for_delete(session, resource, interval, wait):
"""Wait for the resource to be deleted.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param resource: The resource to wait on to be deleted.
:type resource: :class:`~openstack.resource.Resource`
:param interval: Number of seconds to wait between checks.
:param wait: Maximum number of seconds to wait for the delete.
:return: Method returns self on success.
:raises: :class:`~openstack.exceptions.ResourceTimeout` transition
to status failed to occur in wait seconds.
"""
orig_resource = resource
for count in utils.iterate_timeout(
timeout=wait,
message="Timeout waiting for {res}:{id} to delete".format(
res=resource.__class__.__name__,
id=resource.id),
wait=interval):
try:
resource = resource.get(session)
if not resource:
return orig_resource
if resource.status.lower() == 'deleted':
return resource
except exceptions.NotFoundException:
return orig_resource
def delete_cluster_receiver(self, name_or_id, wait=False, timeout=3600):
receiver = self.get_cluster_receiver(name_or_id)
if receiver is None:
self.log.debug("Receiver %s not found for deleting", name_or_id)
return False
receiver_id = receiver['id']
self._clustering_client.delete(
"/receivers/{receiver_id}".format(receiver_id=receiver_id),
error_message="Error deleting receiver {name}".format(
name=name_or_id))
if not wait:
return True
for count in utils.iterate_timeout(
timeout, "Timeout waiting for cluster receiver to delete"):
receiver = self.get_cluster_receiver_by_id(receiver_id)
if not receiver:
break
return True
def wait_for_provision_state(self, session, expected_state, timeout=None,
abort_on_failed_state=True):
"""Wait for the node to reach the expected state.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param expected_state: The expected provisioning state to reach.
:param timeout: If ``wait`` is set to ``True``, specifies how much (in
seconds) to wait for the expected state to be reached. The value of
``None`` (the default) means no client-side timeout.
:param abort_on_failed_state: If ``True`` (the default), abort waiting
if the node reaches a failure state which does not match the
expected one. Note that the failure state for ``enroll`` ->
``manageable`` transition is ``enroll`` again.
:return: This :class:`Node` instance.
"""
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for node %(node)s to reach "
"target state '%(state)s'" % {'node': self.id,
'state': expected_state}):
self.fetch(session)
if self._check_state_reached(session, expected_state,
abort_on_failed_state):
return self
_logger.debug('Still waiting for node %(node)s to reach state '
'"%(target)s", the current state is "%(state)s"',
{'node': self.id, 'target': expected_state,
'state': self.provision_state})
def test_detach_ip_from_server(self):
self._setup_networks()
new_server = self.user_cloud.create_server(wait=True,
name=self.new_item_name +
'_server',
image=self.image,
flavor=self.flavor,
nics=[self.nic])
# ToDo: remove the following iteration when create_server waits for
# the IP to be attached
ip = None
for _ in utils.iterate_timeout(
self.timeout, "Timeout waiting for IP address to be attached"):
ip = meta.get_server_external_ipv4(self.user_cloud, new_server)
if ip is not None:
break
new_server = self.user_cloud.get_server(new_server.id)
self.addCleanup(self._cleanup_ips, new_server)
f_ip = self.user_cloud.get_floating_ip(
id=None, filters={'floating_ip_address': ip})
self.user_cloud.detach_ip_from_server(server_id=new_server.id,
floating_ip_id=f_ip['id'])
def delete_volume_snapshot(self, name_or_id=None, wait=False,
timeout=None):
"""Delete a volume snapshot.
:param name_or_id: Name or unique ID of the volume snapshot.
:param wait: If true, waits for volume snapshot to be deleted.
:param timeout: Seconds to wait for volume snapshot deletion. None is
forever.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
volumesnapshot = self.get_volume_snapshot(name_or_id)
if not volumesnapshot:
return False
self._volume_client.delete(
'/snapshots/{snapshot_id}'.format(
snapshot_id=volumesnapshot['id']),
error_message="Error in deleting volume snapshot")
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the volume snapshot to be deleted."):
if not self.get_volume_snapshot(volumesnapshot['id']):
break
return True
def wait(self, session, timeout=None, ignore_error=False):
"""Wait for the allocation to become active.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param timeout: How much (in seconds) to wait for the allocation.
The value of ``None`` (the default) means no client-side timeout.
:param ignore_error: If ``True``, this call will raise an exception
if the allocation reaches the ``error`` state. Otherwise the error
state is considered successful and the call returns.
:return: This :class:`Allocation` instance.
"""
if self.state == 'active':
return self
for count in utils.iterate_timeout(
timeout, "Timeout waiting for the allocation %s" % self.id):
self.fetch(session)
if self.state == 'error' and not ignore_error:
raise exceptions.SDKException(
"Allocation %(allocation)s failed: %(error)s" % {
'allocation': self.id,
'error': self.last_error
})
elif self.state != 'allocating':
return self
session.log.debug(
'Still waiting for the allocation %(allocation)s '
'to become active, the current state is %(state)s', {
'allocation': self.id,
'state': self.state
})
def test_iterate_timeout_bad_wait(self):
with testtools.ExpectedException(
exc.OpenStackCloudException,
"Wait value must be an int or float value."):
for count in utils.iterate_timeout(
1, "test_iterate_timeout_bad_wait", wait="timeishard"):
pass
def delete_volume_snapshot(self,
name_or_id=None,
wait=False,
timeout=None):
"""Delete a volume snapshot.
:param name_or_id: Name or unique ID of the volume snapshot.
:param wait: If true, waits for volume snapshot to be deleted.
:param timeout: Seconds to wait for volume snapshot deletion. None is
forever.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
volumesnapshot = self.get_volume_snapshot(name_or_id)
if not volumesnapshot:
return False
self._volume_client.delete(
'/snapshots/{snapshot_id}'.format(
snapshot_id=volumesnapshot['id']),
error_message="Error in deleting volume snapshot")
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the volume snapshot to be deleted."):
if not self.get_volume_snapshot(volumesnapshot['id']):
break
return True
def wait(self, session, timeout=None, ignore_error=False):
"""Wait for the node to reach the expected state.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param timeout: How much (in seconds) to wait for the introspection.
The value of ``None`` (the default) means no client-side timeout.
:param ignore_error: If ``True``, this call will raise an exception
if the introspection reaches the ``error`` state. Otherwise the
error state is considered successful and the call returns.
:return: This :class:`Introspection` instance.
:raises: :class:`~openstack.exceptions.ResourceFailure` if
introspection fails and ``ignore_error`` is ``False``.
:raises: :class:`~openstack.exceptions.ResourceTimeout` on timeout.
"""
if self._check_state(ignore_error):
return self
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for introspection on node %s" % self.id):
self.fetch(session)
if self._check_state(ignore_error):
return self
_logger.debug(
'Still waiting for introspection of node %(node)s, '
'the current state is "%(state)s"', {
'node': self.id,
'state': self.state
})
def wait(self, session, timeout=None, ignore_error=False):
"""Wait for the node to reach the expected state.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param timeout: How much (in seconds) to wait for the introspection.
The value of ``None`` (the default) means no client-side timeout.
:param ignore_error: If ``True``, this call will raise an exception
if the introspection reaches the ``error`` state. Otherwise the
error state is considered successful and the call returns.
:return: This :class:`Introspection` instance.
"""
if self._check_state(ignore_error):
return self
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for introspection on node %s" % self.id):
self.fetch(session)
if self._check_state(ignore_error):
return self
_logger.debug('Still waiting for introspection of node %(node)s, '
'the current state is "%(state)s"',
{'node': self.id, 'state': self.state})
def test_iterate_timeout_bad_wait(self):
with testtools.ExpectedException(
exc.OpenStackCloudException,
"Wait value must be an int or float value."):
for count in utils.iterate_timeout(
1, "test_iterate_timeout_bad_wait", wait="timeishard"):
pass
def wait_for_provision_state(self, session, expected_state, timeout=None,
abort_on_failed_state=True):
"""Wait for the node to reach the expected state.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param expected_state: The expected provisioning state to reach.
:param timeout: If ``wait`` is set to ``True``, specifies how much (in
seconds) to wait for the expected state to be reached. The value of
``None`` (the default) means no client-side timeout.
:param abort_on_failed_state: If ``True`` (the default), abort waiting
if the node reaches a failure state which does not match the
expected one. Note that the failure state for ``enroll`` ->
``manageable`` transition is ``enroll`` again.
:return: This :class:`Node` instance.
"""
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for node %(node)s to reach "
"target state '%(state)s'" % {'node': self.id,
'state': expected_state}):
self.fetch(session)
if self._check_state_reached(session, expected_state,
abort_on_failed_state):
return self
session.log.debug(
'Still waiting for node %(node)s to reach state '
'"%(target)s", the current state is "%(state)s"',
{'node': self.id, 'target': expected_state,
'state': self.provision_state})
def wait_for_delete(session, resource, interval, wait):
"""Wait for the resource to be deleted.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param resource: The resource to wait on to be deleted.
:type resource: :class:`~openstack.resource.Resource`
:param interval: Number of seconds to wait between checks.
:param wait: Maximum number of seconds to wait for the delete.
:return: Method returns self on success.
:raises: :class:`~openstack.exceptions.ResourceTimeout` transition
to status failed to occur in wait seconds.
"""
orig_resource = resource
for count in utils.iterate_timeout(
timeout=wait,
message="Timeout waiting for {res}:{id} to delete".format(
res=resource.__class__.__name__, id=resource.id),
wait=interval):
try:
resource = resource.get(session)
if not resource:
return orig_resource
if resource.status.lower() == 'deleted':
return resource
except exceptions.NotFoundException:
return orig_resource
def _wait_for_instance(self, as_group):
timeout = int(os.environ.get('OS_TEST_TIMEOUT'))
for count in utils.iterate_timeout(
timeout=timeout, message="Timeout waiting for instance"):
instances = list(self.conn.auto_scaling.instances(group=as_group))
if len(instances) == self.MAX_INSTANCE_NUMBER and instances[0].id:
return self.conn.auto_scaling.wait_for_instance(instances[0])
def test_iterate_timeout_timeout(self, mock_sleep):
message = "timeout test"
with testtools.ExpectedException(
exc.OpenStackCloudTimeout,
message):
for count in utils.iterate_timeout(0.1, message, wait=1):
pass
mock_sleep.assert_called_with(1.0)
def test_iterate_timeout_timeout(self, mock_sleep):
message = "timeout test"
with testtools.ExpectedException(
exc.OpenStackCloudTimeout,
message):
for count in utils.iterate_timeout(0.1, message, wait=1):
pass
mock_sleep.assert_called_with(1.0)
def _wait_for_instance(self, as_group):
timeout = int(os.environ.get('OS_TEST_TIMEOUT'))
for count in utils.iterate_timeout(
timeout=timeout, message="Timeout waiting for instance"):
instances = list(self.conn.auto_scaling.instances(group=as_group))
if ((len(instances) == self.MAX_INSTANCE_NUMBER)
and (instances[0].lifecycle_state == 'INSERVICE')):
return instances[0]
else:
continue
def _wait_for_instances(self, as_group, timeout, desire_instance_number=0):
for count in utils.iterate_timeout(
timeout=timeout, message="Timeout waiting for instance"):
instances = list(self.conn.auto_scaling.instances(group=as_group))
if (len(instances) == desire_instance_number
and [instance.id
for instance in instances if instance.id]):
for instance in instances:
self.conn.auto_scaling.wait_for_instance(instance=instance)
return
def create_volume_backup(self,
volume_id,
name=None,
description=None,
force=False,
wait=True,
timeout=None):
"""Create a volume backup.
:param volume_id: the ID of the volume to backup.
:param name: name of the backup, one will be generated if one is
not provided
:param description: description of the backup, one will be generated
if one is not provided
:param force: If set to True the backup will be created even if the
volume is attached to an instance, if False it will not
:param wait: If true, waits for volume backup to be created.
:param timeout: Seconds to wait for volume backup creation. None is
forever.
:returns: The created volume backup object.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
payload = {
'name': name,
'volume_id': volume_id,
'description': description,
'force': force,
}
data = self._volume_client.post(
'/backups',
json=dict(backup=payload),
error_message="Error creating backup of volume "
"{volume_id}".format(volume_id=volume_id))
backup = self._get_and_munchify('backup', data)
if wait:
backup_id = backup['id']
msg = ("Timeout waiting for the volume backup {} to be "
"available".format(backup_id))
for _ in utils.iterate_timeout(timeout, msg):
backup = self.get_volume_backup(backup_id)
if backup['status'] == 'available':
break
if backup['status'] == 'error':
raise exc.OpenStackCloudException(
"Error in creating volume backup {id}".format(
id=backup_id))
return backup
def create_volume_snapshot(self,
volume_id,
force=False,
wait=True,
timeout=None,
**kwargs):
"""Create a volume.
:param volume_id: the ID of the volume to snapshot.
:param force: If set to True the snapshot will be created even if the
volume is attached to an instance, if False it will not
:param name: name of the snapshot, one will be generated if one is
not provided
:param description: description of the snapshot, one will be generated
if one is not provided
:param wait: If true, waits for volume snapshot to be created.
:param timeout: Seconds to wait for volume snapshot creation. None is
forever.
:returns: The created volume object.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
kwargs = self._get_volume_kwargs(kwargs)
payload = {'volume_id': volume_id, 'force': force}
payload.update(kwargs)
resp = self.block_storage.post('/snapshots',
json=dict(snapshot=payload))
data = proxy._json_response(
resp,
error_message="Error creating snapshot of volume "
"{volume_id}".format(volume_id=volume_id))
snapshot = self._get_and_munchify('snapshot', data)
if wait:
snapshot_id = snapshot['id']
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the volume snapshot to be available."
):
snapshot = self.get_volume_snapshot_by_id(snapshot_id)
if snapshot['status'] == 'available':
break
if snapshot['status'] == 'error':
raise exc.OpenStackCloudException(
"Error in creating volume snapshot")
# TODO(mordred) need to normalize snapshots. We were normalizing them
# as volumes, which is an error. They need to be normalized as
# volume snapshots, which are completely different objects
return snapshot
def _wait_for_detach(self, volume_id):
# Volumes do not show up as unattached for a bit immediately after
# deleting a server that had had a volume attached. Yay for eventual
# consistency!
for count in utils.iterate_timeout(
60, 'Timeout waiting for volume {volume_id} to detach'.format(
volume_id=volume_id)):
volume = self.user_cloud.get_volume(volume_id)
if volume.status in ('available', 'error', 'error_restoring',
'error_extending'):
return
def wait_for_bridge_mapping(conn, node):
client = conn.network
timeout_msg = ('Timeout waiting for node %s to have bridge_mappings '
'set in the ironic-neutron-agent entry' % node)
# default agent polling period is 30s, so wait 60s
timeout = 60
for count in iterate_timeout(timeout, timeout_msg):
agents = list(client.agents(host=node, binary='ironic-neutron-agent'))
if agents:
if agents[0].configuration.get('bridge_mappings'):
return
def _delete_instance(self, instance, as_group):
timeout = int(os.environ.get('OS_TEST_TIMEOUT'))
self.conn.auto_scaling.remove_instance(instance=instance,
delete_instance=True)
for count in utils.iterate_timeout(
timeout=timeout,
message="Timeout waiting for deleting instance"):
instances = list(self.conn.auto_scaling.instances(group=as_group))
if len(instances) == 0:
return None
else:
continue
def wait_for_nodes_provision_state(self,
nodes,
expected_state,
timeout=None,
abort_on_failed_state=True):
"""Wait for the nodes to reach the expected state.
:param nodes: List of nodes - name, ID or
:class:`~openstack.baremetal.v1.node.Node` instance.
:param expected_state: The expected provisioning state to reach.
:param timeout: If ``wait`` is set to ``True``, specifies how much (in
seconds) to wait for the expected state to be reached. The value of
``None`` (the default) means no client-side timeout.
:param abort_on_failed_state: If ``True`` (the default), abort waiting
if any node reaches a failure state which does not match the
expected one. Note that the failure state for ``enroll`` ->
``manageable`` transition is ``enroll`` again.
:return: The list of :class:`~openstack.baremetal.v1.node.Node`
instances that reached the requested state.
:raises: :class:`~openstack.exceptions.ResourceFailure` if a node
reaches an error state and ``abort_on_failed_state`` is ``True``.
:raises: :class:`~openstack.exceptions.ResourceTimeout` on timeout.
"""
log_nodes = ', '.join(n.id if isinstance(n, _node.Node) else n
for n in nodes)
finished = []
remaining = nodes
for count in utils.iterate_timeout(
timeout, "Timeout waiting for nodes %(nodes)s to reach "
"target state '%(state)s'" % {
'nodes': log_nodes,
'state': expected_state
}):
nodes = [self.get_node(n) for n in remaining]
remaining = []
for n in nodes:
if n._check_state_reached(self, expected_state,
abort_on_failed_state):
finished.append(n)
else:
remaining.append(n)
if not remaining:
return finished
self.log.debug(
'Still waiting for nodes %(nodes)s to reach state '
'"%(target)s"', {
'nodes': ', '.join(n.id for n in remaining),
'target': expected_state
})
def project_cleanup(self, dry_run=True,
wait_timeout=120, status_queue=None):
"""Cleanup the project resources.
Cleanup all resources in all services, which provide cleanup methods.
:param bool dry_run: Cleanup or only list identified resources.
:param int wait_timeout: Maximum amount of time given to each service
to comlete the cleanup.
:param queue status_queue: a threading queue object used to get current
process status. The queue contain processed resources.
"""
dependencies = {}
get_dep_fn_name = '_get_cleanup_dependencies'
cleanup_fn_name = '_service_cleanup'
if not status_queue:
status_queue = queue.Queue()
for service in self.config.get_enabled_services():
if hasattr(self, service):
proxy = getattr(self, service)
if (proxy
and hasattr(proxy, get_dep_fn_name)
and hasattr(proxy, cleanup_fn_name)):
deps = getattr(proxy, get_dep_fn_name)()
if deps:
dependencies.update(deps)
dep_graph = utils.TinyDAG()
for k, v in dependencies.items():
dep_graph.add_node(k)
for dep in v['before']:
dep_graph.add_node(dep)
dep_graph.add_edge(k, dep)
for service in dep_graph.walk(timeout=wait_timeout):
fn = None
if hasattr(self, service):
proxy = getattr(self, service)
cleanup_fn = getattr(proxy, cleanup_fn_name, None)
if cleanup_fn:
fn = functools.partial(cleanup_fn, dry_run=dry_run,
status_queue=status_queue)
if fn:
self._pool_executor.submit(cleanup_task, dep_graph,
service, fn)
else:
dep_graph.node_done(service)
for count in utils.iterate_timeout(
timeout=wait_timeout,
message="Timeout waiting for cleanup to finish",
wait=1):
if dep_graph.is_complete():
return
def delete_volume(self,
name_or_id=None,
wait=True,
timeout=None,
force=False):
"""Delete a volume.
:param name_or_id: Name or unique ID of the volume.
:param wait: If true, waits for volume to be deleted.
:param timeout: Seconds to wait for volume deletion. None is forever.
:param force: Force delete volume even if the volume is in deleting
or error_deleting state.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
self.list_volumes.invalidate(self)
volume = self.get_volume(name_or_id)
if not volume:
self.log.debug("Volume %(name_or_id)s does not exist",
{'name_or_id': name_or_id},
exc_info=True)
return False
with _utils.shade_exceptions("Error in deleting volume"):
try:
if force:
proxy._json_response(
self.block_storage.post(
'volumes/{id}/action'.format(id=volume['id']),
json={'os-force_delete': None}))
else:
proxy._json_response(
self.block_storage.delete(
'volumes/{id}'.format(id=volume['id'])))
except exc.OpenStackCloudURINotFound:
self.log.debug(
"Volume {id} not found when deleting. Ignoring.".format(
id=volume['id']))
return False
self.list_volumes.invalidate(self)
if wait:
for count in utils.iterate_timeout(
timeout, "Timeout waiting for the volume to be deleted."):
if not self.get_volume(volume['id']):
break
return True
def wait_for_image(self, image, timeout=3600):
image_id = image['id']
for count in utils.iterate_timeout(
timeout, "Timeout waiting for image to snapshot"):
self.list_images.invalidate(self)
image = self.get_image(image_id)
if not image:
continue
if image['status'] == 'active':
return image
elif image['status'] == 'error':
raise exc.OpenStackCloudException(
'Image {image} hit error state'.format(image=image_id))
def _wait_for_detach(self, volume_id):
# Volumes do not show up as unattached for a bit immediately after
# deleting a server that had had a volume attached. Yay for eventual
# consistency!
for count in utils.iterate_timeout(
60,
'Timeout waiting for volume {volume_id} to detach'.format(
volume_id=volume_id)):
volume = self.user_cloud.get_volume(volume_id)
if volume.status in (
'available', 'error',
'error_restoring', 'error_extending'):
return
def wait_for_image(self, image, timeout=3600):
image_id = image['id']
for count in utils.iterate_timeout(
timeout, "Timeout waiting for image to snapshot"):
self.list_images.invalidate(self)
image = self.get_image(image_id)
if not image:
continue
if image['status'] == 'active':
return image
elif image['status'] == 'error':
raise exc.OpenStackCloudException(
'Image {image} hit error state'.format(image=image_id))
def wait_for_status(session, resource, status, failures, interval, wait):
"""Wait for the resource to be in a particular status.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param resource: The resource to wait on to reach the status. The resource
must have a status attribute.
:type resource: :class:`~openstack.resource.Resource`
:param status: Desired status of the resource.
:param list failures: Statuses that would indicate the transition
failed such as 'ERROR'. Defaults to ['ERROR'].
:param interval: Number of seconds to wait between checks.
:param wait: Maximum number of seconds to wait for transition.
:return: Method returns self on success.
:raises: :class:`~openstack.exceptions.ResourceTimeout` transition
to status failed to occur in wait seconds.
:raises: :class:`~openstack.exceptions.ResourceFailure` resource
transitioned to one of the failure states.
:raises: :class:`~AttributeError` if the resource does not have a status
attribute
"""
if resource.status == status:
return resource
if failures is None:
failures = ['ERROR']
failures = [f.lower() for f in failures]
name = "{res}:{id}".format(res=resource.__class__.__name__, id=resource.id)
msg = "Timeout waiting for {name} to transition to {status}".format(
name=name, status=status)
for count in utils.iterate_timeout(
timeout=wait,
message=msg,
wait=interval):
resource = resource.get(session)
new_status = resource.status
if not resource:
raise exceptions.ResourceFailure(
"{name} went away while waiting for {status}".format(
name=name, status=status))
if new_status.lower() == status.lower():
return resource
if resource.status.lower() in failures:
raise exceptions.ResourceFailure(
"{name} transitioned to failure state {status}".format(
name=name, status=resource.status))
def wait_for_status(session, resource, status, failures, interval, wait):
"""Wait for the resource to be in a particular status.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param resource: The resource to wait on to reach the status. The resource
must have a status attribute.
:type resource: :class:`~openstack.resource.Resource`
:param status: Desired status of the resource.
:param list failures: Statuses that would indicate the transition
failed such as 'ERROR'. Defaults to ['ERROR'].
:param interval: Number of seconds to wait between checks.
:param wait: Maximum number of seconds to wait for transition.
:return: Method returns self on success.
:raises: :class:`~openstack.exceptions.ResourceTimeout` transition
to status failed to occur in wait seconds.
:raises: :class:`~openstack.exceptions.ResourceFailure` resource
transitioned to one of the failure states.
:raises: :class:`~AttributeError` if the resource does not have a status
attribute
"""
if resource.status == status:
return resource
if failures is None:
failures = ['ERROR']
failures = [f.lower() for f in failures]
name = "{res}:{id}".format(res=resource.__class__.__name__, id=resource.id)
msg = "Timeout waiting for {name} to transition to {status}".format(
name=name, status=status)
for count in utils.iterate_timeout(
timeout=wait,
message=msg,
wait=interval):
resource = resource.get(session)
new_status = resource.status
if not resource:
raise exceptions.ResourceFailure(
"{name} went away while waiting for {status}".format(
name=name, status=status))
if new_status.lower() == status.lower():
return resource
if resource.status.lower() in failures:
raise exceptions.ResourceFailure(
"{name} transitioned to failure state {status}".format(
name=name, status=resource.status))
def create_volume_backup(self, volume_id, name=None, description=None,
force=False, wait=True, timeout=None):
"""Create a volume backup.
:param volume_id: the ID of the volume to backup.
:param name: name of the backup, one will be generated if one is
not provided
:param description: description of the backup, one will be generated
if one is not provided
:param force: If set to True the backup will be created even if the
volume is attached to an instance, if False it will not
:param wait: If true, waits for volume backup to be created.
:param timeout: Seconds to wait for volume backup creation. None is
forever.
:returns: The created volume backup object.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
payload = {
'name': name,
'volume_id': volume_id,
'description': description,
'force': force,
}
data = self._volume_client.post(
'/backups', json=dict(backup=payload),
error_message="Error creating backup of volume "
"{volume_id}".format(volume_id=volume_id))
backup = self._get_and_munchify('backup', data)
if wait:
backup_id = backup['id']
msg = ("Timeout waiting for the volume backup {} to be "
"available".format(backup_id))
for _ in utils.iterate_timeout(timeout, msg):
backup = self.get_volume_backup(backup_id)
if backup['status'] == 'available':
break
if backup['status'] == 'error':
raise exc.OpenStackCloudException(
"Error in creating volume backup {id}".format(
id=backup_id))
return backup
def delete_volume(self, name_or_id=None, wait=True, timeout=None,
force=False):
"""Delete a volume.
:param name_or_id: Name or unique ID of the volume.
:param wait: If true, waits for volume to be deleted.
:param timeout: Seconds to wait for volume deletion. None is forever.
:param force: Force delete volume even if the volume is in deleting
or error_deleting state.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
self.list_volumes.invalidate(self)
volume = self.get_volume(name_or_id)
if not volume:
self.log.debug(
"Volume %(name_or_id)s does not exist",
{'name_or_id': name_or_id},
exc_info=True)
return False
with _utils.shade_exceptions("Error in deleting volume"):
try:
if force:
self._volume_client.post(
'volumes/{id}/action'.format(id=volume['id']),
json={'os-force_delete': None})
else:
self._volume_client.delete(
'volumes/{id}'.format(id=volume['id']))
except exc.OpenStackCloudURINotFound:
self.log.debug(
"Volume {id} not found when deleting. Ignoring.".format(
id=volume['id']))
return False
self.list_volumes.invalidate(self)
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the volume to be deleted."):
if not self.get_volume(volume['id']):
break
return True
def create_volume_snapshot(self, volume_id, force=False,
wait=True, timeout=None, **kwargs):
"""Create a volume.
:param volume_id: the ID of the volume to snapshot.
:param force: If set to True the snapshot will be created even if the
volume is attached to an instance, if False it will not
:param name: name of the snapshot, one will be generated if one is
not provided
:param description: description of the snapshot, one will be generated
if one is not provided
:param wait: If true, waits for volume snapshot to be created.
:param timeout: Seconds to wait for volume snapshot creation. None is
forever.
:returns: The created volume object.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
kwargs = self._get_volume_kwargs(kwargs)
payload = {'volume_id': volume_id, 'force': force}
payload.update(kwargs)
data = self._volume_client.post(
'/snapshots',
json=dict(snapshot=payload),
error_message="Error creating snapshot of volume "
"{volume_id}".format(volume_id=volume_id))
snapshot = self._get_and_munchify('snapshot', data)
if wait:
snapshot_id = snapshot['id']
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the volume snapshot to be available."
):
snapshot = self.get_volume_snapshot_by_id(snapshot_id)
if snapshot['status'] == 'available':
break
if snapshot['status'] == 'error':
raise exc.OpenStackCloudException(
"Error in creating volume snapshot")
# TODO(mordred) need to normalize snapshots. We were normalizing them
# as volumes, which is an error. They need to be normalized as
# volume snapshots, which are completely different objects
return snapshot
def wait_for_nodes_provision_state(self, nodes, expected_state,
timeout=None,
abort_on_failed_state=True):
"""Wait for the nodes to reach the expected state.
:param nodes: List of nodes - name, ID or
:class:`~openstack.baremetal.v1.node.Node` instance.
:param expected_state: The expected provisioning state to reach.
:param timeout: If ``wait`` is set to ``True``, specifies how much (in
seconds) to wait for the expected state to be reached. The value of
``None`` (the default) means no client-side timeout.
:param abort_on_failed_state: If ``True`` (the default), abort waiting
if any node reaches a failure state which does not match the
expected one. Note that the failure state for ``enroll`` ->
``manageable`` transition is ``enroll`` again.
:return: The list of :class:`~openstack.baremetal.v1.node.Node`
instances that reached the requested state.
"""
log_nodes = ', '.join(n.id if isinstance(n, _node.Node) else n
for n in nodes)
finished = []
remaining = nodes
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for nodes %(nodes)s to reach "
"target state '%(state)s'" % {'nodes': log_nodes,
'state': expected_state}):
nodes = [self.get_node(n) for n in remaining]
remaining = []
for n in nodes:
if n._check_state_reached(self, expected_state,
abort_on_failed_state):
finished.append(n)
else:
remaining.append(n)
if not remaining:
return finished
self.log.debug(
'Still waiting for nodes %(nodes)s to reach state '
'"%(target)s"',
{'nodes': ', '.join(n.id for n in remaining),
'target': expected_state})
def find_best_address(addresses, public=False, cloud_public=True):
do_check = public == cloud_public
if not addresses:
return None
if len(addresses) == 1:
return addresses[0]
if len(addresses) > 1 and do_check:
# We only want to do this check if the address is supposed to be
# reachable. Otherwise we're just debug log spamming on every listing
# of private ip addresses
for address in addresses:
try:
for count in utils.iterate_timeout(5,
"Timeout waiting for %s" %
address,
wait=0.1):
# Return the first one that is reachable
try:
for res in socket.getaddrinfo(address, 22,
socket.AF_UNSPEC,
socket.SOCK_STREAM, 0):
family, socktype, proto, _, sa = res
connect_socket = socket.socket(
family, socktype, proto)
connect_socket.settimeout(1)
connect_socket.connect(sa)
return address
except socket.error:
# Sometimes a "no route to address" type error
# will fail fast, but can often come alive
# when retried.
continue
except Exception:
pass
# Give up and return the first - none work as far as we can tell
if do_check:
log = _log.setup_logging('openstack')
log.debug(
"The cloud returned multiple addresses %s:, and we could not "
"connect to port 22 on either. That might be what you wanted, "
"but we have no clue what's going on, so we picked the first one "
"%s" % (addresses, addresses[0]))
return addresses[0]
def test_add_auto_ip(self):
self._setup_networks()
new_server = self.user_cloud.create_server(
wait=True, name=self.new_item_name + '_server',
image=self.image,
flavor=self.flavor, nics=[self.nic])
# ToDo: remove the following iteration when create_server waits for
# the IP to be attached
ip = None
for _ in utils.iterate_timeout(
self.timeout, "Timeout waiting for IP address to be attached"):
ip = meta.get_server_external_ipv4(self.user_cloud, new_server)
if ip is not None:
break
new_server = self.user_cloud.get_server(new_server.id)
self.addCleanup(self._cleanup_ips, new_server)
def cleanup(self, volume, snapshot_name=None, image_name=None):
# Need to delete snapshots before volumes
if snapshot_name:
snapshot = self.user_cloud.get_volume_snapshot(snapshot_name)
if snapshot:
self.user_cloud.delete_volume_snapshot(snapshot_name,
wait=True)
if image_name:
image = self.user_cloud.get_image(image_name)
if image:
self.user_cloud.delete_image(image_name, wait=True)
if not isinstance(volume, list):
self.user_cloud.delete_volume(volume, wait=True)
else:
# We have more than one volume to clean up - submit all of the
# deletes without wait, then poll until none of them are found
# in the volume list anymore
for v in volume:
self.user_cloud.delete_volume(v, wait=False)
try:
for count in utils.iterate_timeout(
180, "Timeout waiting for volume cleanup"):
found = False
for existing in self.user_cloud.list_volumes():
for v in volume:
if v['id'] == existing['id']:
found = True
break
if found:
break
if not found:
break
except (exc.OpenStackCloudTimeout, TimeoutException):
# NOTE(slaweq): ups, some volumes are still not removed
# so we should try to force delete it once again and move
# forward
for existing in self.user_cloud.list_volumes():
for v in volume:
if v['id'] == existing['id']:
self.operator_cloud.delete_volume(v,
wait=False,
force=True)
def find_best_address(addresses, public=False, cloud_public=True):
do_check = public == cloud_public
if not addresses:
return None
if len(addresses) == 1:
return addresses[0]
if len(addresses) > 1 and do_check:
# We only want to do this check if the address is supposed to be
# reachable. Otherwise we're just debug log spamming on every listing
# of private ip addresses
for address in addresses:
try:
for count in utils.iterate_timeout(
5, "Timeout waiting for %s" % address, wait=0.1):
# Return the first one that is reachable
try:
for res in socket.getaddrinfo(
address, 22, socket.AF_UNSPEC,
socket.SOCK_STREAM, 0):
family, socktype, proto, _, sa = res
connect_socket = socket.socket(
family, socktype, proto)
connect_socket.settimeout(1)
connect_socket.connect(sa)
return address
except socket.error:
# Sometimes a "no route to address" type error
# will fail fast, but can often come alive
# when retried.
continue
except Exception:
pass
# Give up and return the first - none work as far as we can tell
if do_check:
log = _log.setup_logging('openstack')
log.debug(
"The cloud returned multiple addresses %s:, and we could not "
"connect to port 22 on either. That might be what you wanted, "
"but we have no clue what's going on, so we picked the first one "
"%s" % (addresses, addresses[0]))
return addresses[0]
def cleanup(self, volume, snapshot_name=None, image_name=None):
# Need to delete snapshots before volumes
if snapshot_name:
snapshot = self.user_cloud.get_volume_snapshot(snapshot_name)
if snapshot:
self.user_cloud.delete_volume_snapshot(
snapshot_name, wait=True)
if image_name:
image = self.user_cloud.get_image(image_name)
if image:
self.user_cloud.delete_image(image_name, wait=True)
if not isinstance(volume, list):
self.user_cloud.delete_volume(volume, wait=True)
else:
# We have more than one volume to clean up - submit all of the
# deletes without wait, then poll until none of them are found
# in the volume list anymore
for v in volume:
self.user_cloud.delete_volume(v, wait=False)
try:
for count in utils.iterate_timeout(
180, "Timeout waiting for volume cleanup"):
found = False
for existing in self.user_cloud.list_volumes():
for v in volume:
if v['id'] == existing['id']:
found = True
break
if found:
break
if not found:
break
except (exc.OpenStackCloudTimeout, TimeoutException):
# NOTE(slaweq): ups, some volumes are still not removed
# so we should try to force delete it once again and move
# forward
for existing in self.user_cloud.list_volumes():
for v in volume:
if v['id'] == existing['id']:
self.operator_cloud.delete_volume(
v, wait=False, force=True)
def wait_for_reservation(self, session, timeout=None):
"""Wait for a lock on the node to be released.
Bare metal nodes in ironic have a reservation lock that
is used to represent that a conductor has locked the node
while performing some sort of action, such as changing
configuration as a result of a machine state change.
This lock can occur during power syncronization, and prevents
updates to objects attached to the node, such as ports.
Note that nothing prevents a conductor from acquiring the lock again
after this call returns, so it should be treated as best effort.
Returns immediately if there is no reservation on the node.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param timeout: How much (in seconds) to wait for the lock to be
released. The value of ``None`` (the default) means no timeout.
:return: This :class:`Node` instance.
"""
if self.reservation is None:
return self
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the lock to be released on node %s" %
self.id):
self.fetch(session)
if self.reservation is None:
return self
_logger.debug(
'Still waiting for the lock to be released on node '
'%(node)s, currently locked by conductor %(host)s', {
'node': self.id,
'host': self.reservation
})
def delete_image(self,
name_or_id,
wait=False,
timeout=3600,
delete_objects=True):
"""Delete an existing image.
:param name_or_id: Name of the image to be deleted.
:param wait: If True, waits for image to be deleted.
:param timeout: Seconds to wait for image deletion. None is forever.
:param delete_objects: If True, also deletes uploaded swift objects.
:returns: True if delete succeeded, False otherwise.
:raises: OpenStackCloudException if there are problems deleting.
"""
image = self.get_image(name_or_id)
if not image:
return False
self.image.delete_image(image)
self.list_images.invalidate(self)
# Task API means an image was uploaded to swift
# TODO(gtema) does it make sense to move this into proxy?
if self.image_api_use_tasks and (self.image._IMAGE_OBJECT_KEY in image
or self.image._SHADE_IMAGE_OBJECT_KEY
in image):
(container, objname) = image.get(
self.image._IMAGE_OBJECT_KEY,
image.get(self.image._SHADE_IMAGE_OBJECT_KEY)).split('/', 1)
self.delete_object(container=container, name=objname)
if wait:
for count in utils.iterate_timeout(
timeout, "Timeout waiting for the image to be deleted."):
self._get_cache(None).invalidate()
if self.get_image(image.id) is None:
break
return True
def delete_volume_backup(self,
name_or_id=None,
force=False,
wait=False,
timeout=None):
"""Delete a volume backup.
:param name_or_id: Name or unique ID of the volume backup.
:param force: Allow delete in state other than error or available.
:param wait: If true, waits for volume backup to be deleted.
:param timeout: Seconds to wait for volume backup deletion. None is
forever.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
volume_backup = self.get_volume_backup(name_or_id)
if not volume_backup:
return False
msg = "Error in deleting volume backup"
if force:
self._volume_client.post('/backups/{backup_id}/action'.format(
backup_id=volume_backup['id']),
json={'os-force_delete': None},
error_message=msg)
else:
self._volume_client.delete(
'/backups/{backup_id}'.format(backup_id=volume_backup['id']),
error_message=msg)
if wait:
msg = "Timeout waiting for the volume backup to be deleted."
for count in utils.iterate_timeout(timeout, msg):
if not self.get_volume_backup(volume_backup['id']):
break
return True
def delete_volume_backup(self, name_or_id=None, force=False, wait=False,
timeout=None):
"""Delete a volume backup.
:param name_or_id: Name or unique ID of the volume backup.
:param force: Allow delete in state other than error or available.
:param wait: If true, waits for volume backup to be deleted.
:param timeout: Seconds to wait for volume backup deletion. None is
forever.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
volume_backup = self.get_volume_backup(name_or_id)
if not volume_backup:
return False
msg = "Error in deleting volume backup"
if force:
self._volume_client.post(
'/backups/{backup_id}/action'.format(
backup_id=volume_backup['id']),
json={'os-force_delete': None},
error_message=msg)
else:
self._volume_client.delete(
'/backups/{backup_id}'.format(
backup_id=volume_backup['id']),
error_message=msg)
if wait:
msg = "Timeout waiting for the volume backup to be deleted."
for count in utils.iterate_timeout(timeout, msg):
if not self.get_volume_backup(volume_backup['id']):
break
return True
def detach_volume(self, server, volume, wait=True, timeout=None):
"""Detach a volume from a server.
:param server: The server dict to detach from.
:param volume: The volume dict to detach.
:param wait: If true, waits for volume to be detached.
:param timeout: Seconds to wait for volume detachment. None is forever.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
proxy._json_response(self.compute.delete(
'/servers/{server_id}/os-volume_attachments/{volume_id}'.format(
server_id=server['id'], volume_id=volume['id'])),
error_message=(
"Error detaching volume {volume} from server {server}".format(
volume=volume['id'], server=server['id'])))
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for volume %s to detach." % volume['id']):
try:
vol = self.get_volume(volume['id'])
except Exception:
self.log.debug(
"Error getting volume info %s", volume['id'],
exc_info=True)
continue
if vol['status'] == 'available':
return
if vol['status'] == 'error':
raise exc.OpenStackCloudException(
"Error in detaching volume %s" % volume['id']
)
def wait_for_reservation(self, session, timeout=None):
"""Wait for a lock on the node to be released.
Bare metal nodes in ironic have a reservation lock that
is used to represent that a conductor has locked the node
while performing some sort of action, such as changing
configuration as a result of a machine state change.
This lock can occur during power syncronization, and prevents
updates to objects attached to the node, such as ports.
Note that nothing prevents a conductor from acquiring the lock again
after this call returns, so it should be treated as best effort.
Returns immediately if there is no reservation on the node.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param timeout: How much (in seconds) to wait for the lock to be
released. The value of ``None`` (the default) means no timeout.
:return: This :class:`Node` instance.
"""
if self.reservation is None:
return self
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the lock to be released on node %s" %
self.id):
self.fetch(session)
if self.reservation is None:
return self
session.log.debug(
'Still waiting for the lock to be released on node '
'%(node)s, currently locked by conductor %(host)s',
{'node': self.id, 'host': self.reservation})
def delete_image(
self, name_or_id, wait=False, timeout=3600, delete_objects=True):
"""Delete an existing image.
:param name_or_id: Name of the image to be deleted.
:param wait: If True, waits for image to be deleted.
:param timeout: Seconds to wait for image deletion. None is forever.
:param delete_objects: If True, also deletes uploaded swift objects.
:returns: True if delete succeeded, False otherwise.
:raises: OpenStackCloudException if there are problems deleting.
"""
image = self.get_image(name_or_id)
if not image:
return False
self.image.delete_image(image)
self.list_images.invalidate(self)
# Task API means an image was uploaded to swift
# TODO(gtema) does it make sense to move this into proxy?
if self.image_api_use_tasks and (
self._IMAGE_OBJECT_KEY in image
or self._SHADE_IMAGE_OBJECT_KEY in image):
(container, objname) = image.get(
self._IMAGE_OBJECT_KEY, image.get(
self._SHADE_IMAGE_OBJECT_KEY)).split('/', 1)
self.delete_object(container=container, name=objname)
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the image to be deleted."):
self._get_cache(None).invalidate()
if self.get_image(image.id) is None:
break
return True
def create_image(
self, name, filename=None,
container=None,
md5=None, sha256=None,
disk_format=None, container_format=None,
disable_vendor_agent=True,
wait=False, timeout=3600,
allow_duplicates=False, meta=None, volume=None, **kwargs):
"""Upload an image.
:param str name: Name of the image to create. If it is a pathname
of an image, the name will be constructed from the
extensionless basename of the path.
:param str filename: The path to the file to upload, if needed.
(optional, defaults to None)
:param str container: Name of the container in swift where images
should be uploaded for import if the cloud
requires such a thing. (optiona, defaults to
'images')
:param str md5: md5 sum of the image file. If not given, an md5 will
be calculated.
:param str sha256: sha256 sum of the image file. If not given, an md5
will be calculated.
:param str disk_format: The disk format the image is in. (optional,
defaults to the os-client-config config value
for this cloud)
:param str container_format: The container format the image is in.
(optional, defaults to the
os-client-config config value for this
cloud)
:param bool disable_vendor_agent: Whether or not to append metadata
flags to the image to inform the
cloud in question to not expect a
vendor agent to be runing.
(optional, defaults to True)
:param bool wait: If true, waits for image to be created. Defaults to
true - however, be aware that one of the upload
methods is always synchronous.
:param timeout: Seconds to wait for image creation. None is forever.
:param allow_duplicates: If true, skips checks that enforce unique
image name. (optional, defaults to False)
:param meta: A dict of key/value pairs to use for metadata that
bypasses automatic type conversion.
:param volume: Name or ID or volume object of a volume to create an
image from. Mutually exclusive with (optional, defaults
to None)
Additional kwargs will be passed to the image creation as additional
metadata for the image and will have all values converted to string
except for min_disk, min_ram, size and virtual_size which will be
converted to int.
If you are sure you have all of your data types correct or have an
advanced need to be explicit, use meta. If you are just a normal
consumer, using kwargs is likely the right choice.
If a value is in meta and kwargs, meta wins.
:returns: A ``munch.Munch`` of the Image object
:raises: OpenStackCloudException if there are problems uploading
"""
if volume:
image = self.block_storage.create_image(
name=name, volume=volume,
allow_duplicates=allow_duplicates,
container_format=container_format, disk_format=disk_format,
wait=wait, timeout=timeout)
else:
image = self.image.create_image(
name, filename=filename,
container=container,
md5=sha256, sha256=sha256,
disk_format=disk_format, container_format=container_format,
disable_vendor_agent=disable_vendor_agent,
wait=wait, timeout=timeout,
allow_duplicates=allow_duplicates, meta=meta, **kwargs)
self._get_cache(None).invalidate()
if not wait:
return image
try:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the image to finish."):
image_obj = self.get_image(image.id)
if image_obj and image_obj.status not in ('queued', 'saving'):
return image_obj
except exc.OpenStackCloudTimeout:
self.log.debug(
"Timeout waiting for image to become ready. Deleting.")
self.delete_image(image.id, wait=True)
raise
def attach_volume(self, server, volume, device=None,
wait=True, timeout=None):
"""Attach a volume to a server.
This will attach a volume, described by the passed in volume
dict (as returned by get_volume()), to the server described by
the passed in server dict (as returned by get_server()) on the
named device on the server.
If the volume is already attached to the server, or generally not
available, then an exception is raised. To re-attach to a server,
but under a different device, the user must detach it first.
:param server: The server dict to attach to.
:param volume: The volume dict to attach.
:param device: The device name where the volume will attach.
:param wait: If true, waits for volume to be attached.
:param timeout: Seconds to wait for volume attachment. None is forever.
:returns: a volume attachment object.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
dev = self.get_volume_attach_device(volume, server['id'])
if dev:
raise exc.OpenStackCloudException(
"Volume %s already attached to server %s on device %s"
% (volume['id'], server['id'], dev)
)
if volume['status'] != 'available':
raise exc.OpenStackCloudException(
"Volume %s is not available. Status is '%s'"
% (volume['id'], volume['status'])
)
payload = {'volumeId': volume['id']}
if device:
payload['device'] = device
data = proxy._json_response(
self.compute.post(
'/servers/{server_id}/os-volume_attachments'.format(
server_id=server['id']),
json=dict(volumeAttachment=payload)),
error_message="Error attaching volume {volume_id} to server "
"{server_id}".format(volume_id=volume['id'],
server_id=server['id']))
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for volume %s to attach." % volume['id']):
try:
self.list_volumes.invalidate(self)
vol = self.get_volume(volume['id'])
except Exception:
self.log.debug(
"Error getting volume info %s", volume['id'],
exc_info=True)
continue
if self.get_volume_attach_device(vol, server['id']):
break
# TODO(Shrews) check to see if a volume can be in error status
# and also attached. If so, we should move this
# above the get_volume_attach_device call
if vol['status'] == 'error':
raise exc.OpenStackCloudException(
"Error in attaching volume %s" % volume['id']
)
return self._normalize_volume_attachment(
self._get_and_munchify('volumeAttachment', data))
def wait_for_task(self, task, status='success', failures=None,
interval=2, wait=120):
"""Wait for a task to be in a particular status.
:param task: The resource to wait on to reach the specified status.
The resource must have a ``status`` attribute.
:type resource: A :class:`~openstack.resource.Resource` object.
:param status: Desired status.
:param failures: Statuses that would be interpreted as failures.
:type failures: :py:class:`list`
:param interval: Number of seconds to wait before to consecutive
checks. Default to 2.
:param wait: Maximum number of seconds to wait before the change.
Default to 120.
:returns: The resource is returned on success.
:raises: :class:`~openstack.exceptions.ResourceTimeout` if transition
to the desired status failed to occur in specified seconds.
:raises: :class:`~openstack.exceptions.ResourceFailure` if the resource
has transited to one of the failure statuses.
:raises: :class:`~AttributeError` if the resource does not have a
``status`` attribute.
"""
if failures is None:
failures = ['failure']
else:
failures = [f.lower() for f in failures]
if task.status.lower() == status.lower():
return task
name = "{res}:{id}".format(res=task.__class__.__name__, id=task.id)
msg = "Timeout waiting for {name} to transition to {status}".format(
name=name, status=status)
for count in utils.iterate_timeout(
timeout=wait,
message=msg,
wait=interval):
task = task.fetch(self)
if not task:
raise exceptions.ResourceFailure(
"{name} went away while waiting for {status}".format(
name=name, status=status))
new_status = task.status
normalized_status = new_status.lower()
if normalized_status == status.lower():
return task
elif normalized_status in failures:
if task.message == _IMAGE_ERROR_396:
task_args = dict(input=task.input, type=task.type)
task = self.create_task(**task_args)
self.log.debug('Got error 396. Recreating task %s' % task)
else:
raise exceptions.ResourceFailure(
"{name} transitioned to failure state {status}".format(
name=name, status=new_status))
self.log.debug('Still waiting for resource %s to reach state %s, '
'current state is %s', name, status, new_status)
def test_iterate_timeout_int_wait(self, mock_sleep):
iter = utils.iterate_timeout(
10, "test_iterate_timeout_int_wait", wait=1)
next(iter)
next(iter)
mock_sleep.assert_called_with(1.0)
def unregister_machine(self, nics, uuid, wait=False, timeout=600):
"""Unregister Baremetal from Ironic
Removes entries for Network Interfaces and baremetal nodes
from an Ironic API
:param nics: An array of strings that consist of MAC addresses
to be removed.
:param string uuid: The UUID of the node to be deleted.
:param wait: Boolean value, defaults to false, if to block the method
upon the final step of unregistering the machine.
:param timeout: Integer value, representing seconds with a default
value of 600, which controls the maximum amount of
time to block the method's completion on.
:raises: OpenStackCloudException on operation failure.
"""
machine = self.get_machine(uuid)
invalid_states = ['active', 'cleaning', 'clean wait', 'clean failed']
if machine['provision_state'] in invalid_states:
raise exc.OpenStackCloudException(
"Error unregistering node '%s' due to current provision "
"state '%s'" % (uuid, machine['provision_state']))
# NOTE(TheJulia) There is a high possibility of a lock being present
# if the machine was just moved through the state machine. This was
# previously concealed by exception retry logic that detected the
# failure, and resubitted the request in python-ironicclient.
try:
self.wait_for_baremetal_node_lock(machine, timeout=timeout)
except exc.OpenStackCloudException as e:
raise exc.OpenStackCloudException(
"Error unregistering node '%s': Exception occured while"
" waiting to be able to proceed: %s" % (machine['uuid'], e))
for nic in nics:
port_msg = ("Error removing NIC {nic} from baremetal API for "
"node {uuid}").format(nic=nic, uuid=uuid)
port_url = '/ports/detail?address={mac}'.format(mac=nic['mac'])
port = self._baremetal_client.get(port_url, microversion=1.6,
error_message=port_msg)
port_url = '/ports/{uuid}'.format(uuid=port['ports'][0]['uuid'])
_utils._call_client_and_retry(self._baremetal_client.delete,
port_url, retry_on=[409, 503],
error_message=port_msg)
with _utils.shade_exceptions(
"Error unregistering machine {node_id} from the baremetal "
"API".format(node_id=uuid)):
# NOTE(TheJulia): While this should not matter microversion wise,
# ironic assumes all calls without an explicit microversion to be
# version 1.0. Ironic expects to deprecate support for older
# microversions in future releases, as such, we explicitly set
# the version to what we have been using with the client library..
version = "1.6"
msg = "Baremetal machine failed to be deleted"
url = '/nodes/{node_id}'.format(
node_id=uuid)
_utils._call_client_and_retry(self._baremetal_client.delete,
url, retry_on=[409, 503],
error_message=msg,
microversion=version)
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for machine to be deleted"):
if not self.get_machine(uuid):
break
def register_machine(self, nics, wait=False, timeout=3600,
lock_timeout=600, **kwargs):
"""Register Baremetal with Ironic
Allows for the registration of Baremetal nodes with Ironic
and population of pertinant node information or configuration
to be passed to the Ironic API for the node.
This method also creates ports for a list of MAC addresses passed
in to be utilized for boot and potentially network configuration.
If a failure is detected creating the network ports, any ports
created are deleted, and the node is removed from Ironic.
:param nics:
An array of MAC addresses that represent the
network interfaces for the node to be created.
Example::
[
{'mac': 'aa:bb:cc:dd:ee:01'},
{'mac': 'aa:bb:cc:dd:ee:02'}
]
:param wait: Boolean value, defaulting to false, to wait for the
node to reach the available state where the node can be
provisioned. It must be noted, when set to false, the
method will still wait for locks to clear before sending
the next required command.
:param timeout: Integer value, defautling to 3600 seconds, for the
wait state to reach completion.
:param lock_timeout: Integer value, defaulting to 600 seconds, for
locks to clear.
:param kwargs: Key value pairs to be passed to the Ironic API,
including uuid, name, chassis_uuid, driver_info,
parameters.
:raises: OpenStackCloudException on operation error.
:returns: Returns a ``munch.Munch`` representing the new
baremetal node.
"""
msg = ("Baremetal machine node failed to be created.")
port_msg = ("Baremetal machine port failed to be created.")
url = '/nodes'
# TODO(TheJulia): At some point we need to figure out how to
# handle data across when the requestor is defining newer items
# with the older api.
machine = self._baremetal_client.post(url,
json=kwargs,
error_message=msg,
microversion="1.6")
created_nics = []
try:
for row in nics:
payload = {'address': row['mac'],
'node_uuid': machine['uuid']}
nic = self._baremetal_client.post('/ports',
json=payload,
error_message=port_msg)
created_nics.append(nic['uuid'])
except Exception as e:
self.log.debug("ironic NIC registration failed", exc_info=True)
# TODO(mordred) Handle failures here
try:
for uuid in created_nics:
try:
port_url = '/ports/{uuid}'.format(uuid=uuid)
# NOTE(TheJulia): Added in hope that it is logged.
port_msg = ('Failed to delete port {port} for node '
'{node}').format(port=uuid,
node=machine['uuid'])
self._baremetal_client.delete(port_url,
error_message=port_msg)
except Exception:
pass
finally:
version = "1.6"
msg = "Baremetal machine failed to be deleted."
url = '/nodes/{node_id}'.format(
node_id=machine['uuid'])
self._baremetal_client.delete(url,
error_message=msg,
microversion=version)
raise exc.OpenStackCloudException(
"Error registering NICs with the baremetal service: %s"
% str(e))
with _utils.shade_exceptions(
"Error transitioning node to available state"):
if wait:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for node transition to "
"available state"):
machine = self.get_machine(machine['uuid'])
# Note(TheJulia): Per the Ironic state code, a node
# that fails returns to enroll state, which means a failed
# node cannot be determined at this point in time.
if machine['provision_state'] in ['enroll']:
self.node_set_provision_state(
machine['uuid'], 'manage')
elif machine['provision_state'] in ['manageable']:
self.node_set_provision_state(
machine['uuid'], 'provide')
elif machine['last_error'] is not None:
raise exc.OpenStackCloudException(
"Machine encountered a failure: %s"
% machine['last_error'])
# Note(TheJulia): Earlier versions of Ironic default to
# None and later versions default to available up until
# the introduction of enroll state.
# Note(TheJulia): The node will transition through
# cleaning if it is enabled, and we will wait for
# completion.
elif machine['provision_state'] in ['available', None]:
break
else:
if machine['provision_state'] in ['enroll']:
self.node_set_provision_state(machine['uuid'], 'manage')
# Note(TheJulia): We need to wait for the lock to clear
# before we attempt to set the machine into provide state
# which allows for the transition to available.
for count in utils.iterate_timeout(
lock_timeout,
"Timeout waiting for reservation to clear "
"before setting provide state"):
machine = self.get_machine(machine['uuid'])
if (machine['reservation'] is None
and machine['provision_state'] != 'enroll'):
# NOTE(TheJulia): In this case, the node has
# has moved on from the previous state and is
# likely not being verified, as no lock is
# present on the node.
self.node_set_provision_state(
machine['uuid'], 'provide')
machine = self.get_machine(machine['uuid'])
break
elif machine['provision_state'] in [
'cleaning',
'available']:
break
elif machine['last_error'] is not None:
raise exc.OpenStackCloudException(
"Machine encountered a failure: %s"
% machine['last_error'])
if not isinstance(machine, str):
return self._normalize_machine(machine)
else:
return machine
def create_volume(
self, size,
wait=True, timeout=None, image=None, bootable=None, **kwargs):
"""Create a volume.
:param size: Size, in GB of the volume to create.
:param name: (optional) Name for the volume.
:param description: (optional) Name for the volume.
:param wait: If true, waits for volume to be created.
:param timeout: Seconds to wait for volume creation. None is forever.
:param image: (optional) Image name, ID or object from which to create
the volume
:param bootable: (optional) Make this volume bootable. If set, wait
will also be set to true.
:param kwargs: Keyword arguments as expected for cinder client.
:returns: The created volume object.
:raises: OpenStackCloudTimeout if wait time exceeded.
:raises: OpenStackCloudException on operation error.
"""
if bootable is not None:
wait = True
if image:
image_obj = self.get_image(image)
if not image_obj:
raise exc.OpenStackCloudException(
"Image {image} was requested as the basis for a new"
" volume, but was not found on the cloud".format(
image=image))
kwargs['imageRef'] = image_obj['id']
kwargs = self._get_volume_kwargs(kwargs)
kwargs['size'] = size
payload = dict(volume=kwargs)
if 'scheduler_hints' in kwargs:
payload['OS-SCH-HNT:scheduler_hints'] = kwargs.pop(
'scheduler_hints', None)
data = self._volume_client.post(
'/volumes',
json=dict(payload),
error_message='Error in creating volume')
volume = self._get_and_munchify('volume', data)
self.list_volumes.invalidate(self)
if volume['status'] == 'error':
raise exc.OpenStackCloudException("Error in creating volume")
if wait:
vol_id = volume['id']
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the volume to be available."):
volume = self.get_volume(vol_id)
if not volume:
continue
if volume['status'] == 'available':
if bootable is not None:
self.set_volume_bootable(volume, bootable=bootable)
# no need to re-fetch to update the flag, just set it.
volume['bootable'] = bootable
return volume
if volume['status'] == 'error':
raise exc.OpenStackCloudException("Error creating volume")
return self._normalize_volume(volume)
