def wait_for(resource,
is_ready,
update_resource=None,
timeout=60,
check_interval=1):
"""Waits for the given resource to come into the desired state.
Uses the readiness check function passed as a parameter and (optionally)
a function that updates the resource being waited for.
:param is_ready: A predicate that should take the resource object and
return True iff it is ready to be returned
:param update_resource: Function that should take the resource object
and return an 'updated' resource. If set to
None, no result updating is performed
:param timeout: Timeout in seconds after which a TimeoutException will be
raised
:param check_interval: Interval in seconds between the two consecutive
readiness checks
:returns: The "ready" resource object
"""
start = time.time()
while True:
# NOTE(boden): mitigate 1st iteration waits by updating immediately
if update_resource:
resource = update_resource(resource)
if is_ready(resource):
break
time.sleep(check_interval)
if time.time() - start > timeout:
raise exceptions.TimeoutException()
return resource
def _create_v1rc(self, manifest):
"""Create rc on the specify cluster.
:param manifest: manifest use to create the replication controller
"""
k8s_api = self._get_k8s_api_client()
suffix = "-"
for i in range(5):
suffix = suffix + random.choice(string.ascii_lowercase)
rcname = manifest["metadata"]["name"] + suffix
manifest["metadata"]["name"] = rcname
resp = k8s_api.create_namespaced_replication_controller(
body=manifest, namespace="default")
expectd_status = resp.spec.replicas
start = time.time()
while True:
resp = k8s_api.read_namespaced_replication_controller(
name=rcname, namespace="default")
status = resp.status.replicas
if status == expectd_status:
return resp
else:
if time.time() - start > CONF.openstack.k8s_rc_create_timeout:
raise exceptions.TimeoutException(
desired_status=expectd_status,
resource_name=rcname,
resource_type="ReplicationController",
resource_id=resp.metadata.uid,
resource_status=status,
timeout=CONF.openstack.k8s_rc_create_timeout)
common_utils.interruptable_sleep(
CONF.openstack.k8s_rc_create_poll_interval)
def wait_for_delete(resource, update_resource=None, timeout=60,
check_interval=1):
"""Wait for the full deletion of resource.
:param update_resource: Function that should take the resource object
and return an 'updated' resource, or raise
exception rally.exceptions.GetResourceNotFound
that means that resource is deleted.
:param timeout: Timeout in seconds after which a TimeoutException will be
raised
:param check_interval: Interval in seconds between the two consecutive
readiness checks
"""
start = time.time()
while True:
try:
resource = update_resource(resource)
except exceptions.GetResourceNotFound:
break
time.sleep(check_interval)
if time.time() - start > timeout:
raise exceptions.TimeoutException(
desired_status="deleted",
resource_name=getattr(resource, "name", repr(resource)),
resource_type=resource.__class__.__name__,
resource_id=getattr(resource, "id", "<no id>"),
resource_status=get_status(resource))
def wait_is_ready(resource,
is_ready,
update_resource=None,
timeout=60,
check_interval=1):
resource_repr = getattr(resource, "name", repr(resource))
start = time.time()
while True:
if update_resource is not None:
resource = update_resource(resource)
if is_ready(resource):
return resource
time.sleep(check_interval)
if time.time() - start > timeout:
raise exceptions.TimeoutException(
desired_status=str(is_ready),
resource_name=resource_repr,
resource_type=resource.__class__.__name__,
resource_id=getattr(resource, "id", "<no id>"),
resource_status=get_status(resource),
timeout=timeout)
def wait_for_not_found(name, read_method, resource_type=None, **kwargs):
"""Util method for polling status while resource exists.
:param name: resource name
:param read_method: method to poll
:param resource_type: resource type for extended exceptions
:param kwargs: additional kwargs for read_method
"""
sleep_time = CONF.kubernetes.status_poll_interval
retries_total = CONF.kubernetes.status_total_retries
commonutils.interruptable_sleep(CONF.kubernetes.start_prepoll_delay)
i = 0
while i < retries_total:
try:
resp = read_method(name=name, **kwargs)
resp_id = resp.metadata.uid
current_status = resp.status.phase
except rest.ApiException as ex:
if ex.status == 404:
return
else:
raise
else:
commonutils.interruptable_sleep(sleep_time)
i += 1
if i == retries_total:
raise exceptions.TimeoutException(desired_status="Terminated",
resource_name=name,
resource_type=resource_type,
resource_id=resp_id or "<no id>",
resource_status=current_status,
timeout=(retries_total *
sleep_time))
def wait_for_delete(resource,
update_resource=None,
timeout=60,
check_interval=1):
"""Wait for the full deletion of resource.
:param update_resource: Function that should take the resource object
and return an 'updated' resource, or raise
exception rally.exceptions.GetResourceNotFound
that means that resource is deleted.
:param timeout: Timeout in seconds after which a TimeoutException will be
raised
:param check_interval: Interval in seconds between the two consecutive
readiness checks
"""
start = time.time()
while True:
try:
resource = update_resource(resource)
except exceptions.GetResourceNotFound:
break
time.sleep(check_interval)
if time.time() - start > timeout:
raise exceptions.TimeoutException()
def wait_for_status(name, status, read_method, resource_type=None, **kwargs):
"""Util method for polling status until it won't be equals to `status`.
:param name: resource name
:param status: status waiting for
:param read_method: method to poll
:param resource_type: resource type for extended exceptions
:param kwargs: additional kwargs for read_method
"""
sleep_time = CONF.kubernetes.status_poll_interval
retries_total = CONF.kubernetes.status_total_retries
commonutils.interruptable_sleep(CONF.kubernetes.start_prepoll_delay)
i = 0
while i < retries_total:
resp = read_method(name=name, **kwargs)
resp_id = resp.metadata.uid
current_status = resp.status.phase
if resp.status.phase != status:
i += 1
commonutils.interruptable_sleep(sleep_time)
else:
return
if i == retries_total:
raise exceptions.TimeoutException(desired_status=status,
resource_name=name,
resource_type=resource_type,
resource_id=resp_id or "<no id>",
resource_status=current_status,
timeout=(retries_total *
sleep_time))
def test_boot_runcommand_delete_ping_wait_timeouts(self):
scenario = self.create_env(vmtasks.BootRuncommandDelete(self.context))
scenario._wait_for_ping.side_effect = exceptions.TimeoutException(
resource_type="foo_resource",
resource_name="foo_name",
resource_id="foo_id",
desired_status="foo_desired_status",
resource_status="foo_resource_status",
timeout=2)
exc = self.assertRaises(exceptions.TimeoutException,
scenario.run,
"foo_image",
"foo_flavor",
"foo_interpreter",
"foo_script",
"foo_username",
wait_for_ping=True)
self.assertEqual(exc.kwargs["resource_type"], "foo_resource")
self.assertEqual(exc.kwargs["resource_name"], "foo_name")
self.assertEqual(exc.kwargs["resource_id"], "foo_id")
self.assertEqual(exc.kwargs["desired_status"], "foo_desired_status")
self.assertEqual(exc.kwargs["resource_status"], "foo_resource_status")
scenario._delete_server_with_fip.assert_called_once_with(
"foo_server", self.ip, force_delete=False)
self.assertFalse(scenario.add_output.called)
def get_ip(self, name):
"""Get container's ip by name."""
cmd = 'lxc-attach -n %s ip addr list dev eth0' % name
for attempt in range(1, 16):
code, stdout = self.server.ssh.execute(cmd)[:2]
if code:
continue
for line in stdout.splitlines():
m = INET_ADDR_RE.match(line)
if m:
return m.group(1)
time.sleep(attempt)
msg = _('Timeout waiting for ip address of container "%s"') % name
raise exceptions.TimeoutException(msg)
def test_test_existing_designate_from_vm_wait_timeout(
self, mock_rally_task_utils_wait_for_status,
mock_rally_task_utils_get_from_manager):
scenario, args = self.create_env_for_designate()
mock_rally_task_utils_wait_for_status.side_effect = \
exceptions.TimeoutException(
resource_type="foo_resource",
resource_name="foo_name",
resource_id="foo_id",
desired_status="foo_desired_status",
resource_status="foo_resource_status",
timeout=2)
self.assertRaises(exceptions.TimeoutException, scenario.run,
"foo_flavor", "foo_image", "foo_interpreter",
"foo_script", "foo_username")
scenario._delete_server_with_fip.assert_called_once_with(
"foo_server", self.ip, force_delete=False)
self.assertFalse(scenario.add_output.called)
def _create_v1pod(self, manifest):
"""Create a pod on the specify cluster.
:param manifest: manifest use to create the pod
"""
k8s_api = self._get_k8s_api_client()
podname = manifest["metadata"]["name"] + "-"
for i in range(5):
podname = podname + random.choice(string.ascii_lowercase)
manifest["metadata"]["name"] = podname
for i in range(150):
try:
k8s_api.create_namespaced_pod(body=manifest,
namespace="default")
break
except ApiException as e:
if e.status != 403:
raise
time.sleep(2)
start = time.time()
while True:
resp = k8s_api.read_namespaced_pod(name=podname,
namespace="default")
if resp.status.conditions:
for condition in resp.status.conditions:
if condition.type.lower() == "ready" and \
condition.status.lower() == "true":
return resp
if (time.time() - start > CONF.openstack.k8s_pod_create_timeout):
raise exceptions.TimeoutException(
desired_status="Ready",
resource_name=podname,
resource_type="Pod",
resource_id=resp.metadata.uid,
resource_status=resp.status,
timeout=CONF.openstack.k8s_pod_create_timeout)
common_utils.interruptable_sleep(
CONF.openstack.k8s_pod_create_poll_interval)
def test_create_share_and_access_from_vm_wait_timeout(
self, params, mock_rally_task_utils_wait_for_status,
mock_rally_task_utils_get_from_manager):
scenario, fake_share = self.create_env(
shares.CreateShareAndAccessFromVM(self.context))
mock_rally_task_utils_wait_for_status.side_effect = \
exceptions.TimeoutException(
resource_type="foo_resource",
resource_name="foo_name",
resource_id="foo_id",
desired_status="foo_desired_status",
resource_status="foo_resource_status",
timeout=2)
self.assertRaises(exceptions.TimeoutException, scenario.run,
"foo_flavor", "foo_image", "foo_interpreter",
"foo_script", "foo_username")
scenario._delete_server_with_fip.assert_called_once_with(
"foo_server", self.ip, force_delete=False)
self.assertFalse(scenario.add_output.called)
scenario._delete_share.assert_called_once_with(fake_share)
def wait_for_status(resource,
ready_statuses,
failure_statuses=None,
status_attr="status",
update_resource=None,
timeout=60,
check_interval=1,
check_deletion=False):
resource_repr = getattr(resource, "name", repr(resource))
if not isinstance(ready_statuses, (set, list, tuple)):
raise ValueError("Ready statuses should be supplied as set, list or "
"tuple")
if failure_statuses and not isinstance(failure_statuses,
(set, list, tuple)):
raise ValueError("Failure statuses should be supplied as set, list or "
"tuple")
# make all statuses upper case
ready_statuses = set([s.upper() for s in ready_statuses or []])
failure_statuses = set([s.upper() for s in failure_statuses or []])
if len(ready_statuses & failure_statuses) > 0:
raise ValueError(
"Can't wait for resource's %s status. Ready and Failure"
"statuses conflict." % resource_repr)
if not ready_statuses:
raise ValueError("Can't wait for resource's %s status. No ready "
"statuses provided" % resource_repr)
if not update_resource:
raise ValueError(
"Can't wait for resource's %s status. No update method." %
resource_repr)
start = time.time()
latest_status = get_status(resource, status_attr)
latest_status_update = start
while True:
try:
resource = update_resource(resource)
except exceptions.GetResourceNotFound:
if check_deletion:
return
else:
raise
status = get_status(resource, status_attr)
if status != latest_status:
current_time = time.time()
delta = current_time - latest_status_update
LOG.debug(
"Waiting for resource %(resource)s. Status changed: "
"%(latest)s => %(current)s in %(delta)s" % {
"resource": resource_repr,
"latest": latest_status,
"current": status,
"delta": delta
})
latest_status = status
latest_status_update = current_time
if status in ready_statuses:
return resource
if status in failure_statuses:
raise exceptions.GetResourceErrorStatus(
resource=resource,
status=status,
fault="Status in failure list %s" % str(failure_statuses))
time.sleep(check_interval)
if time.time() - start > timeout:
raise exceptions.TimeoutException(
desired_status="('%s')" % "', '".join(ready_statuses),
resource_name=resource_repr,
resource_type=resource.__class__.__name__,
resource_id=getattr(resource, "id", "<no id>"),
resource_status=get_status(resource))
