def wait_for_event(self, execution, message, timeout_seconds=60,
allow_connection_error=True, client=None):
""" Wait until a specific event is listed in the DB.
Events are stored asynchronously, so we might need to wait for
an event to be stored in the database.
:param execution: Check events for this execution
:param message: Wait for an event with this message
:param timeout_seconds: How long to keep polling
:param allow_connection_error: Catch the exception if a connection
error happens when polling for events. This is useful for tests
that also change the db in the meantime (snapshots)
:param client: The restclient to use
"""
client = client or self.client
deadline = time.time() + timeout_seconds
all_events = []
while not any(message in e['message'] for e in all_events):
time.sleep(0.5)
if time.time() > deadline:
raise utils.TimeoutException(
'Execution timed out when waiting for message {0}: \n{1}'
.format(message, json.dumps(execution, indent=2))
)
# This might fail due to the fact that we're changing the DB in
# real time - it's OK. When restoring a snapshot we also restart
# the rest service and nginx, which might lead to intermittent
# connection errors. Just try again
try:
all_events = client.events.list(
execution_id=execution.id, include_logs=True)
except (CloudifyClientError, ConnectionError):
if not allow_connection_error:
raise
def wait_for_execution_to_end(self,
execution,
is_group=False,
timeout_seconds=240,
client=None):
client = client or self.client
if is_group:
get = client.execution_groups.get
else:
get = client.executions.get
deadline = time.time() + timeout_seconds
while execution.status not in Execution.END_STATES:
if not is_group:
assert execution.ended_at is None
time.sleep(0.5)
execution = get(execution.id)
if time.time() > deadline:
raise utils.TimeoutException(
'Execution timed out: \n{0}'.format(
json.dumps(execution, indent=2)))
if execution.status == Execution.FAILED:
if is_group:
# no .error in exec-groups
raise RuntimeError('Group execution failed: {0}'.format(
execution.status))
raise RuntimeError('Workflow execution failed: {0} [{1}]'.format(
execution.error, execution.status))
return execution
def try_fetch_status():
status = ''
while status != STATES.NOT_RUNNING:
time.sleep(0.5)
status = client.snapshots.get_status()['status']
if time.time() > deadline:
raise utils.TimeoutException(
'Snapshot restore timed out.{0}'
''.format(error_message_suffix))
def wait_for_execution_to_end(execution, timeout_seconds=240):
client = test_utils.create_rest_client()
deadline = time.time() + timeout_seconds
while execution.status not in Execution.END_STATES:
time.sleep(0.5)
execution = client.executions.get(execution.id)
if time.time() > deadline:
raise utils.TimeoutException(
'Execution timed out: \n{0}'.format(
json.dumps(execution, indent=2)))
if execution.status == Execution.FAILED:
raise RuntimeError('Workflow execution failed: {0} [{1}]'.format(
execution.error, execution.status))
return execution
