def _wait_for_events_to_update_in_DB(
self, execution, message, timeout_seconds=60):
""" It might take longer for events to show up in the DB than it takes
for Execution status to return, this method waits until a specific
event is listed in the DB, and will fail in case of a time out.
"""
deadline = time.time() + timeout_seconds
all_events = []
while message not in [e['message'] for e in all_events]:
time.sleep(0.5)
if time.time() > deadline:
raise utils.TimeoutException(
'Execution timed out: \n{0}'.format(
json.dumps(execution, indent=2)
)
)
# This might fail due to the fact that we're changing the DB in
# real time - it's OK. When restoring a snapshot we also restart
# the rest service and nginx, which might lead to intermittent
# connection errors. Just try again
try:
all_events = self.client.events.list(include_logs=True)
except (CloudifyClientError, ConnectionError):
pass
def _wait_for_execution_to_end(self, execution, timeout_seconds=30):
"""Can't use the `wait_for_execution_to_end` in the class because
we need to be able to handle client errors
"""
deadline = time.time() + timeout_seconds
while execution.status not in Execution.END_STATES:
time.sleep(0.5)
# This might fail due to the fact that we're changing the DB in
# real time - it's OK. Just try again
try:
execution = self.client.executions.get(execution.id)
except CloudifyClientError:
pass
if time.time() > deadline:
raise utils.TimeoutException(
'Execution timed out: \n{0}'
.format(json.dumps(execution, indent=2)))
return execution