def wait_for_event(self, execution, message, timeout_seconds=60, allow_connection_error=True, client=None): """ Wait until a specific event is listed in the DB. Events are stored asynchronously, so we might need to wait for an event to be stored in the database. :param execution: Check events for this execution :param message: Wait for an event with this message :param timeout_seconds: How long to keep polling :param allow_connection_error: Catch the exception if a connection error happens when polling for events. This is useful for tests that also change the db in the meantime (snapshots) :param client: The restclient to use """ client = client or self.client deadline = time.time() + timeout_seconds all_events = [] while not any(message in e['message'] for e in all_events): time.sleep(0.5) if time.time() > deadline: raise utils.TimeoutException( 'Execution timed out when waiting for message {0}: \n{1}' .format(message, json.dumps(execution, indent=2)) ) # This might fail due to the fact that we're changing the DB in # real time - it's OK. When restoring a snapshot we also restart # the rest service and nginx, which might lead to intermittent # connection errors. Just try again try: all_events = client.events.list( execution_id=execution.id, include_logs=True) except (CloudifyClientError, ConnectionError): if not allow_connection_error: raise
def wait_for_execution_to_end(self, execution, is_group=False, timeout_seconds=240, client=None): client = client or self.client if is_group: get = client.execution_groups.get else: get = client.executions.get deadline = time.time() + timeout_seconds while execution.status not in Execution.END_STATES: if not is_group: assert execution.ended_at is None time.sleep(0.5) execution = get(execution.id) if time.time() > deadline: raise utils.TimeoutException( 'Execution timed out: \n{0}'.format( json.dumps(execution, indent=2))) if execution.status == Execution.FAILED: if is_group: # no .error in exec-groups raise RuntimeError('Group execution failed: {0}'.format( execution.status)) raise RuntimeError('Workflow execution failed: {0} [{1}]'.format( execution.error, execution.status)) return execution
def try_fetch_status(): status = '' while status != STATES.NOT_RUNNING: time.sleep(0.5) status = client.snapshots.get_status()['status'] if time.time() > deadline: raise utils.TimeoutException( 'Snapshot restore timed out.{0}' ''.format(error_message_suffix))
def wait_for_execution_to_end(execution, timeout_seconds=240): client = test_utils.create_rest_client() deadline = time.time() + timeout_seconds while execution.status not in Execution.END_STATES: time.sleep(0.5) execution = client.executions.get(execution.id) if time.time() > deadline: raise utils.TimeoutException( 'Execution timed out: \n{0}'.format( json.dumps(execution, indent=2))) if execution.status == Execution.FAILED: raise RuntimeError('Workflow execution failed: {0} [{1}]'.format( execution.error, execution.status)) return execution