def test_only_final_states_cause_detailed_request(self):
from unittest import mock
# The state ERROR_CREATING_JOB is only handled when running the job,
# and not while checking the status, so it is not tested.
all_state_apis = {
'COMPLETED': NonQueuedAPI,
'CANCELLED': CancellableAPI,
'ERROR_VALIDATING_JOB': ErrorWhileValidatingAPI,
'ERROR_RUNNING_JOB': ErrorWhileRunningAPI
}
for status, api in all_state_apis.items():
with self.subTest(status=status):
job = self.run_with_api(api())
self.wait_for_initialization(job)
with suppress(BaseFakeAPI.NoMoreStatesError):
self._current_api.progress()
with mock.patch.object(self._current_api,
'get_job',
wraps=self._current_api.get_job):
job.status()
if ApiJobStatus(status) in API_JOB_FINAL_STATES:
self.assertTrue(self._current_api.get_job.called)
else:
self.assertFalse(self._current_api.get_job.called)
def _job_final_status_polling(self,
job_id: str,
timeout: Optional[float] = None,
wait: float = 5) -> Dict[str, Any]:
"""Return the final status of a job via polling.
Args:
job_id: the id of the job.
timeout: seconds to wait for job. If None, wait indefinitely.
wait: seconds between queries.
Returns:
job status.
Raises:
UserTimeoutExceededError: if the user specified timeout has been exceeded.
"""
start_time = time.time()
status_response = self.job_status(job_id)
while ApiJobStatus(
status_response['status']) not in API_JOB_FINAL_STATES:
elapsed_time = time.time() - start_time
if timeout is not None and elapsed_time >= timeout:
raise UserTimeoutExceededError(
'Timeout while waiting for job {}'.format(job_id))
logger.info('API job status = %s (%d seconds)',
status_response['status'], elapsed_time)
time.sleep(wait)
status_response = self.job_status(job_id)
return status_response
def job_status(self, job_id):
summary_fields = ['status', 'error', 'infoQueue']
complete_response = self.job_get(job_id)
try:
ApiJobStatus(complete_response['status'])
except ValueError:
raise ApiIBMQProtocolError('Api Error')
return {key: value for key, value in complete_response.items()
if key in summary_fields}
def job_final_status(self, job_id, *_args, **_kwargs):
start_time = time.time()
status_response = self.job_status(job_id)
while ApiJobStatus(status_response['status']) not in API_JOB_FINAL_STATES:
elapsed_time = time.time() - start_time
timeout = _kwargs.get('timeout', None)
if timeout is not None and elapsed_time >= timeout:
raise UserTimeoutExceededError(
'Timeout while waiting for job {}'.format(job_id))
time.sleep(5)
status_response = self.job_status(job_id)
return status_response
def _handle_status_response(self, message: str) -> None:
"""Handle status response.
Args:
message: Status response message.
"""
response = WebsocketResponseMethod.from_json(message)
if logger.getEffectiveLevel() is logging.DEBUG:
logger.debug('Received message from websocket: %s',
filter_data(response.data))
self._last_message = map_job_status_response(response.data)
if self._message_queue is not None:
self._message_queue.put(self._last_message)
self._current_retry = 0
job_status = response.data.get('status')
if job_status and ApiJobStatus(job_status) in API_JOB_FINAL_STATES:
self.disconnect()
def tearDown(self) -> None:
"""Test level tear down."""
super().tearDown()
failed = False
# It's surprisingly difficult to find out whether the test failed.
# Using a private attribute is not ideal but it'll have to do.
for _, exc_info in self._outcome.errors:
if exc_info is not None:
failed = True
if not failed:
for client, job_id in self._jobs:
try:
job_status = client.job_get(job_id)['status']
if ApiJobStatus(job_status) not in API_JOB_FINAL_STATES:
client.job_cancel(job_id)
time.sleep(1)
client.job_delete(job_id)
except Exception: # pylint: disable=broad-except
pass
def _job_final_status_polling(
self,
job_id: str,
timeout: Optional[float] = None,
wait: float = 5,
status_queue: Optional[RefreshQueue] = None) -> Dict[str, Any]:
"""Return the final status of the job via polling.
Args:
job_id: The ID of the job.
timeout: Time to wait for job, in seconds. If ``None``, wait indefinitely.
wait: Seconds between queries.
status_queue: Queue used to share the latest status.
Returns:
Job status.
Raises:
UserTimeoutExceededError: If the user specified timeout has been exceeded.
"""
start_time = time.time()
status_response = self.job_status(job_id)
while ApiJobStatus(
status_response['status']) not in API_JOB_FINAL_STATES:
# Share the new status.
if status_queue is not None:
status_queue.put(status_response)
elapsed_time = time.time() - start_time
if timeout is not None and elapsed_time >= timeout:
raise UserTimeoutExceededError(
'Timeout while waiting for job {}.'.format(job_id))
logger.info('API job status = %s (%d seconds)',
status_response['status'], elapsed_time)
time.sleep(wait)
status_response = self.job_status(job_id)
return status_response
async def get_job_status(
self,
job_id: str,
timeout: Optional[float] = None,
retries: int = 5,
backoff_factor: float = 0.5,
status_queue: Optional[RefreshQueue] = None) -> Dict[str, str]:
"""Return the status of a job.
Read status messages from the server, which are issued at regular
intervals. When a final state is reached, the server
closes the socket. If the websocket connection is closed without
a reason, the exponential backoff algorithm is used as a basis to
re-establish the connection. The steps are:
1. When a connection closes, sleep for a calculated backoff
time.
2. Try to make a new connection and increment the retry
counter.
3. Attempt to get the job status.
- If the connection is closed, go back to step 1.
- If the job status is read successfully, reset the retry
counter.
4. Continue until the job reaches a final state or the maximum
number of retries is met.
Args:
job_id: ID of the job.
timeout: Timeout value, in seconds.
retries: Max number of retries.
backoff_factor: Backoff factor used to calculate the
time to wait between retries.
status_queue: Queue used to share the latest status.
Returns:
The final API response for the status of the job, as a dictionary that
contains at least the keys ``status`` and ``id``.
Raises:
WebsocketError: If the websocket connection ended unexpectedly.
WebsocketTimeoutError: If the timeout has been reached.
"""
url = '{}/jobs/{}/status/v/1'.format(self.websocket_url, job_id)
original_timeout = timeout
start_time = time.time()
attempt_retry = True # By default, attempt to retry if the websocket connection closes.
current_retry_attempt = 0
last_status = None
websocket = None
while current_retry_attempt <= retries:
try:
websocket = await self._connect(url)
# Read messages from the server until the connection is closed or
# a timeout has been reached.
while True:
try:
if timeout:
response_raw = await asyncio.wait_for(
websocket.recv(), timeout=timeout)
# Decrease the timeout.
timeout = original_timeout - (time.time() -
start_time)
else:
response_raw = await websocket.recv()
response = WebsocketResponseMethod.from_bytes(
response_raw) # type: ignore[arg-type]
if logger.getEffectiveLevel() is logging.DEBUG:
logger.debug('Received message from websocket: %s',
filter_data(response.get_data()))
last_status = map_job_status_response(
response.get_data())
# Share the new status.
if status_queue is not None:
status_queue.put(last_status)
# Successfully received and parsed a message, reset retry counter.
current_retry_attempt = 0
job_status = response.data.get('status')
if (job_status and ApiJobStatus(job_status)
in API_JOB_FINAL_STATES):
return last_status
if timeout and timeout <= 0:
raise WebsocketTimeoutError(
'Timeout reached while getting job status.')
except (futures.TimeoutError, asyncio.TimeoutError):
# Timeout during our wait.
raise WebsocketTimeoutError(
'Timeout reached while getting job status.'
) from None
except ConnectionClosed as ex:
# From the API:
# 4001: closed due to an internal errors
# 4002: closed on purpose (no more updates to send)
# 4003: closed due to job not found.
message = 'Unexpected error'
if ex.code == 4001:
message = 'Internal server error'
elif ex.code == 4002:
logger.debug(
"Websocket connection closed with code 4002: %s",
str(ex))
if status_queue is not None:
status_queue.put(last_status)
return last_status # type: ignore[return-value]
elif ex.code == 4003:
attempt_retry = False # No point in retrying.
message = 'Job id not found'
exception_to_raise = WebsocketError(
'Connection with websocket closed unexpectedly: '
'{}(status_code={})'.format(message, ex.code))
logger.info(
'An exception occurred. Raising "%s" from "%s"',
repr(exception_to_raise), repr(ex))
raise exception_to_raise from ex
except WebsocketError as ex:
logger.info(
'A websocket error occurred while getting job status: %s',
str(ex))
# Specific `WebsocketError` exceptions that are not worth retrying.
if isinstance(
ex,
(WebsocketTimeoutError, WebsocketIBMQProtocolError)):
logger.info(
'The websocket error that occurred could not '
'be retried: %s', str(ex))
raise ex
# Check whether the websocket error should be retried.
current_retry_attempt = current_retry_attempt + 1
if (current_retry_attempt > retries) or (not attempt_retry):
logger.info(
'Max retries exceeded: Failed to establish a websocket '
'connection due to a network error.')
raise ex
# Sleep, and then `continue` with retrying.
backoff_time = self._backoff_time(backoff_factor,
current_retry_attempt)
logger.info(
'Retrying get_job_status via websocket after %s seconds: '
'Attempt #%s', backoff_time, current_retry_attempt)
await asyncio.sleep(
backoff_time) # Block asyncio loop for given backoff time.
continue # Continues next iteration after `finally` block.
finally:
if websocket is not None:
await websocket.close()
# Execution should not reach here, sanity check.
exception_message = 'Max retries exceeded: Failed to establish a websocket ' \
'connection due to a network error.'
logger.info(exception_message)
raise WebsocketError(exception_message)
def get_job_status(
self,
job_id: str,
timeout: Optional[float] = None,
retries: int = 5,
backoff_factor: float = 0.5
) -> Generator[Any, None, Dict[str, str]]:
"""Return the status of a job.
Reads status messages from the API, which are issued at regular
intervals. When a final state is reached, the server
closes the socket. If the websocket connection is closed without
a reason, the exponential backoff algorithm is used as a basis to
reestablish connections. The algorithm takes effect when a
connection closes, it is given by:
1. When a connection closes, sleep for a calculated backoff
time.
2. Try to retrieve another socket and increment a retry
counter.
3. Attempt to get the job status.
- If the connection is closed, go back to step 1.
- If the job status is read successfully, reset the retry
counter.
4. Continue until the job status is complete or the maximum
number of retries is met.
Args:
job_id (str): id of the job.
timeout (float): timeout, in seconds.
retries (int): max number of retries.
backoff_factor (float): backoff factor used to calculate the
time to wait between retries.
Returns:
dict: the API response for the status of a job, as a dict that
contains at least the keys ``status`` and ``id``.
Raises:
WebsocketError: if the websocket connection ended unexpectedly.
WebsocketTimeoutError: if the timeout has been reached.
"""
url = '{}/jobs/{}/status'.format(self.websocket_url, job_id)
original_timeout = timeout
start_time = time.time()
attempt_retry = True # By default, attempt to retry if the websocket connection closes.
current_retry_attempt = 0
last_status = None
websocket = None
while current_retry_attempt <= retries:
try:
websocket = yield from self._connect(url)
# Read messages from the server until the connection is closed or
# a timeout has been reached.
while True:
try:
with warnings.catch_warnings():
# Suppress websockets deprecation warnings until the fix is available
warnings.filterwarnings(
"ignore", category=DeprecationWarning)
if timeout:
response_raw = yield from asyncio.wait_for(
websocket.recv(), timeout=timeout)
# Decrease the timeout.
timeout = original_timeout - (time.time() -
start_time)
else:
response_raw = yield from websocket.recv()
logger.debug('Received message from websocket: %s',
response_raw)
response = WebsocketResponseMethod.from_bytes(
response_raw)
last_status = response.data
# Successfully received and parsed a message, reset retry counter.
current_retry_attempt = 0
job_status = response.data.get('status')
if (job_status and ApiJobStatus(job_status)
in API_JOB_FINAL_STATES):
return last_status
if timeout and timeout <= 0:
raise WebsocketTimeoutError('Timeout reached')
except futures.TimeoutError:
# Timeout during our wait.
raise WebsocketTimeoutError(
'Timeout reached') from None
except ConnectionClosed as ex:
# From the API:
# 4001: closed due to an internal errors
# 4002: closed on purpose (no more updates to send)
# 4003: closed due to job not found.
message = 'Unexpected error'
if ex.code == 4001:
message = 'Internal server error'
elif ex.code == 4002:
return last_status # type: ignore[return-value]
elif ex.code == 4003:
attempt_retry = False # No point in retrying.
message = 'Job id not found'
raise WebsocketError(
'Connection with websocket closed '
'unexpectedly: {}(status_code={})'.format(
message, ex.code)) from ex
except WebsocketError as ex:
logger.warning('%s', ex)
# Specific `WebsocketError` exceptions that are not worth retrying.
if isinstance(
ex,
(WebsocketTimeoutError, WebsocketIBMQProtocolError)):
raise ex
current_retry_attempt = current_retry_attempt + 1
if (current_retry_attempt > retries) or (not attempt_retry):
raise ex
# Sleep, and then `continue` with retrying.
backoff_time = self._backoff_time(backoff_factor,
current_retry_attempt)
logger.warning(
'Retrying get_job_status after %s seconds: '
'Attempt #%s.', backoff_time, current_retry_attempt)
yield from asyncio.sleep(
backoff_time) # Block asyncio loop for given backoff time.
continue # Continues next iteration after `finally` block.
finally:
with warnings.catch_warnings():
# Suppress websockets deprecation warnings until the fix is available
warnings.filterwarnings("ignore",
category=DeprecationWarning)
if websocket is not None:
yield from websocket.close()
# Execution should not reach here, sanity check.
raise WebsocketError('Failed to establish a websocket '
'connection after {} retries.'.format(retries))
def get_job_status(
self,
job_id: str,
timeout: Optional[float] = None
) -> Generator[Any, None, Dict[str, str]]:
"""Return the status of a job.
Reads status messages from the API, which are issued at regular
intervals (20 seconds). When a final state is reached, the server
closes the socket. If the websocket connection is closed without
a reason, there is an attempt to retry one time.
Args:
job_id (str): id of the job.
timeout (float): timeout, in seconds.
Returns:
dict: the API response for the status of a job, as a dict that
contains at least the keys ``status`` and ``id``.
Raises:
WebsocketError: if the websocket connection ended unexpectedly.
WebsocketTimeoutError: if the timeout has been reached.
"""
url = '{}/jobs/{}/status'.format(self.websocket_url, job_id)
websocket = yield from self._connect(url)
original_timeout = timeout
start_time = time.time()
attempt_retry = True # By default, attempt to retry if the websocket connection closes.
last_status = None
try:
# Read messages from the server until the connection is closed or
# a timeout has been reached.
while True:
try:
with warnings.catch_warnings():
# Suppress websockets deprecation warnings until the fix is available
warnings.filterwarnings("ignore", category=DeprecationWarning)
if timeout:
response_raw = yield from asyncio.wait_for(
websocket.recv(), timeout=timeout)
# Decrease the timeout, with a 5-second grace period.
elapsed_time = time.time() - start_time
timeout = max(5, int(original_timeout - elapsed_time))
else:
response_raw = yield from websocket.recv()
logger.debug('Received message from websocket: %s',
response_raw)
response = WebsocketMessage.from_bytes(response_raw)
last_status = response.data
job_status = response.data.get('status')
if (job_status and
ApiJobStatus(job_status) in API_JOB_FINAL_STATES):
break
except futures.TimeoutError:
# Timeout during our wait.
raise WebsocketTimeoutError('Timeout reached') from None
except ConnectionClosed as ex:
# From the API:
# 4001: closed due to an internal errors
# 4002: closed on purpose (no more updates to send)
# 4003: closed due to job not found.
message = 'Unexpected error'
if ex.code == 4001:
message = 'Internal server error'
elif ex.code == 4002:
break
elif ex.code == 4003:
attempt_retry = False # No point in retrying.
message = 'Job id not found'
if attempt_retry:
logger.warning('Connection with the websocket closed '
'unexpectedly: %s(status_code=%s). '
'Retrying get_job_status.', message, ex.code)
attempt_retry = False # Disallow further retries.
websocket = yield from self._connect(url)
continue
raise WebsocketError('Connection with websocket closed '
'unexpectedly: {}'.format(message)) from ex
finally:
with warnings.catch_warnings():
# Suppress websockets deprecation warnings until the fix is available
warnings.filterwarnings("ignore", category=DeprecationWarning)
yield from websocket.close()
return last_status
def get_job_status(self, job_id, timeout=None):
"""Return the status of a job.
Reads status messages from the API, which are issued at regular
intervals (20 seconds). When a final state is reached, the server
closes the socket.
Args:
job_id (str): id of the job.
timeout (int): timeout, in seconds.
Returns:
dict: the API response for the status of a job, as a dict that
contains at least the keys ``status`` and ``id``.
Raises:
WebsocketError: if the websocket connection ended unexpectedly.
WebsocketTimeoutError: if the timeout has been reached.
"""
url = '{}/jobs/{}/status'.format(self.websocket_url, job_id)
websocket = yield from self._connect(url)
original_timeout = timeout
start_time = time.time()
last_status = None
try:
# Read messages from the server until the connection is closed or
# a timeout has been reached.
while True:
try:
if timeout:
response_raw = yield from asyncio.wait_for(
websocket.recv(), timeout=timeout)
# Decrease the timeout, with a 5-second grace period.
elapsed_time = time.time() - start_time
timeout = max(5, int(original_timeout - elapsed_time))
else:
response_raw = yield from websocket.recv()
logger.debug('Received message from websocket: %s',
response_raw)
response = WebsocketMessage.from_bytes(response_raw)
last_status = response.data
job_status = response.data.get('status')
if (job_status and ApiJobStatus(job_status)
in API_JOB_FINAL_STATES):
# Force closing the connection.
# TODO: revise with API team the automatic closing.
raise ConnectionClosed(
code=4002,
reason='IBMQProvider closed the connection')
except futures.TimeoutError:
# Timeout during our wait.
raise WebsocketTimeoutError('Timeout reached') from None
except ConnectionClosed as ex:
# From the API:
# 4001: closed due to an internal erros
# 4002: closed on purpose (no more updates to send)
# 4003: closed due to job not found.
message = 'Unexpected error'
if ex.code == 4001:
message = 'Internal server error'
if ex.code == 4002:
break
elif ex.code == 4003:
message = 'Job id not found'
raise WebsocketError(
'Connection with websocket closed '
'unexpectedly: {}'.format(message)) from ex
finally:
yield from websocket.close()
return last_status
