def _handle_status_response(self, message: str) -> None:
"""Handle status response.
Args:
message: Status response message.
"""
response = WebsocketResponseMethod.from_json(message)
if logger.getEffectiveLevel() is logging.DEBUG:
logger.debug('Received message from websocket: %s',
filter_data(response.data))
self._last_message = map_job_status_response(response.data)
if self._message_queue is not None:
self._message_queue.put(self._last_message)
self._current_retry = 0
job_status = response.data.get('status')
if job_status and ApiJobStatus(job_status) in API_JOB_FINAL_STATES:
self.disconnect()
def _log_request_info(
self,
url: str,
method: str,
request_data: Dict[str, Any]
) -> None:
"""Log the request data, filtering out specific information.
Note:
The string ``...`` is used to denote information that has been filtered out
from the request, within the url and request data. Currently, the backend name
is filtered out from endpoint URLs, using a regex to capture the name, and from
the data sent to the server when submitting a job.
The request data is only logged for the following URLs, since they contain useful
information: ``/Jobs`` (POST), ``/Jobs/status`` (GET),
and ``/devices/<device_name>/properties`` (GET).
Args:
url: URL for the new request.
method: Method for the new request (e.g. ``POST``)
request_data:Additional arguments for the request.
Raises:
Exception: If there was an error logging the request information.
"""
# Replace the device name in the URL with `...` if it matches, otherwise leave it as is.
filtered_url = re.sub(RE_DEVICES_ENDPOINT, '\\1...\\3', url)
if self._is_worth_logging(filtered_url):
try:
if logger.getEffectiveLevel() is logging.DEBUG:
request_data_to_log = ""
if filtered_url in ('/devices/.../properties', '/Jobs'):
# Log filtered request data for these endpoints.
request_data_to_log = 'Request Data: {}.'.format(filter_data(request_data))
logger.debug('Endpoint: %s. Method: %s. %s',
filtered_url, method.upper(), request_data_to_log)
except Exception as ex: # pylint: disable=broad-except
# Catch general exception so as not to disturb the program if filtering fails.
logger.info('Filtering failed when logging request information: %s', str(ex))
def jobs(self,
limit: int = 10,
skip: int = 0,
descending: bool = True,
extra_filter: Dict[str, Any] = None) -> List[Dict[str, Any]]:
"""Return a list of job information.
Args:
limit: Maximum number of items to return.
skip: Offset for the items to return.
descending: Whether the jobs should be in descending order.
extra_filter: Additional filtering passed to the query.
Returns:
JSON response.
"""
url = self.get_url('jobs_status')
order = 'DESC' if descending else 'ASC'
query = {
'order': 'creationDate ' + order,
'limit': limit,
'skip': skip,
}
if extra_filter:
query['where'] = extra_filter
if logger.getEffectiveLevel() is logging.DEBUG:
logger.debug(
"Endpoint: %s. Method: GET. Request Data: {'filter': %s}", url,
filter_data(query))
data = self.session.get(url, params={
'filter': json.dumps(query)
}).json()
for job_data in data:
map_job_response(job_data)
return data
async def get_job_status(
self,
job_id: str,
timeout: Optional[float] = None,
retries: int = 5,
backoff_factor: float = 0.5,
status_queue: Optional[RefreshQueue] = None) -> Dict[str, str]:
"""Return the status of a job.
Read status messages from the server, which are issued at regular
intervals. When a final state is reached, the server
closes the socket. If the websocket connection is closed without
a reason, the exponential backoff algorithm is used as a basis to
re-establish the connection. The steps are:
1. When a connection closes, sleep for a calculated backoff
time.
2. Try to make a new connection and increment the retry
counter.
3. Attempt to get the job status.
- If the connection is closed, go back to step 1.
- If the job status is read successfully, reset the retry
counter.
4. Continue until the job reaches a final state or the maximum
number of retries is met.
Args:
job_id: ID of the job.
timeout: Timeout value, in seconds.
retries: Max number of retries.
backoff_factor: Backoff factor used to calculate the
time to wait between retries.
status_queue: Queue used to share the latest status.
Returns:
The final API response for the status of the job, as a dictionary that
contains at least the keys ``status`` and ``id``.
Raises:
WebsocketError: If the websocket connection ended unexpectedly.
WebsocketTimeoutError: If the timeout has been reached.
"""
url = '{}/jobs/{}/status/v/1'.format(self.websocket_url, job_id)
original_timeout = timeout
start_time = time.time()
attempt_retry = True # By default, attempt to retry if the websocket connection closes.
current_retry_attempt = 0
last_status = None
websocket = None
while current_retry_attempt <= retries:
try:
websocket = await self._connect(url)
# Read messages from the server until the connection is closed or
# a timeout has been reached.
while True:
try:
if timeout:
response_raw = await asyncio.wait_for(
websocket.recv(), timeout=timeout)
# Decrease the timeout.
timeout = original_timeout - (time.time() -
start_time)
else:
response_raw = await websocket.recv()
response = WebsocketResponseMethod.from_bytes(
response_raw) # type: ignore[arg-type]
if logger.getEffectiveLevel() is logging.DEBUG:
logger.debug('Received message from websocket: %s',
filter_data(response.get_data()))
last_status = map_job_status_response(
response.get_data())
# Share the new status.
if status_queue is not None:
status_queue.put(last_status)
# Successfully received and parsed a message, reset retry counter.
current_retry_attempt = 0
job_status = response.data.get('status')
if (job_status and ApiJobStatus(job_status)
in API_JOB_FINAL_STATES):
return last_status
if timeout and timeout <= 0:
raise WebsocketTimeoutError(
'Timeout reached while getting job status.')
except (futures.TimeoutError, asyncio.TimeoutError):
# Timeout during our wait.
raise WebsocketTimeoutError(
'Timeout reached while getting job status.'
) from None
except ConnectionClosed as ex:
# From the API:
# 4001: closed due to an internal errors
# 4002: closed on purpose (no more updates to send)
# 4003: closed due to job not found.
message = 'Unexpected error'
if ex.code == 4001:
message = 'Internal server error'
elif ex.code == 4002:
logger.debug(
"Websocket connection closed with code 4002: %s",
str(ex))
if status_queue is not None:
status_queue.put(last_status)
return last_status # type: ignore[return-value]
elif ex.code == 4003:
attempt_retry = False # No point in retrying.
message = 'Job id not found'
exception_to_raise = WebsocketError(
'Connection with websocket closed unexpectedly: '
'{}(status_code={})'.format(message, ex.code))
logger.info(
'An exception occurred. Raising "%s" from "%s"',
repr(exception_to_raise), repr(ex))
raise exception_to_raise from ex
except WebsocketError as ex:
logger.info(
'A websocket error occurred while getting job status: %s',
str(ex))
# Specific `WebsocketError` exceptions that are not worth retrying.
if isinstance(
ex,
(WebsocketTimeoutError, WebsocketIBMQProtocolError)):
logger.info(
'The websocket error that occurred could not '
'be retried: %s', str(ex))
raise ex
# Check whether the websocket error should be retried.
current_retry_attempt = current_retry_attempt + 1
if (current_retry_attempt > retries) or (not attempt_retry):
logger.info(
'Max retries exceeded: Failed to establish a websocket '
'connection due to a network error.')
raise ex
# Sleep, and then `continue` with retrying.
backoff_time = self._backoff_time(backoff_factor,
current_retry_attempt)
logger.info(
'Retrying get_job_status via websocket after %s seconds: '
'Attempt #%s', backoff_time, current_retry_attempt)
await asyncio.sleep(
backoff_time) # Block asyncio loop for given backoff time.
continue # Continues next iteration after `finally` block.
finally:
if websocket is not None:
await websocket.close()
# Execution should not reach here, sanity check.
exception_message = 'Max retries exceeded: Failed to establish a websocket ' \
'connection due to a network error.'
logger.info(exception_message)
raise WebsocketError(exception_message)
