def _handle_status_response(self, message: str) -> None: """Handle status response. Args: message: Status response message. """ response = WebsocketResponseMethod.from_json(message) if logger.getEffectiveLevel() is logging.DEBUG: logger.debug('Received message from websocket: %s', filter_data(response.data)) self._last_message = map_job_status_response(response.data) if self._message_queue is not None: self._message_queue.put(self._last_message) self._current_retry = 0 job_status = response.data.get('status') if job_status and ApiJobStatus(job_status) in API_JOB_FINAL_STATES: self.disconnect()
def _log_request_info( self, url: str, method: str, request_data: Dict[str, Any] ) -> None: """Log the request data, filtering out specific information. Note: The string ``...`` is used to denote information that has been filtered out from the request, within the url and request data. Currently, the backend name is filtered out from endpoint URLs, using a regex to capture the name, and from the data sent to the server when submitting a job. The request data is only logged for the following URLs, since they contain useful information: ``/Jobs`` (POST), ``/Jobs/status`` (GET), and ``/devices/<device_name>/properties`` (GET). Args: url: URL for the new request. method: Method for the new request (e.g. ``POST``) request_data:Additional arguments for the request. Raises: Exception: If there was an error logging the request information. """ # Replace the device name in the URL with `...` if it matches, otherwise leave it as is. filtered_url = re.sub(RE_DEVICES_ENDPOINT, '\\1...\\3', url) if self._is_worth_logging(filtered_url): try: if logger.getEffectiveLevel() is logging.DEBUG: request_data_to_log = "" if filtered_url in ('/devices/.../properties', '/Jobs'): # Log filtered request data for these endpoints. request_data_to_log = 'Request Data: {}.'.format(filter_data(request_data)) logger.debug('Endpoint: %s. Method: %s. %s', filtered_url, method.upper(), request_data_to_log) except Exception as ex: # pylint: disable=broad-except # Catch general exception so as not to disturb the program if filtering fails. logger.info('Filtering failed when logging request information: %s', str(ex))
def jobs(self, limit: int = 10, skip: int = 0, descending: bool = True, extra_filter: Dict[str, Any] = None) -> List[Dict[str, Any]]: """Return a list of job information. Args: limit: Maximum number of items to return. skip: Offset for the items to return. descending: Whether the jobs should be in descending order. extra_filter: Additional filtering passed to the query. Returns: JSON response. """ url = self.get_url('jobs_status') order = 'DESC' if descending else 'ASC' query = { 'order': 'creationDate ' + order, 'limit': limit, 'skip': skip, } if extra_filter: query['where'] = extra_filter if logger.getEffectiveLevel() is logging.DEBUG: logger.debug( "Endpoint: %s. Method: GET. Request Data: {'filter': %s}", url, filter_data(query)) data = self.session.get(url, params={ 'filter': json.dumps(query) }).json() for job_data in data: map_job_response(job_data) return data
async def get_job_status( self, job_id: str, timeout: Optional[float] = None, retries: int = 5, backoff_factor: float = 0.5, status_queue: Optional[RefreshQueue] = None) -> Dict[str, str]: """Return the status of a job. Read status messages from the server, which are issued at regular intervals. When a final state is reached, the server closes the socket. If the websocket connection is closed without a reason, the exponential backoff algorithm is used as a basis to re-establish the connection. The steps are: 1. When a connection closes, sleep for a calculated backoff time. 2. Try to make a new connection and increment the retry counter. 3. Attempt to get the job status. - If the connection is closed, go back to step 1. - If the job status is read successfully, reset the retry counter. 4. Continue until the job reaches a final state or the maximum number of retries is met. Args: job_id: ID of the job. timeout: Timeout value, in seconds. retries: Max number of retries. backoff_factor: Backoff factor used to calculate the time to wait between retries. status_queue: Queue used to share the latest status. Returns: The final API response for the status of the job, as a dictionary that contains at least the keys ``status`` and ``id``. Raises: WebsocketError: If the websocket connection ended unexpectedly. WebsocketTimeoutError: If the timeout has been reached. """ url = '{}/jobs/{}/status/v/1'.format(self.websocket_url, job_id) original_timeout = timeout start_time = time.time() attempt_retry = True # By default, attempt to retry if the websocket connection closes. current_retry_attempt = 0 last_status = None websocket = None while current_retry_attempt <= retries: try: websocket = await self._connect(url) # Read messages from the server until the connection is closed or # a timeout has been reached. while True: try: if timeout: response_raw = await asyncio.wait_for( websocket.recv(), timeout=timeout) # Decrease the timeout. timeout = original_timeout - (time.time() - start_time) else: response_raw = await websocket.recv() response = WebsocketResponseMethod.from_bytes( response_raw) # type: ignore[arg-type] if logger.getEffectiveLevel() is logging.DEBUG: logger.debug('Received message from websocket: %s', filter_data(response.get_data())) last_status = map_job_status_response( response.get_data()) # Share the new status. if status_queue is not None: status_queue.put(last_status) # Successfully received and parsed a message, reset retry counter. current_retry_attempt = 0 job_status = response.data.get('status') if (job_status and ApiJobStatus(job_status) in API_JOB_FINAL_STATES): return last_status if timeout and timeout <= 0: raise WebsocketTimeoutError( 'Timeout reached while getting job status.') except (futures.TimeoutError, asyncio.TimeoutError): # Timeout during our wait. raise WebsocketTimeoutError( 'Timeout reached while getting job status.' ) from None except ConnectionClosed as ex: # From the API: # 4001: closed due to an internal errors # 4002: closed on purpose (no more updates to send) # 4003: closed due to job not found. message = 'Unexpected error' if ex.code == 4001: message = 'Internal server error' elif ex.code == 4002: logger.debug( "Websocket connection closed with code 4002: %s", str(ex)) if status_queue is not None: status_queue.put(last_status) return last_status # type: ignore[return-value] elif ex.code == 4003: attempt_retry = False # No point in retrying. message = 'Job id not found' exception_to_raise = WebsocketError( 'Connection with websocket closed unexpectedly: ' '{}(status_code={})'.format(message, ex.code)) logger.info( 'An exception occurred. Raising "%s" from "%s"', repr(exception_to_raise), repr(ex)) raise exception_to_raise from ex except WebsocketError as ex: logger.info( 'A websocket error occurred while getting job status: %s', str(ex)) # Specific `WebsocketError` exceptions that are not worth retrying. if isinstance( ex, (WebsocketTimeoutError, WebsocketIBMQProtocolError)): logger.info( 'The websocket error that occurred could not ' 'be retried: %s', str(ex)) raise ex # Check whether the websocket error should be retried. current_retry_attempt = current_retry_attempt + 1 if (current_retry_attempt > retries) or (not attempt_retry): logger.info( 'Max retries exceeded: Failed to establish a websocket ' 'connection due to a network error.') raise ex # Sleep, and then `continue` with retrying. backoff_time = self._backoff_time(backoff_factor, current_retry_attempt) logger.info( 'Retrying get_job_status via websocket after %s seconds: ' 'Attempt #%s', backoff_time, current_retry_attempt) await asyncio.sleep( backoff_time) # Block asyncio loop for given backoff time. continue # Continues next iteration after `finally` block. finally: if websocket is not None: await websocket.close() # Execution should not reach here, sanity check. exception_message = 'Max retries exceeded: Failed to establish a websocket ' \ 'connection due to a network error.' logger.info(exception_message) raise WebsocketError(exception_message)