def test_run_process_should_record_that_all_items_failed_when_content_api_call_returns_error( self): dummy_http_error = errors.HttpError( mock.MagicMock(status=http.HTTPStatus.BAD_REQUEST, reason='Bad Request'), b'') self.mock_content_api_client.return_value.process_items.side_effect = dummy_http_error dummy_failures = [ failure.Failure(str(item.get('item_id', 'Missing ID')), dummy_http_error.resp.reason) for item in DUMMY_ROWS ] expected_result = process_result.ProcessResult([], dummy_failures, []) expected_batch_id = int(DUMMY_START_INDEX / DUMMY_BATCH_SIZE) + 1 self.mock_bq_client.from_service_account_json.return_value.load_items.return_value = DUMMY_ROWS self.test_client.post(INSERT_URL, data=DUMMY_REQUEST_BODY, headers={'X-AppEngine-TaskExecutionCount': '0'}) self.mock_recorder.from_service_account_json.return_value.insert_result.assert_called_once_with( constants.Operation.UPSERT.value, expected_result, DUMMY_TIMESTAMP, expected_batch_id, )
def test_add_failure(self): result = process_result.ProcessResult([], [], []) result.add_content_api_failure(failure.Failure('0001', 'Error msg')) self.assertEqual(1, result.get_failure_count()) self.assertEqual('0001', result.content_api_failures[-1].item_id) self.assertEqual('Error msg', result.content_api_failures[-1].error_msg)
def test_get_counts_str(self): result = process_result.ProcessResult( ['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')], ['0005', '0006']) count_str = result.get_counts_str() self.assertEqual('Success: 3, Failure: 1, Skipped: 2.', count_str)
def test_counts(self): result = process_result.ProcessResult( ['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')], ['0005', '0006']) self.assertEqual(3, result.get_success_count()) self.assertEqual(1, result.get_failure_count()) self.assertEqual(2, result.get_skipped_count())
def test_insert_count_result_with_correct_types(self): operation = 'insert' timestamp = '000101010100' batch_id = 0 result = process_result.ProcessResult( ['0001'], [failure.Failure('0002', 'Error message')], ['0003']) self.recorder.insert_result(operation, result, timestamp, batch_id) self.client.insert_rows.assert_called()
def test_get_ids_str(self): result = process_result.ProcessResult( ['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')], ['0005', '0006']) ids_str = result.get_ids_str() self.assertEqual( 'Success: 0001, 0002, 0003, Failure: [\'ID: 0004, Error: Error message, \'], Skipped: 0005, 0006.', ids_str)
def test_run_process_should_record_result_when_content_api_call_returns_ok( self): expected_batch_id = int(DUMMY_START_INDEX / DUMMY_BATCH_SIZE) + 1 expected_result = process_result.ProcessResult(DUMMY_SUCCESSES, DUMMY_FAILURES, []) self.test_client.post(INSERT_URL, data=DUMMY_REQUEST_BODY, headers={'X-AppEngine-TaskExecutionCount': '0'}) self.mock_recorder.from_service_account_json.return_value.insert_result.assert_called_once_with( constants.Operation.UPSERT.value, expected_result, DUMMY_TIMESTAMP, expected_batch_id, )
def _handle_content_api_error( error_status_code: int, error_reason: str, batch_num: int, error: Exception, item_rows: List[bigquery.Row], operation: constants.Operation, task: upload_task.UploadTask) -> process_result.ProcessResult: """Logs network related errors returned from Content API and returns a list of item failures. Args: error_status_code: HTTP status code from Content API. error_reason: The reason for the error. batch_num: The batch number. error: The error thrown by Content API. item_rows: The items being processed in this batch. operation: The operation to be performed on this batch of items. task: The Cloud Task object that initiated this request. Returns: The list of items that failed due to the error, wrapped in a process_result. """ logging.warning( 'Batch #%d with operation %s and initiation timestamp %s failed. HTTP status: %s. Error: %s', batch_num, operation.value, task.timestamp, error_status_code, error_reason) # If the batch API call received an HttpError, mark every id as failed. item_failures = [ failure.Failure(str(item_row.get('item_id', 'Missing ID')), error_reason) for item_row in item_rows ] api_result = process_result.ProcessResult([], item_failures, []) if content_api_client.suggest_retry( error_status_code) and _get_execution_attempt() < TASK_RETRY_LIMIT: logging.warning( 'Batch #%d with operation %s and initiation timestamp %s will be requeued for retry', batch_num, operation.value, task.timestamp) else: logging.error( 'Batch #%d with operation %s and initiation timestamp %s failed and will not be retried. Error: %s', batch_num, operation.value, task.timestamp, error) return api_result
def _run_process( operation: constants.Operation) -> Tuple[str, http.HTTPStatus]: """Handles tasks pushed from Task Queue. When tasks are enqueued to Task Queue by initiator, this method will be called. It extracts necessary information from a Task Queue message. The following processes are executed in this function: - Loading items to process from BigQuery. - Converts items into a batch that can be sent to Content API for Shopping. - Sending items to Content API for Shopping (Merchant Center). - Records the results of the Content API for Shopping call. Args: operation: Type of operation to perform on the items. Returns: The result of HTTP request. """ request_body = json.loads(flask.request.data.decode('utf-8')) task = upload_task.UploadTask.from_json(request_body) if task.batch_size == 0: return 'OK', http.HTTPStatus.OK batch_number = int(task.start_index / task.batch_size) + 1 logging.info( '%s started. Batch #%d info: start_index: %d, batch_size: %d,' 'initiation timestamp: %s', operation.value, batch_number, task.start_index, task.batch_size, task.timestamp) try: items = _load_items_from_bigquery(operation, task) except errors.HttpError: return 'Error loading items from BigQuery', http.HTTPStatus.INTERNAL_SERVER_ERROR result = process_result.ProcessResult([], [], []) try: if not items: logging.error( 'Batch #%d, operation %s: 0 items loaded from BigQuery so batch not sent to Content API. Start_index: %d, batch_size: %d,' 'initiation timestamp: %s', batch_number, operation.value, task.start_index, task.batch_size, task.timestamp) return 'No items to process', http.HTTPStatus.OK method = OPERATION_TO_METHOD.get(operation) # Creates batch from items loaded from BigQuery original_batch, skipped_item_ids, batch_id_to_item_id = batch_creator.create_batch( batch_number, items, method) # Optimizes batch via Shoptimizer for upsert/prevent_expiring operations if operation != constants.Operation.DELETE and constants.SHOPTIMIZER_API_INTEGRATION_ON: batch_to_send_to_content_api = _create_optimized_batch( original_batch, batch_number, operation) else: batch_to_send_to_content_api = original_batch # Sends batch of items to Content API for Shopping api_client = content_api_client.ContentApiClient() successful_item_ids, item_failures = api_client.process_items( batch_to_send_to_content_api, batch_number, batch_id_to_item_id, method) result = process_result.ProcessResult( successfully_processed_item_ids=successful_item_ids, content_api_failures=item_failures, skipped_item_ids=skipped_item_ids) except errors.HttpError as http_error: error_status_code = http_error.resp.status error_reason = http_error.resp.reason result = _handle_content_api_error(error_status_code, error_reason, batch_number, http_error, items, operation, task) return error_reason, error_status_code except socket.timeout as timeout_error: error_status_code = http.HTTPStatus.REQUEST_TIMEOUT error_reason = 'Socket timeout' result = _handle_content_api_error(error_status_code, error_reason, batch_number, timeout_error, items, operation, task) return error_reason, error_status_code else: logging.info( 'Batch #%d with operation %s and initiation timestamp %s successfully processed %s items, failed to process %s items and skipped %s items.', batch_number, operation.value, task.timestamp, result.get_success_count(), result.get_failure_count(), result.get_skipped_count()) finally: recorder = result_recorder.ResultRecorder.from_service_account_json( constants.GCP_SERVICE_ACCOUNT_PATH, constants.DATASET_ID_FOR_MONITORING, constants.TABLE_ID_FOR_RESULT_COUNTS_MONITORING, constants.TABLE_ID_FOR_ITEM_RESULTS_MONITORING) recorder.insert_result(operation.value, result, task.timestamp, batch_number) return 'OK', http.HTTPStatus.OK
def test_add_skipped(self): result = process_result.ProcessResult([], [], []) result.add_skipped_item_id('0001') self.assertEqual(1, result.get_skipped_count()) self.assertEqual('0001', result.skipped_item_ids[-1])