def test_run_process_should_record_that_all_items_failed_when_content_api_call_returns_error(
self):
dummy_http_error = errors.HttpError(
mock.MagicMock(status=http.HTTPStatus.BAD_REQUEST,
reason='Bad Request'), b'')
self.mock_content_api_client.return_value.process_items.side_effect = dummy_http_error
dummy_failures = [
failure.Failure(str(item.get('item_id', 'Missing ID')),
dummy_http_error.resp.reason)
for item in DUMMY_ROWS
]
expected_result = process_result.ProcessResult([], dummy_failures, [])
expected_batch_id = int(DUMMY_START_INDEX / DUMMY_BATCH_SIZE) + 1
self.mock_bq_client.from_service_account_json.return_value.load_items.return_value = DUMMY_ROWS
self.test_client.post(INSERT_URL,
data=DUMMY_REQUEST_BODY,
headers={'X-AppEngine-TaskExecutionCount': '0'})
self.mock_recorder.from_service_account_json.return_value.insert_result.assert_called_once_with(
constants.Operation.UPSERT.value,
expected_result,
DUMMY_TIMESTAMP,
expected_batch_id,
)
def test_add_failure(self):
result = process_result.ProcessResult([], [], [])
result.add_content_api_failure(failure.Failure('0001', 'Error msg'))
self.assertEqual(1, result.get_failure_count())
self.assertEqual('0001', result.content_api_failures[-1].item_id)
self.assertEqual('Error msg', result.content_api_failures[-1].error_msg)
def test_get_counts_str(self):
result = process_result.ProcessResult(
['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')],
['0005', '0006'])
count_str = result.get_counts_str()
self.assertEqual('Success: 3, Failure: 1, Skipped: 2.', count_str)
def test_counts(self):
result = process_result.ProcessResult(
['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')],
['0005', '0006'])
self.assertEqual(3, result.get_success_count())
self.assertEqual(1, result.get_failure_count())
self.assertEqual(2, result.get_skipped_count())
def test_insert_count_result_with_correct_types(self):
operation = 'insert'
timestamp = '000101010100'
batch_id = 0
result = process_result.ProcessResult(
['0001'], [failure.Failure('0002', 'Error message')], ['0003'])
self.recorder.insert_result(operation, result, timestamp, batch_id)
self.client.insert_rows.assert_called()
def test_get_ids_str(self):
result = process_result.ProcessResult(
['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')],
['0005', '0006'])
ids_str = result.get_ids_str()
self.assertEqual(
'Success: 0001, 0002, 0003, Failure: [\'ID: 0004, Error: Error message, \'], Skipped: 0005, 0006.',
ids_str)
def test_run_process_should_record_result_when_content_api_call_returns_ok(
self):
expected_batch_id = int(DUMMY_START_INDEX / DUMMY_BATCH_SIZE) + 1
expected_result = process_result.ProcessResult(DUMMY_SUCCESSES,
DUMMY_FAILURES, [])
self.test_client.post(INSERT_URL,
data=DUMMY_REQUEST_BODY,
headers={'X-AppEngine-TaskExecutionCount': '0'})
self.mock_recorder.from_service_account_json.return_value.insert_result.assert_called_once_with(
constants.Operation.UPSERT.value,
expected_result,
DUMMY_TIMESTAMP,
expected_batch_id,
)
def _handle_content_api_error(
error_status_code: int, error_reason: str, batch_num: int,
error: Exception, item_rows: List[bigquery.Row],
operation: constants.Operation,
task: upload_task.UploadTask) -> process_result.ProcessResult:
"""Logs network related errors returned from Content API and returns a list of item failures.
Args:
error_status_code: HTTP status code from Content API.
error_reason: The reason for the error.
batch_num: The batch number.
error: The error thrown by Content API.
item_rows: The items being processed in this batch.
operation: The operation to be performed on this batch of items.
task: The Cloud Task object that initiated this request.
Returns:
The list of items that failed due to the error, wrapped in a
process_result.
"""
logging.warning(
'Batch #%d with operation %s and initiation timestamp %s failed. HTTP status: %s. Error: %s',
batch_num, operation.value, task.timestamp, error_status_code,
error_reason)
# If the batch API call received an HttpError, mark every id as failed.
item_failures = [
failure.Failure(str(item_row.get('item_id', 'Missing ID')),
error_reason) for item_row in item_rows
]
api_result = process_result.ProcessResult([], item_failures, [])
if content_api_client.suggest_retry(
error_status_code) and _get_execution_attempt() < TASK_RETRY_LIMIT:
logging.warning(
'Batch #%d with operation %s and initiation timestamp %s will be requeued for retry',
batch_num, operation.value, task.timestamp)
else:
logging.error(
'Batch #%d with operation %s and initiation timestamp %s failed and will not be retried. Error: %s',
batch_num, operation.value, task.timestamp, error)
return api_result
def _run_process(
operation: constants.Operation) -> Tuple[str, http.HTTPStatus]:
"""Handles tasks pushed from Task Queue.
When tasks are enqueued to Task Queue by initiator, this method will be
called. It extracts necessary information from a Task Queue message. The
following processes are executed in this function:
- Loading items to process from BigQuery.
- Converts items into a batch that can be sent to Content API for Shopping.
- Sending items to Content API for Shopping (Merchant Center).
- Records the results of the Content API for Shopping call.
Args:
operation: Type of operation to perform on the items.
Returns:
The result of HTTP request.
"""
request_body = json.loads(flask.request.data.decode('utf-8'))
task = upload_task.UploadTask.from_json(request_body)
if task.batch_size == 0:
return 'OK', http.HTTPStatus.OK
batch_number = int(task.start_index / task.batch_size) + 1
logging.info(
'%s started. Batch #%d info: start_index: %d, batch_size: %d,'
'initiation timestamp: %s', operation.value, batch_number,
task.start_index, task.batch_size, task.timestamp)
try:
items = _load_items_from_bigquery(operation, task)
except errors.HttpError:
return 'Error loading items from BigQuery', http.HTTPStatus.INTERNAL_SERVER_ERROR
result = process_result.ProcessResult([], [], [])
try:
if not items:
logging.error(
'Batch #%d, operation %s: 0 items loaded from BigQuery so batch not sent to Content API. Start_index: %d, batch_size: %d,'
'initiation timestamp: %s', batch_number, operation.value,
task.start_index, task.batch_size, task.timestamp)
return 'No items to process', http.HTTPStatus.OK
method = OPERATION_TO_METHOD.get(operation)
# Creates batch from items loaded from BigQuery
original_batch, skipped_item_ids, batch_id_to_item_id = batch_creator.create_batch(
batch_number, items, method)
# Optimizes batch via Shoptimizer for upsert/prevent_expiring operations
if operation != constants.Operation.DELETE and constants.SHOPTIMIZER_API_INTEGRATION_ON:
batch_to_send_to_content_api = _create_optimized_batch(
original_batch, batch_number, operation)
else:
batch_to_send_to_content_api = original_batch
# Sends batch of items to Content API for Shopping
api_client = content_api_client.ContentApiClient()
successful_item_ids, item_failures = api_client.process_items(
batch_to_send_to_content_api, batch_number, batch_id_to_item_id,
method)
result = process_result.ProcessResult(
successfully_processed_item_ids=successful_item_ids,
content_api_failures=item_failures,
skipped_item_ids=skipped_item_ids)
except errors.HttpError as http_error:
error_status_code = http_error.resp.status
error_reason = http_error.resp.reason
result = _handle_content_api_error(error_status_code, error_reason,
batch_number, http_error, items,
operation, task)
return error_reason, error_status_code
except socket.timeout as timeout_error:
error_status_code = http.HTTPStatus.REQUEST_TIMEOUT
error_reason = 'Socket timeout'
result = _handle_content_api_error(error_status_code, error_reason,
batch_number, timeout_error, items,
operation, task)
return error_reason, error_status_code
else:
logging.info(
'Batch #%d with operation %s and initiation timestamp %s successfully processed %s items, failed to process %s items and skipped %s items.',
batch_number, operation.value, task.timestamp,
result.get_success_count(), result.get_failure_count(),
result.get_skipped_count())
finally:
recorder = result_recorder.ResultRecorder.from_service_account_json(
constants.GCP_SERVICE_ACCOUNT_PATH,
constants.DATASET_ID_FOR_MONITORING,
constants.TABLE_ID_FOR_RESULT_COUNTS_MONITORING,
constants.TABLE_ID_FOR_ITEM_RESULTS_MONITORING)
recorder.insert_result(operation.value, result, task.timestamp,
batch_number)
return 'OK', http.HTTPStatus.OK
def test_add_skipped(self):
result = process_result.ProcessResult([], [], [])
result.add_skipped_item_id('0001')
self.assertEqual(1, result.get_skipped_count())
self.assertEqual('0001', result.skipped_item_ids[-1])