def _handle_response(
response: Dict[str, Any], batch_number: int,
batch_id_to_item_id: constants.BatchIdToItemId
) -> Tuple[List[str], List[failure.Failure]]:
"""Processes the response from an API call.
Args:
response: A response from Content API which includes product data. See
https://developers.google.com/shopping-content/v2/reference/v2/products#resource
batch_number: Identifier for this batch
batch_id_to_item_id: A dictionary that maps batch ids to items ids. Content
API responses will only return the batch id.
Returns:
A list of successful ids and failures parsed from the result of
the Content API call
"""
successful_item_ids = []
item_failures = []
response_kind = response.get('kind', '')
if response_kind != 'content#productsCustomBatchResponse':
logging.warning('Batch #%d: Invalid response format. Response: %s',
batch_number, response)
# If the batch as a whole returned an error, log every id in the batch as
# a failure.
item_failures = [
failure.Failure(item_id, 'API returned unexpected response.')
for item_id in batch_id_to_item_id.values()
]
return successful_item_ids, item_failures
entries = response.get('entries', [])
for entry in entries:
batch_id = entry.get('batchId')
item_id = batch_id_to_item_id.get(batch_id)
response_errors = entry.get('errors')
if response_errors:
error_msg = (
f'Code: '
f'{response_errors.get("code", "Error code not found")}.'
f' Message: '
f'{response_errors.get("message", "Error message not found.")}'
)
item_failures.append(failure.Failure(item_id, error_msg))
logging.info('Batch #%d: Errors for batch entry %d. Error: %s',
batch_number, batch_id, response_errors)
else:
successful_item_ids.append(item_id)
return successful_item_ids, item_failures
def test_run_process_should_record_that_all_items_failed_when_content_api_call_returns_error(
self):
dummy_http_error = errors.HttpError(
mock.MagicMock(status=http.HTTPStatus.BAD_REQUEST,
reason='Bad Request'), b'')
self.mock_content_api_client.return_value.process_items.side_effect = dummy_http_error
dummy_failures = [
failure.Failure(str(item.get('item_id', 'Missing ID')),
dummy_http_error.resp.reason)
for item in DUMMY_ROWS
]
expected_result = process_result.ProcessResult([], dummy_failures, [])
expected_batch_id = int(DUMMY_START_INDEX / DUMMY_BATCH_SIZE) + 1
self.mock_bq_client.from_service_account_json.return_value.load_items.return_value = DUMMY_ROWS
self.test_client.post(INSERT_URL,
data=DUMMY_REQUEST_BODY,
headers={'X-AppEngine-TaskExecutionCount': '0'})
self.mock_recorder.from_service_account_json.return_value.insert_result.assert_called_once_with(
constants.Operation.UPSERT.value,
expected_result,
DUMMY_TIMESTAMP,
expected_batch_id,
)
def test_add_failure(self):
result = process_result.ProcessResult([], [], [])
result.add_content_api_failure(failure.Failure('0001', 'Error msg'))
self.assertEqual(1, result.get_failure_count())
self.assertEqual('0001', result.content_api_failures[-1].item_id)
self.assertEqual('Error msg', result.content_api_failures[-1].error_msg)
def test_get_counts_str(self):
result = process_result.ProcessResult(
['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')],
['0005', '0006'])
count_str = result.get_counts_str()
self.assertEqual('Success: 3, Failure: 1, Skipped: 2.', count_str)
def test_counts(self):
result = process_result.ProcessResult(
['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')],
['0005', '0006'])
self.assertEqual(3, result.get_success_count())
self.assertEqual(1, result.get_failure_count())
self.assertEqual(2, result.get_skipped_count())
def test_insert_count_result_with_correct_types(self):
operation = 'insert'
timestamp = '000101010100'
batch_id = 0
result = process_result.ProcessResult(
['0001'], [failure.Failure('0002', 'Error message')], ['0003'])
self.recorder.insert_result(operation, result, timestamp, batch_id)
self.client.insert_rows.assert_called()
def test_get_ids_str(self):
result = process_result.ProcessResult(
['0001', '0002', '0003'], [failure.Failure('0004', 'Error message')],
['0005', '0006'])
ids_str = result.get_ids_str()
self.assertEqual(
'Success: 0001, 0002, 0003, Failure: [\'ID: 0004, Error: Error message, \'], Skipped: 0005, 0006.',
ids_str)
def _handle_content_api_error(
error_status_code: int, error_reason: str, batch_num: int,
error: Exception, item_rows: List[bigquery.Row],
operation: constants.Operation,
task: upload_task.UploadTask) -> process_result.ProcessResult:
"""Logs network related errors returned from Content API and returns a list of item failures.
Args:
error_status_code: HTTP status code from Content API.
error_reason: The reason for the error.
batch_num: The batch number.
error: The error thrown by Content API.
item_rows: The items being processed in this batch.
operation: The operation to be performed on this batch of items.
task: The Cloud Task object that initiated this request.
Returns:
The list of items that failed due to the error, wrapped in a
process_result.
"""
logging.warning(
'Batch #%d with operation %s and initiation timestamp %s failed. HTTP status: %s. Error: %s',
batch_num, operation.value, task.timestamp, error_status_code,
error_reason)
# If the batch API call received an HttpError, mark every id as failed.
item_failures = [
failure.Failure(str(item_row.get('item_id', 'Missing ID')),
error_reason) for item_row in item_rows
]
api_result = process_result.ProcessResult([], item_failures, [])
if content_api_client.suggest_retry(
error_status_code) and _get_execution_attempt() < TASK_RETRY_LIMIT:
logging.warning(
'Batch #%d with operation %s and initiation timestamp %s will be requeued for retry',
batch_num, operation.value, task.timestamp)
else:
logging.error(
'Batch #%d with operation %s and initiation timestamp %s failed and will not be retried. Error: %s',
batch_num, operation.value, task.timestamp, error)
return api_result
import constants
import main
from models import failure
from models import process_result
DUMMY_ROWS = [bigquery.Row(['0001'], {'item_id': 0})]
DUMMY_START_INDEX = 0
DUMMY_BATCH_SIZE = 1000
DUMMY_TIMESTAMP = '0001-01-01:00:00:00'
DUMMY_REQUEST_BODY = json.dumps({
'start_index': DUMMY_START_INDEX,
'batch_size': DUMMY_BATCH_SIZE,
'timestamp': DUMMY_TIMESTAMP,
})
DUMMY_SUCCESSES = ['0001', '0002', '0003']
DUMMY_FAILURES = [failure.Failure('0004', 'Error message')]
DUMMY_SKIPPED = ['0005']
INSERT_URL = '/insert_items'
DELETE_URL = '/delete_items'
PREVENT_EXPIRING_URL = '/prevent_expiring_items'
class MainTest(unittest.TestCase):
def setUp(self):
super(MainTest, self).setUp()
main.app.testing = True
self.test_client = main.app.test_client()
self.mock_bq_client = mock.patch('bigquery_client.BigQueryClient',
autospec=True).start()
self.mock_content_api_client = mock.patch(