def test_validation_row_too_short(patch_storage, patch_rabbit, patch_db_helper,
tmp_path):
# Given
mock_processor = setup_mock_processor({'COL_1': [], 'COL_2': []}, None)
bulk_processor = BulkProcessor(mock_processor)
bulk_processor.working_dir = tmp_path
mock_blob = Mock()
mock_blob.name = 'mock_blob_name'
patch_storage.Client.return_value.list_blobs.return_value = [mock_blob]
# Mock data misses the 2nd column in it's row entirely
patch_storage.Client.return_value.download_blob_to_file.side_effect = partial(
mock_download_blob, mock_data=b'COL_1,COL_2\n'
b'col_1_value')
# When
bulk_processor.run()
# Then
mock_upload_to_bucket = patch_storage.Client.return_value.bucket.return_value.blob.return_value \
.upload_from_filename
mock_upload_calls = mock_upload_to_bucket.call_args_list
assert len(
mock_upload_calls) == 2, 'Upload to bucket should be called twice'
assert call(str(
tmp_path.joinpath('ERROR_mock_blob_name'))) in mock_upload_calls
assert call(str(
tmp_path.joinpath('ERROR_DETAIL_mock_blob_name'))) in mock_upload_calls
patch_rabbit.return_value.__enter__.return_value.publish_message.assert_not_called(
)
patch_db_helper.connect_to_read_replica_pool.assert_called_once()
assert_no_left_over_files(tmp_path)
def test_run_validation_successful(patch_storage, patch_rabbit,
patch_db_helper, tmp_path):
# Given
test_message = {"test_message": "blah"}
mock_processor = setup_mock_processor({
'header_1': [],
'header_2': []
}, test_message)
bulk_processor = BulkProcessor(mock_processor)
bulk_processor.working_dir = tmp_path
mock_blob = Mock()
mock_blob.name = 'mock_blob_name'
patch_storage.Client.return_value.list_blobs.return_value = [mock_blob]
patch_storage.Client.return_value.download_blob_to_file.side_effect = partial(
mock_download_blob,
mock_data=(b'header_1,header_2\n'
b'value1,value2\n'))
# When
bulk_processor.run()
# Then
mock_upload_to_bucket = patch_storage.Client.return_value.bucket.return_value.blob.return_value \
.upload_from_filename
mock_upload_to_bucket.assert_called_once_with(
str(tmp_path.joinpath('PROCESSED_mock_blob_name')))
patch_rabbit.return_value.__enter__.return_value.publish_message.assert_called_once_with(
message=json.dumps(test_message),
content_type='application/json',
headers=None,
exchange=mock_processor.exchange,
routing_key=mock_processor.routing_key)
patch_db_helper.connect_to_read_replica_pool.assert_called_once()
assert_no_left_over_files(tmp_path)
def test_run_success_failure_mix(patch_storage, patch_rabbit, patch_db_helper,
tmp_path):
# Given
test_message = {"test_message": "blah"}
mock_processor = setup_mock_processor({'header': [no_invalid_validator()]},
test_message)
bulk_processor = BulkProcessor(mock_processor)
bulk_processor.working_dir = tmp_path
mock_blob = Mock()
mock_blob.name = 'mock_blob_name'
patch_storage.Client.return_value.list_blobs.return_value = [mock_blob]
patch_storage.Client.return_value.download_blob_to_file.side_effect = partial(
mock_download_blob, mock_data=(b'header\n'
b'value\n'
b'invalid'))
# When
bulk_processor.run()
# Then
mock_upload_to_bucket = patch_storage.Client.return_value.bucket.return_value.blob.return_value. \
upload_from_filename
mock_upload_calls = mock_upload_to_bucket.call_args_list
assert len(
mock_upload_calls) == 3, 'Upload to bucket should be called twice'
assert call(str(
tmp_path.joinpath('PROCESSED_mock_blob_name'))) in mock_upload_calls
assert call(str(
tmp_path.joinpath('ERROR_mock_blob_name'))) in mock_upload_calls
assert call(str(
tmp_path.joinpath('ERROR_DETAIL_mock_blob_name'))) in mock_upload_calls
patch_rabbit.return_value.__enter__.return_value.publish_message.assert_called_once_with(
message=json.dumps(test_message),
content_type='application/json',
headers=None,
exchange=mock_processor.exchange,
routing_key=mock_processor.routing_key)
patch_db_helper.connect_to_read_replica_pool.assert_called_once()
assert_no_left_over_files(tmp_path)
