def test_validation_row_too_short(patch_storage, patch_rabbit, patch_db_helper, tmp_path): # Given mock_processor = setup_mock_processor({'COL_1': [], 'COL_2': []}, None) bulk_processor = BulkProcessor(mock_processor) bulk_processor.working_dir = tmp_path mock_blob = Mock() mock_blob.name = 'mock_blob_name' patch_storage.Client.return_value.list_blobs.return_value = [mock_blob] # Mock data misses the 2nd column in it's row entirely patch_storage.Client.return_value.download_blob_to_file.side_effect = partial( mock_download_blob, mock_data=b'COL_1,COL_2\n' b'col_1_value') # When bulk_processor.run() # Then mock_upload_to_bucket = patch_storage.Client.return_value.bucket.return_value.blob.return_value \ .upload_from_filename mock_upload_calls = mock_upload_to_bucket.call_args_list assert len( mock_upload_calls) == 2, 'Upload to bucket should be called twice' assert call(str( tmp_path.joinpath('ERROR_mock_blob_name'))) in mock_upload_calls assert call(str( tmp_path.joinpath('ERROR_DETAIL_mock_blob_name'))) in mock_upload_calls patch_rabbit.return_value.__enter__.return_value.publish_message.assert_not_called( ) patch_db_helper.connect_to_read_replica_pool.assert_called_once() assert_no_left_over_files(tmp_path)
def test_run_validation_successful(patch_storage, patch_rabbit, patch_db_helper, tmp_path): # Given test_message = {"test_message": "blah"} mock_processor = setup_mock_processor({ 'header_1': [], 'header_2': [] }, test_message) bulk_processor = BulkProcessor(mock_processor) bulk_processor.working_dir = tmp_path mock_blob = Mock() mock_blob.name = 'mock_blob_name' patch_storage.Client.return_value.list_blobs.return_value = [mock_blob] patch_storage.Client.return_value.download_blob_to_file.side_effect = partial( mock_download_blob, mock_data=(b'header_1,header_2\n' b'value1,value2\n')) # When bulk_processor.run() # Then mock_upload_to_bucket = patch_storage.Client.return_value.bucket.return_value.blob.return_value \ .upload_from_filename mock_upload_to_bucket.assert_called_once_with( str(tmp_path.joinpath('PROCESSED_mock_blob_name'))) patch_rabbit.return_value.__enter__.return_value.publish_message.assert_called_once_with( message=json.dumps(test_message), content_type='application/json', headers=None, exchange=mock_processor.exchange, routing_key=mock_processor.routing_key) patch_db_helper.connect_to_read_replica_pool.assert_called_once() assert_no_left_over_files(tmp_path)
def test_run_success_failure_mix(patch_storage, patch_rabbit, patch_db_helper, tmp_path): # Given test_message = {"test_message": "blah"} mock_processor = setup_mock_processor({'header': [no_invalid_validator()]}, test_message) bulk_processor = BulkProcessor(mock_processor) bulk_processor.working_dir = tmp_path mock_blob = Mock() mock_blob.name = 'mock_blob_name' patch_storage.Client.return_value.list_blobs.return_value = [mock_blob] patch_storage.Client.return_value.download_blob_to_file.side_effect = partial( mock_download_blob, mock_data=(b'header\n' b'value\n' b'invalid')) # When bulk_processor.run() # Then mock_upload_to_bucket = patch_storage.Client.return_value.bucket.return_value.blob.return_value. \ upload_from_filename mock_upload_calls = mock_upload_to_bucket.call_args_list assert len( mock_upload_calls) == 3, 'Upload to bucket should be called twice' assert call(str( tmp_path.joinpath('PROCESSED_mock_blob_name'))) in mock_upload_calls assert call(str( tmp_path.joinpath('ERROR_mock_blob_name'))) in mock_upload_calls assert call(str( tmp_path.joinpath('ERROR_DETAIL_mock_blob_name'))) in mock_upload_calls patch_rabbit.return_value.__enter__.return_value.publish_message.assert_called_once_with( message=json.dumps(test_message), content_type='application/json', headers=None, exchange=mock_processor.exchange, routing_key=mock_processor.routing_key) patch_db_helper.connect_to_read_replica_pool.assert_called_once() assert_no_left_over_files(tmp_path)