def test_sorting_print_file(cleanup_test_files): complete_file_path = Path(shutil.copyfile(RESOURCE_FILE_PATH.joinpath('file_to_sort'), TestConfig.PARTIAL_FILES_DIRECTORY.joinpath('tmp_file_to_sort'))) context_logger = Mock() sorted_print_file = sort_print_file_if_required(complete_file_path, PackCode.D_CE1A_ICLCR1, ActionType.ICHHQE, context_logger) correct_file = RESOURCE_FILE_PATH.joinpath('correctly_sorted_file') assert sorted_print_file.read_text() == correct_file.read_text()
def test_create_manifest(patch_datetime, pack_code): # Given dummy_print_file = RESOURCE_FILE_PATH.joinpath("dummy_print_file.txt") row_count = 100 patched_date = datetime.utcnow() patch_datetime.utcnow.return_value = patched_date expected_manifest = { 'schemaVersion': '1', 'description': PACK_CODE_TO_DESCRIPTION[pack_code], 'dataset': PACK_CODE_TO_DATASET[pack_code].value, 'version': '1', 'manifestCreated': patched_date.isoformat(timespec='milliseconds') + 'Z', 'sourceName': 'ONS_RM', 'files': [{ 'name': dummy_print_file.name, 'relativePath': './', 'sizeBytes': '19', # Calculated size of the dummy print file 'md5sum': '79cad6cda5ebe6b9bdbdbb6a56587e28', # Calculated md5 of the dummy print file 'rows': row_count }] } # When actual_manifest = create_manifest(dummy_print_file, pack_code, row_count) assert expected_manifest == actual_manifest
def test_create_manifest(patch_datetime): # Given dummy_print_file = RESOURCE_FILE_PATH.joinpath("dummy_print_file.txt") row_count = 100 pack_code = 'EXAMPLE_LETTER' supplier = 'SUPPLIER_A' patched_date = datetime.utcnow() patch_datetime.utcnow.return_value = patched_date expected_manifest = { 'packCode': pack_code, 'supplier': supplier, 'manifestCreated': patched_date.isoformat(timespec='milliseconds') + 'Z', 'sourceName': 'ONS_RM', 'files': [{ 'name': dummy_print_file.name, 'relativePath': './', 'sizeBytes': '19', # Calculated size of the dummy print file 'md5sum': '79cad6cda5ebe6b9bdbdbb6a56587e28', # Calculated md5 of the dummy print file 'rows': row_count }] } # When actual_manifest = create_manifest(dummy_print_file, pack_code, supplier, row_count) # Then assert expected_manifest == actual_manifest
def test_check_partial_files_processes_complete_file(cleanup_test_files): # Given partial_file_path = Path( cleanup_test_files.partial_files.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.1')) partial_file_path.touch() mock_storage_client = Mock() # When with patch('app.file_sender.sftp.paramiko.SSHClient') as client: client.return_value.open_sftp.return_value = mock_storage_client # mock the sftp client connection mock_storage_client.stat.return_value.st_mode = paramiko.sftp_client.stat.S_IFDIR # mock directory exists check_partial_files(cleanup_test_files.partial_files) client.assert_not_called() Path( shutil.copyfile( RESOURCE_FILE_PATH.joinpath('SUPPLIER_A.EXAMPLE_LETTER.1.1'), cleanup_test_files.partial_files.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.1'))) check_partial_files(cleanup_test_files.partial_files) # Then client.assert_called_once()
def test_processing_complete_file_uploads_correct_files(cleanup_test_files): complete_file_path = Path( shutil.copyfile( RESOURCE_FILE_PATH.joinpath('SUPPLIER_A.EXAMPLE_LETTER.1.1'), TestConfig.PARTIAL_FILES_DIRECTORY.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.1'))) context_logger = Mock() with patch('app.file_sender.sftp.SftpUtility') as patched_sftp, patch( 'app.file_sender.datetime') as patch_datetime: mock_time = datetime(2019, 1, 1, 7, 6, 5) patch_datetime.utcnow.return_value = mock_time process_complete_file(complete_file_path, 'EXAMPLE_LETTER', 'SUPPLIER_A', context_logger) put_sftp_call_kwargs = [ kwargs for _, kwargs in patched_sftp.return_value.__enter__. return_value.put_file.call_args_list ] iso_mocked = mock_time.strftime("%Y-%m-%dT%H-%M-%S") assert put_sftp_call_kwargs[0]['local_path'] == str( cleanup_test_files.encrypted_files.joinpath( f'EXAMPLE_LETTER_{iso_mocked}.csv.gpg')) assert put_sftp_call_kwargs[0][ 'filename'] == f'EXAMPLE_LETTER_{iso_mocked}.csv.gpg' assert put_sftp_call_kwargs[1]['local_path'] == str( cleanup_test_files.encrypted_files.joinpath( f'EXAMPLE_LETTER_{iso_mocked}.manifest')) assert put_sftp_call_kwargs[1][ 'filename'] == f'EXAMPLE_LETTER_{iso_mocked}.manifest'
def test_failed_encrypted_files_and_manifests_are_deleted(cleanup_test_files): # Given complete_file_path = Path( shutil.copyfile( RESOURCE_FILE_PATH.joinpath('SUPPLIER_A.EXAMPLE_LETTER.1.1'), TestConfig.PARTIAL_FILES_DIRECTORY.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.1'))) context_logger = Mock() sftp_failure_exception_message = 'Simulate SFTP transfer failure' def simulate_sftp_failure(*_args, **_kwargs): raise Exception(sftp_failure_exception_message) with patch('app.file_sender.sftp.paramiko.SSHClient') as client, \ pytest.raises(Exception) as raised_exception: client.return_value.open_sftp.side_effect = simulate_sftp_failure # When process_complete_file(complete_file_path, 'EXAMPLE_LETTER', 'SUPPLIER_A', context_logger) if not str(raised_exception.value) == sftp_failure_exception_message: raise raised_exception.value # Then # Check encrypted_files_directory is empty assert not any(cleanup_test_files.encrypted_files.iterdir()) # Check complete partial file is still there assert complete_file_path.exists() # Check original exception is re-raised assert str(raised_exception.value) == sftp_failure_exception_message
def test_packcode_for_not_sorting_not_sorted(cleanup_test_files): from_file = RESOURCE_FILE_PATH.joinpath('file_to_sort') to_file = TestConfig.PARTIAL_FILES_DIRECTORY.joinpath('tmp_file_to_sort') complete_file_path = Path(shutil.copyfile(from_file, to_file)) unsorted_print_file = sort_print_file_if_required(complete_file_path, PackCode.P_IC_ICL1, ActionType.ICHHQE, None) assert unsorted_print_file.read_text() == from_file.read_text()
def test_check_partial_has_no_duplicates_without_duplicates( cleanup_test_files): # Given partial_duplicate_path = RESOURCE_FILE_PATH.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.2') # When result = check_partial_has_no_duplicates(partial_duplicate_path, Mock()) # Then assert result
def test_failing_write_to_gcp_bucket_is_handled(): # When with patch('app.file_sender.storage.Client') as bucket_client: # Simulate an error from the GCS storage client bucket_client.side_effect = exceptions.GoogleCloudError( "bucket doesn't exist") try: write_file_to_bucket( RESOURCE_FILE_PATH.joinpath('dummy_print_file.txt')) except Exception: assert False, "Exception msgs from writing to GCP bucket should be handled"
def test_check_partial_has_no_duplicates_with_duplicates(cleanup_test_files): # Given partial_duplicate_path = RESOURCE_FILE_PATH.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.2.duplicate_row') mock_logger = Mock() # When result = check_partial_has_no_duplicates(partial_duplicate_path, mock_logger) # Then assert not result, 'Check should return False for file with duplicates' mock_logger.error.assert_called_once_with( 'Duplicate row found in print file', line_number=2)
def test_local_files_are_deleted_after_upload(cleanup_test_files): complete_file_path = Path( shutil.copyfile( RESOURCE_FILE_PATH.joinpath('SUPPLIER_A.EXAMPLE_LETTER.1.1'), TestConfig.PARTIAL_FILES_DIRECTORY.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.1'))) context_logger = Mock() with patch('app.file_sender.sftp.SftpUtility'): process_complete_file(complete_file_path, 'EXAMPLE_LETTER', 'SUPPLIER_A', context_logger) with pytest.raises(StopIteration): next(TestConfig.PARTIAL_FILES_DIRECTORY.iterdir()) with pytest.raises(StopIteration): next(TestConfig.ENCRYPTED_FILES_DIRECTORY.iterdir())
def test_generating_manifest_file(cleanup_test_files): manifest_file = cleanup_test_files.encrypted_files.joinpath( 'EXAMPLE_LETTER_2019-07-05T08-15-41.manifest') print_file = RESOURCE_FILE_PATH.joinpath( 'EXAMPLE_LETTER_2021-06-01T08-15-41.csv.gpg') generate_manifest_file(manifest_file, print_file, 'EXAMPLE_LETTER', 'SUPPLIER_A', row_count=10) manifest_json = json.loads(manifest_file.read_text()) assert manifest_json['sourceName'] == 'ONS_RM' assert manifest_json['packCode'] == 'EXAMPLE_LETTER' assert manifest_json['supplier'] == 'SUPPLIER_A' assert manifest_json['files'][0]['rows'] == 10
def test_quarantine_partial_file(cleanup_test_files): # Given partial_print_file = Path( shutil.copyfile( RESOURCE_FILE_PATH.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.2.duplicate_row'), TestConfig.PARTIAL_FILES_DIRECTORY.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.2'))) partial_print_file_text = partial_print_file.read_text() expected_destination = Path( TestConfig.QUARANTINED_FILES_DIRECTORY.joinpath( 'SUPPLIER_A.EXAMPLE_LETTER.1.2')) # When quarantine_partial_file(partial_print_file) # Then assert not partial_print_file.exists() assert expected_destination.exists() assert expected_destination.read_text() == partial_print_file_text