def test__transfer_single_file_fails(mocked_s3, disable_backoff):
"""Should return False if unable to transfer."""
mocked_s3[0].s3fs.touch(
"DH-PLAYPEN/storage/input/2020-01-01/collection_name.csv.gz")
destination_client = mocked_s3[1]
destination_client._S3FileSystem__base_path = "BUCKET-DOESNT-EXIST"
with pytest.raises(transfer.TransferFailed):
transfer._transfer_single_file("2020-01-01/collection_name.csv.gz",
mocked_s3, 1, 1)
def test__transfer_single_file_unable_to_verify(mocked_s3, disable_backoff,
mocker):
"""Should return False on verification failure."""
mocked_s3[0].s3fs.touch(
"DH-PLAYPEN/storage/input/2020-01-01/collection_name.csv.gz")
mocker.patch("solgate.transfer.verify", return_value=False)
with pytest.raises(transfer.TransferFailed):
transfer._transfer_single_file("2020-01-01/collection_name.csv.gz",
mocked_s3, 1, 1)
def test__transfer_single_file_same_client(mocked_s3):
"""Should transfer faster between the same clients."""
mocked_s3[0].s3c.put_object(Bucket='BUCKET', Key='a/b.csv', Body='foo')
files = ["a/b.csv", "a-copy/b.csv"]
assert transfer._transfer_single_file("a/b.csv", mocked_s3, 1, 1) is None
assert all([mocked_s3[idx].info(f) for idx, f in enumerate(files)])
def test__transfer_single_file_same_client(mocked_s3):
"""Should transfer faster between the same clients."""
mocked_s3[0].s3fs.touch("BUCKET/a/b.csv")
files = ["a/b.csv", "a-copy/b.csv"]
assert transfer._transfer_single_file("a/b.csv", mocked_s3) is None
assert all([mocked_s3[idx].info(f) for idx, f in enumerate(files)])
def test__transfer_single_file_unable_to_verify(mocked_s3, mocker):
"""Should return False on verification failure."""
mocked_s3[0].s3fs.touch(
"DH-PLAYPEN/storage/input/2020-01-01/collection_name.csv.gz")
mocker.patch("solgate.transfer.verify", return_value=False)
assert transfer._transfer_single_file("2020-01-01/collection_name.csv.gz",
mocked_s3) is False
def test__transfer_single_file_fails(mocked_s3):
"""Should return False if unable to transfer."""
mocked_s3[0].s3fs.touch(
"DH-PLAYPEN/storage/input/2020-01-01/collection_name.csv.gz")
destination_client = mocked_s3[1]
destination_client._S3FileSystem__base_path = "BUCKET-DOESNT-EXIST"
assert transfer._transfer_single_file("2020-01-01/collection_name.csv.gz",
mocked_s3) is False
def test__transfer_single_file(mocked_s3):
"""Should transfer and verify file."""
mocked_s3[0].s3fs.touch(
"DH-PLAYPEN/storage/input/2020-01-01/collection_name.csv.gz")
files = [
"2020-01-01/collection_name.csv.gz",
"collection_name/historic/2020-01-01-collection_name.csv",
"collection_name/latest/full_data.csv",
"2020-01-01/collection_name.csv.gz",
]
assert transfer._transfer_single_file("2020-01-01/collection_name.csv.gz",
mocked_s3, 1, 1) is None
assert all([mocked_s3[idx].info(f) for idx, f in enumerate(files)])
def test__transfer_single_file_dry_run(mocked_s3, mocker):
"""Should transfer and verify file."""
mocked_s3[0].s3fs.touch(
"DH-PLAYPEN/storage/input/2020-01-01/collection_name.csv.gz")
mocked_copy = mocker.patch("solgate.transfer.copy")
files = [
"collection_name/historic/2020-01-01-collection_name.csv",
"collection_name/latest/full_data.csv",
"2020-01-01/collection_name.csv.gz",
]
assert transfer._transfer_single_file("2020-01-01/collection_name.csv.gz",
mocked_s3, 1, 1, True) is None
mocked_copy.assert_not_called()
for idx, f in enumerate(files):
with pytest.raises(FileNotFoundError):
mocked_s3[idx + 1].info(f)