def test_list_source_invalid_config(mocker):
"""Should raise when client can't be instantiated."""
mocker.patch("solgate.lookup.read_general_config", return_value=dict())
mocker.patch("solgate.lookup.S3FileSystem.from_config_file",
side_effect=EnvironmentError)
with pytest.raises(SystemExit):
lookup.list_source()
def test_list_source_no_objects(mocked_s3, mocker):
"""Should raise when no files found."""
fs = mocked_s3[0]
mocker.patch("solgate.lookup.read_general_config", return_value=dict())
mocker.patch("solgate.lookup.S3FileSystem.from_config_file",
return_value=[fs])
fs.s3fs.touch("BUCKET/old.csv")
s3_backend.buckets["BUCKET"].keys["old.csv"].last_modified = datetime(
2020, 1, 1)
with pytest.raises(SystemExit):
lookup.list_source()
def test_list_source_backfill(mocked_s3, mocker):
"""Should list all files when backfill is enabled."""
fs = mocked_s3[0]
mocker.patch("solgate.lookup.read_general_config", return_value=dict())
mocker.patch("solgate.lookup.S3FileSystem.from_config_file",
return_value=[fs])
fs.s3fs.touch("BUCKET/old.csv")
s3_backend.buckets["BUCKET"].keys["old.csv"].last_modified = datetime(
2020, 1, 1)
assert len(list(lookup.list_source({}, True))) == 1
def test_list_source(mocked_s3, mocker, config, old_object_modified_date,
found_objects):
"""Should list correct amount of files."""
fs = mocked_s3[0]
mocker.patch("solgate.lookup.read_general_config", return_value=config)
mocker.patch("solgate.lookup.S3FileSystem.from_config_file",
return_value=[fs])
fs.s3fs.touch("BUCKET/new.csv")
fs.s3fs.touch("BUCKET/old.csv")
s3_backend.buckets["BUCKET"].keys[
"old.csv"].last_modified = old_object_modified_date
assert len(lookup.list_source()) == found_objects