def test_compare_time_same(self) -> None:
"""Test compare_time."""
now = datetime.datetime.now()
assert (ExactTimestampsSync().compare_time(
FileStats(src="", last_update=now, operation_name="download"),
FileStats(src="", last_update=now),
) is True)
def test_compare_time_src_older(self) -> None:
"""Test compare_time."""
time_dest = datetime.datetime.now()
time_src = time_dest - datetime.timedelta(days=1)
assert (ExactTimestampsSync().compare_time(
FileStats(src="", last_update=time_src, operation_name="download"),
FileStats(src="", last_update=time_dest),
) is False)
def test_determine_should_sync(self, expected: bool, is_size: bool,
is_time: bool,
mocker: MockerFixture) -> None:
"""Test determine_should_sync."""
mock_compare_size = mocker.patch.object(DeleteSync,
"compare_size",
return_value=is_size)
mock_compare_time = mocker.patch.object(DeleteSync,
"compare_time",
return_value=is_time)
assert (DeleteSync().determine_should_sync(
FileStats(src=""), FileStats(src="")) is expected)
mock_compare_size.assert_not_called()
mock_compare_time.assert_not_called()
def test_determine_should_sync(self, expected: bool, is_size: bool,
is_time: bool,
mocker: MockerFixture) -> None:
"""Test determine_should_sync."""
src_file = FileStats(src="")
dest_file = FileStats(src="")
mock_compare_size = mocker.patch.object(SizeOnlySync,
"compare_size",
return_value=is_size)
mock_compare_time = mocker.patch.object(SizeOnlySync,
"compare_time",
return_value=is_time)
assert SizeOnlySync().determine_should_sync(src_file,
dest_file) is expected
mock_compare_size.assert_called_once_with(src_file, dest_file)
mock_compare_time.assert_not_called()
def test_call_locals3(self, mocker: MockerFixture, tmp_path: Path) -> None:
"""Test call."""
src = str(tmp_path) + os.sep
dest = "s3://bucket/prefix/"
formatted_path = FormatPath.format(src, dest)
extra_info: _LastModifiedAndSize = {"LastModified": NOW, "Size": 13}
mock_find_dest_path_comp_key = mocker.patch(
f"{MODULE}.find_dest_path_comp_key",
return_value=("dest-path", "compare-key"),
)
mock_list_files = mocker.patch.object(FileGenerator,
"list_files",
return_value=[(f"{src}test.txt",
extra_info)])
mock_list_objects = mocker.patch.object(FileGenerator, "list_objects")
assert list(
FileGenerator(self.client, "operation").call(formatted_path)) == [
FileStats(
src=f"{src}test.txt",
compare_key="compare-key",
dest="dest-path",
dest_type="s3",
last_update=extra_info["LastModified"],
operation_name="operation",
response_data=None,
size=extra_info["Size"],
src_type="local",
)
]
mock_list_files.assert_called_once_with(formatted_path["src"]["path"],
formatted_path["dir_op"])
mock_list_objects.assert_not_called()
mock_find_dest_path_comp_key.assert_called_once_with(
formatted_path, f"{src}test.txt")
def test_call_empty_src(self) -> None:
"""Test call empty src."""
self.not_at_src_sync_strategy.determine_should_sync.return_value = True
src_files: List[FileStats] = []
dest_files: List[FileStats] = []
ref_list: List[FileStats] = []
result_list: List[FileStats] = []
dest_file = FileStats(
src="",
dest="",
compare_key="comparator_test.py",
size=10,
last_update=NOW,
src_type="s3",
dest_type="local",
operation_name="",
)
dest_files.append(dest_file)
ref_list.append(dest_file)
files = self.comparator.call(iter(src_files), iter(dest_files))
for filename in files:
result_list.append(filename)
assert result_list == ref_list
# Now try when the sync strategy says not to sync the file.
self.not_at_src_sync_strategy.determine_should_sync.return_value = False
result_list = []
ref_list = []
files = self.comparator.call(iter(src_files), iter(dest_files))
for filename in files:
result_list.append(filename)
assert result_list == ref_list
def test_call_compare_key_equal_should_not_sync(self) -> None:
"""Test call compare key equal should not sync."""
self.sync_strategy.determine_should_sync.return_value = False
ref_list: List[FileStats] = []
result_list: List[FileStats] = []
src_files = [
FileStats(
src="",
dest="",
compare_key="comparator_test.py",
size=10,
last_update=NOW,
src_type="local",
dest_type="s3",
operation_name="upload",
)
]
dest_files = [
FileStats(
src="",
dest="",
compare_key="comparator_test.py",
size=10,
last_update=NOW,
src_type="s3",
dest_type="local",
operation_name="",
)
]
files = self.comparator.call(iter(src_files), iter(dest_files))
for filename in files:
result_list.append(filename)
assert result_list == ref_list
# Try when the sync strategy says to sync the file.
self.sync_strategy.determine_should_sync.return_value = True
ref_list = []
result_list = []
files = self.comparator.call(iter(src_files), iter(dest_files))
ref_list.append(src_files[0])
for filename in files:
result_list.append(filename)
assert result_list == ref_list
def test_call_s3(self) -> None:
"""Test call."""
exclude_md = FileStats(src="/tmp/exclude/README.md",
src_type="local",
dest="")
include_md = FileStats(src="bucket/include/README.md",
src_type="s3",
dest="")
other_file = FileStats(src="bucket/test.txt", src_type="s3", dest="")
params = ParametersDataModel(
src="s3://bucket/",
dest="/tmp",
dir_op=True,
exclude=["/tmp/exclude/README.md"],
)
obj = Filter.parse_params(params)
result = list(obj.call(iter([other_file, include_md, exclude_md])))
assert exclude_md not in result
assert include_md in result
assert other_file in result
assert len(result) == 2
def test_dict(self) -> None:
"""Test dict."""
data: FileStatsDict = {
"src": "/path/test.txt",
"compare_key": "compare-key",
"dest": "s3://dest-path",
"dest_type": "s3",
"last_update": NOW,
"operation_name": "test",
"response_data": None, # type: ignore
"size": 13,
"src_type": "local",
}
assert FileStats(**data).dict() == data
def test_call_inject_info(self) -> None:
"""Test call _inject_info."""
client = Mock()
builder = FileInfoBuilder(client=client)
file_stats = FileStats(
src="src",
dest="dest",
compare_key="compare_key",
size=10,
operation_name="operation_name",
)
results = list(builder.call([file_stats]))
assert len(results) == 1
result = results[0]
assert isinstance(result, FileInfo)
assert result.src == file_stats.src
assert result.dest == file_stats.dest
assert result.compare_key == file_stats.compare_key
assert result.size == file_stats.size
assert result.operation_name == file_stats.operation_name
assert result.client == client
def test_call_delete(self) -> None:
"""Test call delete."""
client = Mock()
source_client = Mock()
builder = FileInfoBuilder(
client=client,
source_client=source_client,
parameters=ParametersDataModel(dest="", src="", delete=True),
)
file_stats = FileStats(
src="src",
dest="dest",
compare_key="compare_key",
size=10,
operation_name="delete",
)
results = list(builder.call([file_stats]))
assert len(results) == 1
result = results[0]
assert result.client == source_client
assert result.source_client == client
def test_compare_time_download(self) -> None:
"""Test compare_time."""
obj = BaseSync()
now = datetime.datetime.now()
future = now + datetime.timedelta(0, 15)
kwargs = {"src": "", "operation_name": "download"}
assert (obj.compare_time(
FileStats(last_update=now, **kwargs),
FileStats(last_update=now, **kwargs),
) is True)
assert (obj.compare_time(
FileStats(last_update=now, **kwargs),
FileStats(last_update=future, **kwargs),
) is False)
assert (obj.compare_time(
FileStats(last_update=future, **kwargs),
FileStats(last_update=now, **kwargs),
) is True)
def test_compare_size(self, dest_size: int, expected: bool,
src_size: int) -> None:
"""Test compare_size."""
src_file = FileStats(src="", size=src_size)
dest_file = FileStats(src="", size=dest_size)
assert BaseSync.compare_size(src_file, dest_file) is expected