def setUp(self) -> None:
self.protocol = randstr()
self.separator = randstr()
self.client = mock.Mock()
self.fs = S3FileSystem(self.client,
uri_protocol=self.protocol,
separator=self.separator)
def setUp(self) -> None:
self.client = boto3.client("s3")
self.sut = S3ToS3CopyStrategy(self.client)
self.fs = S3FileSystem(self.client)
self.src_bucket_name = randstr()
self.dst_bucket_name = randstr()
self.client.create_bucket(Bucket=self.src_bucket_name)
self.client.create_bucket(Bucket=self.dst_bucket_name)
def setUp(self) -> None:
self.client = boto3.client("s3")
self.sut = LocalToS3CopyStrategy(self.client)
self.src_tmp_dir = TemporaryDirectory()
self.src_fs = LocalFileSystem()
self.dst_fs = S3FileSystem(self.client)
self.bucket_name = randstr()
self.client.create_bucket(Bucket=self.bucket_name)
def test_path_to_string(self) -> None:
path: Path = random_path()
result: str = self.fs.path_to_string(path)
assert result == f"{path.drive}/{self.separator.join(path.tail)}"
path = Path(randstr())
result = self.fs.path_to_string(path)
assert result == f"{path.drive}"
def setUp(self) -> None:
self.client: Any = boto3.client("s3")
self.s3: Any = boto3.resource("s3")
self.bucket_name: str = "myBucket"
self.bucket: Any = self.s3.Bucket(self.bucket_name)
self.separator: str = randstr(3)
self.fs = S3FileSystem(self.client, separator=self.separator)
self._create_bucket()
def test_copy_dir_to_dir(self) -> None:
src_path: Path = random_path()
dst_path: Path = random_path()
content1: bytes = randstr().encode()
content2: bytes = randstr().encode()
self.src_fs.file_write(src_path.child("file1"), content1)
self.src_fs.file_write(
src_path.child("subdir").child("file2"), content2)
assert self.src_fs.file_read(src_path.child("file1")) == content1
assert self.src_fs.file_read(
src_path.child("subdir").child("file2")) == content2
self.sut.copy_dir_to_dir(CopyDirectory(self.src_fs, src_path),
CopyDirectory(self.dst_fs, dst_path))
assert self.dst_fs.file_read(dst_path.child("file1")) == content1
assert self.dst_fs.file_read(
dst_path.child("subdir").child("file2")) == content2
def test_file_read(self) -> None:
file_contents: bytes = randstr().encode()
scenario: Scenario = {
'dira': {
'fileaa': file_contents,
},
}
self.setup_scenario(scenario)
assert self.fs.file_read(self.make_path("dira", "fileaa")) == file_contents
def test_copy_dir_to_dir(self) -> None:
src_path: Path = random_path(self.src_bucket_name)
dst_path: Path = random_path(self.dst_bucket_name)
content: bytes = randstr().encode()
scenario: List[Tuple[Path,
bytes]] = [(random_path(), randstr().encode())
for _ in range(1100)]
for path, content in scenario:
self.fs.file_write(
Path(src_path.drive, *src_path.tail, *path.tail), content)
self.sut.copy_dir_to_dir(CopyDirectory(self.fs, src_path),
CopyDirectory(self.fs, dst_path))
for path, content in scenario:
assert self.fs.file_read(
Path(dst_path.drive, *dst_path.tail, *path.tail)) == content
def test_copy_file_to_file(self) -> None:
src_path: Path = random_local_path(root=self.src_tmp_dir.name)[0]
dst_path: Path = random_path(self.bucket_name)
content: bytes = randstr().encode()
self.src_fs.file_write(src_path, content)
self.sut.copy_file_to_file(CopyFile(self.src_fs, src_path),
CopyFile(self.dst_fs, dst_path))
assert self.dst_fs.file_read(dst_path) == content
def test_copy_file_to_file(self) -> None:
src_path: Path = random_path(self.src_bucket_name)
dst_path: Path = random_path(self.dst_bucket_name)
content: bytes = randstr().encode()
self.fs.file_write(src_path, content)
self.sut.copy_file_to_file(CopyFile(self.fs, src_path),
CopyFile(self.fs, dst_path))
assert self.fs.file_read(dst_path) == content
def test_parse_path(self) -> None:
expected_path: Path = random_path()
abs_input_path_str: str = f"{expected_path.drive}/{self.separator.join(expected_path.tail)}"
assert self.fs.parse_path(abs_input_path_str) == expected_path
assert self.fs.parse_path(abs_input_path_str +
self.separator) == expected_path
expected_path = Path(randstr())
abs_input_path_str = f"{expected_path.drive}"
assert self.fs.parse_path(abs_input_path_str) == expected_path
assert self.fs.parse_path(abs_input_path_str + "/") == expected_path
assert self.fs.parse_path(abs_input_path_str + "/" +
self.separator) == expected_path
def test_file_write(self) -> None:
scenario: Scenario = {
'dira': {}
}
file_contents: bytes = randstr().encode()
expected_scenario: Scenario = {
'dira': {
'dirb': {
'fileaba': file_contents
}
}
}
self.setup_scenario(scenario)
self.fs.file_write(self.make_path("dira", "dirb", "fileaba"), file_contents)
self.assert_scenario(expected_scenario)
def test_file_read_on_a_dir_fails(self) -> None:
file_contents: bytes = randstr().encode()
scenario: Scenario = {
'dira': {
'fileaa': file_contents,
},
}
self.setup_scenario(scenario)
error_happened: bool
try:
self.fs.file_read(self.make_path("dira"))
except FileSystemOperationError:
error_happened = True
else:
error_happened = False
assert error_happened
def test_file(self) -> None:
child_name: str = randstr()
assert self.sut.file(child_name) == File(
self.filesystem, None, _path=self.path.child.return_value)
self.path.child.assert_called_once_with(child_name)
def setUp(self) -> None:
self.separator = randstr(3)
self.fs = MemoryFileSystem(mock.Mock(spec=MemoryDirectory), separator=self.separator)
def test_dir_list_continuation_token(self) -> None:
times: int = random.randint(2, 5)
input_path: Path = random_path()
tail_str: str = self.separator.join(input_path.tail)
files: List[List[str]] = [[
randstr() for _ in range(random.randint(1, 3))
] for _ in range(times)]
dirs: List[List[str]] = [[
randstr() for _ in range(random.randint(1, 3))
] for _ in range(times)]
cont_tokens: List[str] = [randstr() for _ in range(times - 1)]
self.client.list_objects.side_effect = [{
"Contents": [{
"Key": tail_str + self.separator + file
} for file in iter_files],
"CommonPrefixes": [{
"Prefix":
tail_str + self.separator + dir + self.separator
} for dir in iter_dirs],
"IsTruncated":
True,
"NextMarker":
cont_token
} for iter_files, iter_dirs, cont_token in zip(
files[:-1], dirs[:-1], cont_tokens)] + [{
"Contents": [{
"Key": tail_str + self.separator + file
} for file in files[-1]],
"CommonPrefixes": [{
"Prefix":
tail_str + self.separator + dir + self.separator
} for dir in dirs[-1]],
"IsTruncated":
False,
}]
result_files = []
result_dirs = []
result_others = []
for obj in self.fs.dir_list(input_path):
if obj.type == FSObjectType.DIR:
result_dirs.append(obj.path)
elif obj.type == FSObjectType.FILE:
result_files.append(obj.path)
else:
result_others.append(obj.path)
expected_call_list: List[mock.call] = [
mock.call(Bucket=input_path.drive,
Prefix=tail_str + self.separator,
Delimiter=self.separator,
Marker="")
] + [
mock.call(Bucket=input_path.drive,
Prefix=tail_str + self.separator,
Delimiter=self.separator,
Marker=cont_token) for cont_token in cont_tokens
]
assert self.client.list_objects.call_args_list == expected_call_list
assert result_files == [
input_path.child(file) for files_it in files for file in files_it
]
assert result_dirs == [
input_path.child(dir) for dirs_it in dirs for dir in dirs_it
]
assert result_others == []