def test_process_existing_sha_files(self):
output = ("/var/tmp/foo.sha1\n"
"/var/tmp/foo.sha224\n"
"/var/tmp/foo.sha256\n"
"/var/tmp/foo.sha384\n"
"/var/tmp/foo.sha512\n")
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = (
"foo@bar:~$ cat filename\n"
"9771ff758f7b66a7933783b8f2e541ed69031daa0cf233acc9eb42f34f885a13 filename\n"
"foo@bar:~$ ")
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
result = scget.process_existing_sha_files(output)
assert result == {
1:
"9771ff758f7b66a7933783b8f2e541ed69031daa0cf233acc9eb42f34f885a13",
224:
"9771ff758f7b66a7933783b8f2e541ed69031daa0cf233acc9eb42f34f885a13",
256:
"9771ff758f7b66a7933783b8f2e541ed69031daa0cf233acc9eb42f34f885a13",
384:
"9771ff758f7b66a7933783b8f2e541ed69031daa0cf233acc9eb42f34f885a13",
512:
"9771ff758f7b66a7933783b8f2e541ed69031daa0cf233acc9eb42f34f885a13",
}
def test_get(self, monkeypatch: MonkeyPatch):
scget = SplitCopyGet()
scget.scs = MockSplitCopyShared()
scget.progress = MockProgress()
def validate_remote_path_get():
pass
def delete_target_local():
pass
def remote_filesize():
return 1000000
def remote_sha_get():
return "abcdef012345"
def split_file_remote(scp_lib, file_size, split_size, remote_tmpdir):
pass
def get_chunk_info(remote_tmpdir):
return [["a", 1234], ["b", 1234], ["c", 1234]]
def get_files(ftp_lib, ssh_lib, scp_lib, chunk, remote_tmpdir,
ssh_kwargs):
return None
def join_files_local():
pass
def local_sha_get(hash):
pass
def inc_percentage():
for n in range(90, 101):
time.sleep(0.1)
scget.progress.totals["percent_done"] = n
monkeypatch.setattr(scget, "validate_remote_path_get",
validate_remote_path_get)
monkeypatch.setattr(scget, "delete_target_local", delete_target_local)
monkeypatch.setattr(scget, "remote_filesize", remote_filesize)
monkeypatch.setattr(scget, "remote_sha_get", remote_sha_get)
monkeypatch.setattr(scget, "split_file_remote", split_file_remote)
monkeypatch.setattr(scget, "get_chunk_info", get_chunk_info)
monkeypatch.setattr(scget, "get_files", get_files)
monkeypatch.setattr(scget, "join_files_local", join_files_local)
monkeypatch.setattr(scget, "local_sha_get", local_sha_get)
thread = Thread(
name="inc_percentage_done",
target=inc_percentage,
)
thread.start()
result = scget.get()
thread.join()
assert isinstance(result[0], datetime.datetime), isinstance(
result[1], datetime.datetime)
def test_validate_remote_path_get_fail(self, monkeypatch: MonkeyPatch):
def path_startswith_tilda():
raise ValueError
scget = SplitCopyGet()
scget.scs = MockSplitCopyShared()
monkeypatch.setattr(scget, "path_startswith_tilda",
path_startswith_tilda)
with raises(SystemExit):
scget.validate_remote_path_get()
def test_find_existing_sha_files(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = "ls output"
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
result = scget.find_existing_sha_files()
assert result == (True, "ls output")
def test_process_existing_sha_files_fail(self):
output = "/var/tmp/foo.sha\n" "/var/tmp/foo.sha224\n"
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
return False, ""
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
result = scget.process_existing_sha_files(output)
assert result == {}
def test_validate_remote_path_get(self, monkeypatch: MonkeyPatch):
scget = SplitCopyGet()
monkeypatch.setattr(scget, "path_startswith_tilda", lambda: True)
monkeypatch.setattr(scget, "verify_path_is_not_directory",
lambda: True)
monkeypatch.setattr(scget, "verify_file_exists", lambda: True)
monkeypatch.setattr(scget, "verify_file_is_readable", lambda: True)
monkeypatch.setattr(scget, "verify_file_is_symlink", lambda: True)
result = scget.validate_remote_path_get()
expected = None
assert expected == result
def test_split_file_remote(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd, timeout):
result = True
stdout = ""
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
result = scget.split_file_remote(MockSCPClient, 1000000, 100000,
"/var/tmp")
assert result == None
def test_get_files_scp(self):
scget = SplitCopyGet()
scget.progress = MockProgress()
scget.copy_proto = "scp"
result = scget.get_files(
MockFTP,
MockSSHShell,
MockSCPClient,
["chunk0", 1999],
"/var/tmp/foo",
{},
)
assert result == None
def test_remote_filesize(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = (
"foo@bar:~$ ls -l /var/tmp/foo\n"
"-rw------- 1 foo bar 69927631 Mar 29 06:49 /var/tmp/foo\n"
"foo@bar:~$")
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
result = scget.remote_filesize()
assert result == 69927631
def test_remote_sha_get(self, monkeypatch: MonkeyPatch):
def find_existing_sha_files():
return True, ""
def process_existing_sha_files(stdout):
return {256: "9f77f4653b052b76af5de0fde3f7c58ae15bfaf3"}
scget = SplitCopyGet()
monkeypatch.setattr(scget, "find_existing_sha_files",
find_existing_sha_files)
monkeypatch.setattr(scget, "process_existing_sha_files",
process_existing_sha_files)
scget.sshshell = MockSSHShell()
result = scget.remote_sha_get()
assert result == {256: "9f77f4653b052b76af5de0fde3f7c58ae15bfaf3"}
def test_split_file_remote_fail(self, monkeypatch: MonkeyPatch):
class MockSSHShell2(MockSSHShell):
def run(self, cmd, timeout):
result = False
stdout = ""
return result, stdout
def close(err_str, hard_close):
raise SystemExit
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
monkeypatch.setattr(scget.scs, "close", close)
with raises(SystemExit):
scget.split_file_remote(MockSCPClient, 1000000, 100000, "/var/tmp")
def test_delete_target_local(self, monkeypatch: MonkeyPatch):
def exists(*args):
return True
def remove(*args):
return True
monkeypatch.setattr("os.path.exists", exists)
monkeypatch.setattr("os.remove", remove)
monkeypatch.setattr("os.path.sep", "/")
scget = SplitCopyGet()
scget.local_dir = "/var/tmp"
scget.local_file = "foo"
result = scget.delete_target_local()
expected = None
assert expected == result
def test_local_sha_get(self, monkeypatch: MonkeyPatch):
class MockHash:
def __init__(self):
pass
def update(self, data):
pass
def hexdigest(self):
return "abcdef0123456789"
class Mock512(MockHash):
pass
class Mock384(MockHash):
pass
class Mock256(MockHash):
pass
class Mock224(MockHash):
pass
class Mock1(MockHash):
pass
scget = SplitCopyGet()
monkeypatch.setattr("builtins.open", MockOpen)
monkeypatch.setattr("os.path.sep", "/")
monkeypatch.setattr("hashlib.sha512", Mock512)
monkeypatch.setattr("hashlib.sha384", Mock384)
monkeypatch.setattr("hashlib.sha256", Mock256)
monkeypatch.setattr("hashlib.sha224", Mock224)
monkeypatch.setattr("hashlib.sha1", Mock1)
sha_hash = {
1: "abcdef0123456789",
224: "abcdef0123456789",
256: "abcdef0123456789",
384: "abcdef0123456789",
512: "abcdef0123456789",
}
scget.local_file = "foo"
scget.local_dir = "/var/tmp"
for i in [512, 384, 256, 224, 1]:
result = scget.local_sha_get(sha_hash)
assert result == None
del sha_hash[i]
def test_local_sha_get_mismatch(self, monkeypatch: MonkeyPatch):
class Mock512(MockHash):
pass
class Mock384(MockHash):
pass
class Mock256(MockHash):
pass
class Mock224(MockHash):
pass
class Mock1(MockHash):
pass
scget = SplitCopyGet()
monkeypatch.setattr("builtins.open", MockOpen)
monkeypatch.setattr("os.path.sep", "/")
monkeypatch.setattr("hashlib.sha512", Mock512)
monkeypatch.setattr("hashlib.sha384", Mock384)
monkeypatch.setattr("hashlib.sha256", Mock256)
monkeypatch.setattr("hashlib.sha224", Mock224)
monkeypatch.setattr("hashlib.sha1", Mock1)
scget.scs = MockSplitCopyShared()
sha_hash = {
512: "0bcdef0123456789",
}
scget.local_file = "foo"
scget.local_dir = "/var/tmp"
with raises(SystemExit):
scget.local_sha_get(sha_hash)
def test_get_chunk_info(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = (
"ls -l /var/tmp/splitcopy_somefile.0.gz.220622105712/\n"
"total 6640K\n"
"-rw------- 1 foo bar 677888 Jun 22 00:57 somefile.0.gz_00\n"
"-rw------- 1 foo bar 673790 Jun 22 00:57 somefile.0.gz_01\n"
"-rw------- 1 foo bar 269 Jun 22 00:57 split.sh\n"
"foo@bar:~$")
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.remote_file = "somefile.0.gz"
result = scget.get_chunk_info(
"/var/tmp/splitcopy_somefile.0.gz.220622105712")
expected = [["somefile.0.gz_00", 677888], ["somefile.0.gz_01", 673790]]
assert expected == result
def test_remote_sha_get_shasum(self, monkeypatch: MonkeyPatch):
class MockSplitCopyShared2(MockSplitCopyShared):
def req_sha_binaries(self, sha_hash):
return "shasum", 1
class MockSSHShell2(MockSSHShell):
def run(self, cmd, timeout):
result = True
stdout = ("foo@bar ~ % shasum -a 1 /var/tmp/foo\n"
"9f77f4653b052b76af5de0fde3f7c58ae15bfaf3 foo\n"
"foo@bar ~ %")
return result, stdout
def find_existing_sha_files():
return False, ""
scget = SplitCopyGet()
monkeypatch.setattr(scget, "find_existing_sha_files",
find_existing_sha_files)
scget.sshshell = MockSSHShell2()
scget.scs = MockSplitCopyShared2()
result = scget.remote_sha_get()
assert result == {1: "9f77f4653b052b76af5de0fde3f7c58ae15bfaf3"}
def test_path_startswith_tilda(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = "foo@bar:~$ ls -d ~/bar\n" "/homes/foo/bar\n" "foo@bar:~$"
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.remote_dir = "~foo/bar"
scget.remote_file = "test"
scget.path_startswith_tilda()
assert (scget.remote_dir == "/homes/foo/bar"
and scget.remote_path == "/homes/foo/bar/test")
def test_get_chunk_info_cmdfail(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = False
stdout = ""
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.scs = MockSplitCopyShared()
scget.remote_file = "somefile.0.gz"
with raises(SystemExit):
scget.get_chunk_info(
"/var/tmp/splitcopy_somefile.0.gz.220622105712")
def test_join_files_local_fail(self, monkeypatch: MonkeyPatch):
def isfile(path):
return False
def glob(path):
return ["file1", "file2"]
monkeypatch.setattr("glob.glob", glob)
monkeypatch.setattr("builtins.open", MockOpen)
monkeypatch.setattr("os.path.isfile", isfile)
monkeypatch.setattr("os.path.sep", "/")
scget = SplitCopyGet()
scget.remote_file = "foo"
scget.local_file = "foo"
scget.local_dir = "/var/tmp"
scget.scs = MockSplitCopyShared()
with raises(SystemExit):
scget.join_files_local()
def test_verify_file_is_readable_fail(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = False
stdout = ""
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.remote_path = "/var/tmp"
with raises(ValueError):
scget.verify_file_is_readable()
def test_remote_filesize_fail(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = False
stdout = ""
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.scs = MockSplitCopyShared()
with raises(SystemExit):
scget.remote_filesize()
def test_path_startswith_tilda_cmdfail(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = False
stdout = ""
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.remote_dir = "~foo/bar"
with raises(ValueError):
scget.path_startswith_tilda()
def test_verify_path_is_not_directory_fail(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = ""
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.remote_path = "/var/tmp"
with raises(ValueError):
scget.verify_path_is_not_directory()
def test_verify_file_is_symlink_fail(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = ""
if re.match(r"ls -l", cmd):
result = False
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.remote_path = "/tmp/foo"
with raises(ValueError):
scget.verify_file_is_symlink()
def test_get_chunk_info_matchfail(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = (
"ls -l /var/tmp/splitcopy_somefile.0.gz.220622105712/\n"
"total 2K\n"
"-rw------- 1 foo bar 269 Jun 22 00:57 split.sh\n"
"foo@bar:~$")
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.scs = MockSplitCopyShared()
scget.remote_file = "somefile.0.gz"
with raises(SystemExit):
scget.get_chunk_info(
"/var/tmp/splitcopy_somefile.0.gz.220622105712")
def test_get_files_scp_authfail(self, monkeypatch: MonkeyPatch):
class MockSSHShell2(MockSSHShell):
def worker_thread_auth(self):
return False
def sleep(secs):
pass
monkeypatch.setattr("time.sleep", sleep)
scget = SplitCopyGet()
scget.progress = MockProgress()
scget.copy_proto = "scp"
with raises(TransferError):
scget.get_files(MockFTP, MockSSHShell2, MockSCPClient,
["chunk0", 1999], "/tmp/", {})
def test_get_files_scp_fail(self, monkeypatch: MonkeyPatch):
class MockSCPClient2(MockSCPClient):
def get(self, *args):
raise SCPException
def sleep(secs):
pass
monkeypatch.setattr("time.sleep", sleep)
scget = SplitCopyGet()
scget.progress = MockProgress()
scget.copy_proto = "scp"
with raises(TransferError):
scget.get_files(MockFTP, MockSSHShell, MockSCPClient2,
["chunk0", 1999], "/tmp/", {})
def test_get_files_ftp_filenotfound_fail(self, monkeypatch: MonkeyPatch):
class MockFTP2(MockFTP):
def get(self, *args):
raise error_proto
def sleep(secs):
pass
monkeypatch.setattr("time.sleep", sleep)
scget = SplitCopyGet()
scget.progress = MockProgress()
scget.copy_proto = "ftp"
with raises(TransferError):
scget.get_files(MockFTP2, MockSSHShell, MockSCPClient,
["chunk0", 1999], "/tmp/", {})
def test_join_files_local(self, monkeypatch: MonkeyPatch):
def isfile(path):
return True
def glob(path):
return ["file1", "file2"]
monkeypatch.setattr("glob.glob", glob)
monkeypatch.setattr("builtins.open", MockOpen)
monkeypatch.setattr("os.path.isfile", isfile)
monkeypatch.setattr("os.path.sep", "/")
scget = SplitCopyGet()
scget.remote_file = "foo"
scget.local_file = "foo"
scget.local_dir = "/var/tmp"
scget.scs = MockSplitCopyShared()
result = scget.join_files_local()
assert result == None
def test_verify_file_is_symlink(self):
class MockSSHShell2(MockSSHShell):
def run(self, cmd):
result = True
stdout = (
"foo@bar:~$ ls -l /tmp/foo\n"
"lrwxrwxrwx 1 foo bar 43 Jun 22 05:48 /tmp/foo -> /var/tmp/foo\n"
"foo@bar:~$")
return result, stdout
scget = SplitCopyGet()
scget.sshshell = MockSSHShell2()
scget.remote_path = "/tmp/foo"
scget.verify_file_is_symlink()
assert (scget.remote_path == "/var/tmp/foo"
and scget.remote_dir == "/var/tmp"
and scget.remote_file == "foo")