def test_tmpdir(self):
parent = pathlib.Path('/tmp') / str(uuid.uuid4())
self.shell.mkdir(remote_path=parent)
try:
with spurplus.TemporaryDirectory(
shell=self.shell, prefix='pre pre', suffix='suf suf', tmpdir=parent) as tmpdir:
pth = tmpdir.path
self.assertTrue(tmpdir.path.name.startswith('pre pre'))
self.assertTrue(tmpdir.path.name.endswith('suf suf'))
self.assertIn(parent, list(tmpdir.path.parents))
self.assertTrue(self.shell.exists(remote_path=pth))
self.assertFalse(self.shell.exists(remote_path=pth))
# with default parameters
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth = tmpdir.path
self.assertTrue(self.shell.exists(remote_path=pth))
self.assertFalse(self.shell.exists(remote_path=pth))
finally:
self.shell.run(command=['rm', '-rf', parent.as_posix()])
def test_put(self) -> None:
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_pth = local_tmpdir.path / 'file.txt'
local_pth.write_text("hello")
remote_pth = remote_tmpdir.path / 'file.txt'
# Diff
dir_diff = self.shell.directory_diff(
local_path=local_tmpdir.path, remote_path=remote_tmpdir.path)
self.assertListEqual([pathlib.Path('file.txt')],
dir_diff.local_only_files)
self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access
self.shell.put(local_path=local_pth, remote_path=remote_pth)
# Diff
self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access
dir_diff = self.shell.directory_diff(
local_path=local_tmpdir.path, remote_path=remote_tmpdir.path)
self.assertListEqual([pathlib.Path('file.txt')],
dir_diff.identical_files)
def test_file_instead_of_dir(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_file = local_tmpdir.path / "local-file"
local_file.write_text("hello")
remote_file = remote_tmpdir.path / "remote-file"
self.shell.write_text(remote_path=remote_file, text="hello")
local_direrr = None # type: Optional[NotADirectoryError]
remote_direrr = None # type: Optional[NotADirectoryError]
try:
self.shell.directory_diff(local_path=local_file, remote_path=remote_tmpdir.path)
except NotADirectoryError as err:
local_direrr = err
self.assertEqual("Local path is not a directory: {}".format(local_file), str(local_direrr))
try:
self.shell.directory_diff(local_path=local_tmpdir.path, remote_path=remote_file)
except NotADirectoryError as err:
remote_direrr = err
a_stat = self.shell.stat(remote_path=remote_file)
self.assertEqual("Remote path is not a directory: {} (mode: {})".format(remote_file, a_stat.st_mode),
str(remote_direrr))
def test_is_dir(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth_to_dir = tmpdir.path / "some-dir"
pth_to_file = tmpdir.path / "some-dir/some-file"
pth_to_file_link = tmpdir.path / "some-dir/some-link-to-file"
pth_to_dir_link = tmpdir.path / "some-link-to-dir"
pth_to_nonexisting = tmpdir.path / "some-non-existing-file"
self.shell.mkdir(remote_path=pth_to_dir)
self.shell.write_text(remote_path=pth_to_file, text="hello")
self.shell.symlink(source=pth_to_file, destination=pth_to_file_link)
self.shell.symlink(source=pth_to_dir, destination=pth_to_dir_link)
self.assertTrue(self.shell.is_dir(pth_to_dir))
self.assertTrue(self.shell.is_dir(pth_to_dir_link))
self.assertFalse(self.shell.is_dir(pth_to_file))
self.assertFalse(self.shell.is_dir(pth_to_file_link))
os_err = None # type: Optional[OSError]
try:
self.shell.is_dir(remote_path=pth_to_nonexisting)
except OSError as err:
os_err = err
self.assertIsNotNone(os_err)
self.assertEqual("Remote file does not exist: {}".format(pth_to_nonexisting.as_posix()), str(os_err))
def test_that_reusing_sftp_is_faster_for_big_files(self): # pylint: disable=invalid-name
# Spurplus is slower at copying many small files than Spur due to the added safety overhead.
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
# open/close
spur_shell = self.shell.as_spur()
start = time.time()
number_of_files = 4
size = 1024 * 1024 * 2
content = size * "hello"
for i in range(0, number_of_files):
pth = tmpdir.path / '{}.txt'.format(i)
with spur_shell.open(name=pth.as_posix(), mode='wt') as fid:
fid.write(content)
their_duration = time.time() - start
# re-use sftp client
start = time.time()
for i in range(0, number_of_files):
pth = tmpdir.path / '{}.txt'.format(i)
self.shell.write_text(remote_path=pth, text=content, consistent=False, create_directories=False)
our_duration = time.time() - start
speedup = their_duration / our_duration
self.assertGreater(speedup, 10.0)
def test_is_symlink(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth_to_dir = tmpdir.path / "some-dir"
pth_to_file = tmpdir.path / "some-dir/some-file"
pth_to_file_link = tmpdir.path / "some-dir/some-link-to-file"
pth_to_dir_link = tmpdir.path / "some-link-to-dir"
pth_to_nonexisting = tmpdir.path / "some-non-existing-file"
self.shell.mkdir(remote_path=pth_to_dir)
self.shell.write_text(remote_path=pth_to_file, text="hello")
self.shell.run(command=['ln', '-s', pth_to_file.as_posix(), pth_to_file_link.as_posix()])
self.shell.run(command=['ln', '-s', pth_to_dir.as_posix(), pth_to_dir_link.as_posix()])
self.assertFalse(self.shell.is_symlink(pth_to_dir))
self.assertTrue(self.shell.is_symlink(pth_to_dir_link))
pth_to_dir.is_symlink()
self.assertFalse(self.shell.is_symlink(pth_to_file))
self.assertTrue(self.shell.is_symlink(pth_to_file_link))
self.assertFalse(self.shell.is_symlink(pth_to_dir))
self.assertTrue(self.shell.is_symlink(pth_to_dir_link))
notfounderr = None # type: Optional[FileNotFoundError]
try:
self.shell.is_symlink(remote_path=pth_to_nonexisting)
except FileNotFoundError as err:
notfounderr = err
self.assertEqual("Remote file does not exist: {}".format(pth_to_nonexisting.as_posix()), str(notfounderr))
def test_symlink(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth_to_dir = tmpdir.path / "some-dir"
pth_to_file = tmpdir.path / "some-file"
pth_to_file_link = tmpdir.path / "some-link-to-file"
pth_to_dir_link = tmpdir.path / "some-link-to-dir"
pth_to_another_file = tmpdir.path / "another-file"
self.shell.mkdir(remote_path=pth_to_dir)
self.shell.write_text(remote_path=pth_to_file, text="hello")
self.shell.write_text(remote_path=pth_to_another_file, text="zdravo")
self.shell.symlink(source=pth_to_file, destination=pth_to_file_link)
self.shell.symlink(source=pth_to_dir, destination=pth_to_dir_link)
self.assertFalse(self.shell.is_symlink(pth_to_file))
self.assertTrue(self.shell.is_symlink(pth_to_file_link))
self.assertFalse(self.shell.is_symlink(pth_to_dir))
self.assertTrue(self.shell.is_symlink(pth_to_dir_link))
self.assertEqual("hello", self.shell.read_text(remote_path=pth_to_file_link))
# Overwriting a link is not possible.
file_exists_err = None # type: Optional[FileExistsError]
try:
self.shell.symlink(source=pth_to_another_file, destination=pth_to_file_link)
except FileExistsError as err:
file_exists_err = err
self.assertIsNotNone(file_exists_err)
self.assertEqual("The destination of the symbolic link already exists: {}".format(
pth_to_file_link.as_posix()), str(file_exists_err))
def test_put_no_permission(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir:
with temppathlib.TemporaryDirectory() as local_tmpdir:
local_pth = local_tmpdir.path / 'file.txt'
local_pth.write_text("hello")
remote_pth = remote_tmpdir.path / 'file.txt'
self.shell.put(local_path=local_pth, remote_path=remote_pth)
self.shell.chmod(remote_path=remote_pth, mode=0o444)
# consistent put succeeds even though the remote path has read-only permissions.
self.shell.put(local_path=local_pth, remote_path=remote_pth, consistent=True)
a_stat = self.shell.stat(remote_path=remote_pth.as_posix())
self.assertEqual(0o100444, a_stat.st_mode)
# direct put fails since we can not write to the file.
with self.assertRaises(PermissionError):
self.shell.put(local_path=local_pth, remote_path=remote_pth, consistent=False)
# consistent put fails if we don't have write permissions to the directory
try:
self.shell.chmod(remote_path=remote_tmpdir.path, mode=0o444)
with self.assertRaises(PermissionError):
self.shell.put(local_path=local_pth, remote_path=remote_pth, consistent=True)
finally:
self.shell.chmod(remote_path=remote_tmpdir.path, mode=0o777)
def test_md5_versus_md5s(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
remote_pths = [] # type: List[pathlib.Path]
for i in range(0, 128):
pth = tmpdir.path / "{}.txt".format(i)
remote_pths.append(pth)
if i % 2 == 0:
self.shell.write_text(remote_path=pth, text="hello")
start = time.time()
md5s = self.shell.md5s(remote_paths=remote_pths)
md5s_duration = time.time() - start
# now benchmark the manual implementation
start = time.time()
result = [] # type: List[Optional[str]]
for remote_pth in remote_pths:
if self.shell.exists(remote_path=remote_pth):
result.append(self.shell.md5(remote_path=remote_pth))
else:
result.append(None)
manual_duration = time.time() - start
self.assertListEqual(md5s, result)
speedup = manual_duration / md5s_duration
self.assertGreaterEqual(speedup, 10.0)
def test_exists(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth = tmpdir.path / "some-dir" / "oi"
self.assertFalse(self.shell.exists(remote_path=pth))
self.shell.write_text(remote_path=pth, text="hello")
self.assertTrue(self.shell.exists(remote_path=pth))
def test_mkdir(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth = tmpdir.path / "some-dir" / "oi"
self.shell.mkdir(remote_path=pth, mode=0o700, parents=True)
self.assertTrue(self.shell.exists(remote_path=pth))
# check with SFTP
a_stat = self.shell.stat(pth.as_posix())
self.assertEqual(0o40700, a_stat.st_mode)
# does nothing
self.shell.mkdir(remote_path=pth, mode=0o700, parents=True, exist_ok=True)
# existing directory raises an error on exist_ok=False
existserr = None # type: Optional[FileExistsError]
try:
self.shell.mkdir(remote_path=pth, mode=0o700, parents=True, exist_ok=False)
except FileExistsError as err:
existserr = err
self.assertEqual("The remote directory already exists: {}".format(pth), str(existserr))
# existing directory does not raise an error on exist_ok=True
self.shell.mkdir(remote_path=pth, mode=0o700, exist_ok=True)
def test_write_read_text(self):
for consistent in [True, False]:
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth = tmpdir.path / "some-dir" / str(consistent)
self.shell.write_text(remote_path=pth, text="hello", consistent=consistent)
text = self.shell.read_text(remote_path=pth)
self.assertEqual("hello", text)
def test_md5_with_space(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth = tmpdir.path / "oi hoi"
self.shell.write_text(remote_path=pth, text="hello")
md5digest = self.shell.md5(remote_path=pth)
expected = hashlib.md5("hello".encode()).hexdigest()
self.assertEqual(expected, md5digest)
def test_mkdir(self) -> None:
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth_to_folder = tmpdir.path / "some-folder"
self.assertFalse(self.shell.exists(remote_path=pth_to_folder))
self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access
self.reconnecting_sftp.mkdir(path=pth_to_folder.as_posix())
self.assertTrue(self.shell.exists(remote_path=pth_to_folder))
def test_local_only_file(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_pth_to_file = local_tmpdir.path / "some-file"
local_pth_to_file.write_text("hello")
self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path)
remote_pth_to_file = remote_tmpdir.path / "some-file"
self.assertTrue(self.shell.exists(remote_path=remote_pth_to_file))
self.assertEqual("hello", self.shell.read_text(remote_path=remote_pth_to_file))
def test_local_only_directory(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_dir = local_tmpdir.path / "some-dir"
local_dir.mkdir()
self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path)
remote_dir = remote_tmpdir.path / "some-dir"
self.assertTrue(self.shell.exists(remote_path=remote_dir))
self.assertTrue(self.shell.is_dir(remote_path=remote_dir))
def test_read_bytes_no_permission(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir:
pth = remote_tmpdir.path / "some-file"
self.shell.write_bytes(remote_path=pth, data=b"hello")
try:
self.shell.chmod(remote_path=pth, mode=0o111)
with self.assertRaises(PermissionError):
self.shell.read_bytes(remote_path=pth)
finally:
self.shell.chmod(remote_path=pth, mode=0o777)
def test_read_bytes_not_found(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir:
pth = remote_tmpdir.path / "some-file"
notfounderr = None # type: Optional[FileNotFoundError]
try:
self.shell.read_bytes(remote_path=pth)
except FileNotFoundError as err:
notfounderr = err
self.assertEqual("The remote path was not found: {}".format(pth.as_posix()), str(notfounderr))
def test_remote_only_file_is_deleted(self): # pylint: disable=invalid-name
for delete in [spurplus.Delete.BEFORE, spurplus.Delete.AFTER]:
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
remote_pth_to_file = remote_tmpdir.path / "some-file"
self.shell.write_text(remote_path=remote_pth_to_file, text="hello")
self.assertTrue(self.shell.exists(remote_path=remote_pth_to_file))
self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path, delete=delete)
self.assertFalse(self.shell.exists(remote_path=remote_pth_to_file))
def test_mkdir_with_permission_error(self): # pylint: disable=invalid-name
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
try:
self.shell.chmod(remote_path=tmpdir.path, mode=0o444)
pth = tmpdir.path / "some-dir" / "oi"
with self.assertRaises(PermissionError):
self.shell.mkdir(remote_path=pth, mode=0o700, parents=True)
finally:
self.shell.chmod(remote_path=tmpdir.path, mode=0o777)
def test_nonexisting_local_path(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_pth = local_tmpdir.path / "some-dir"
notfounderr = None # type: Optional[FileNotFoundError]
try:
self.shell.sync_to_remote(local_path=local_pth, remote_path=remote_tmpdir.path)
except FileNotFoundError as err:
notfounderr = err
self.assertEqual("Local path does not exist: {}".format(local_pth), str(notfounderr))
def test_local_path_is_not_a_dir(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_pth_to_file = local_tmpdir.path / "some-file"
local_pth_to_file.write_text("hello")
direrr = None # type: Optional[NotADirectoryError]
try:
self.shell.sync_to_remote(local_path=local_pth_to_file, remote_path=remote_tmpdir.path)
except NotADirectoryError as err:
direrr = err
self.assertEqual("Local path is not a directory: {}".format(local_pth_to_file), str(direrr))
def test_no_local_and_remote_dir(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir:
local_tmpdir = pathlib.Path("nonexisting-local-dir")
remote_tmpdir = remote_tmpdir.path / "nonexisting-remote-dir"
notfounderr = None # type: Optional[FileNotFoundError]
try:
self.shell.directory_diff(local_path=local_tmpdir, remote_path=remote_tmpdir)
except FileNotFoundError as err:
notfounderr = err
self.assertEqual("Both the local and the remote path do not exist: {} and {}".format(
local_tmpdir, remote_tmpdir), str(notfounderr))
def test_remote_only_dir_is_deleted(self):
for delete in [spurplus.Delete.BEFORE, spurplus.Delete.AFTER]:
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
remote_pth_to_dir = remote_tmpdir.path / "some-dir"
self.shell.mkdir(remote_path=remote_pth_to_dir)
self.shell.write_text(remote_path=remote_pth_to_dir / "some-file", text="hello")
self.assertTrue(self.shell.exists(remote_path=remote_pth_to_dir))
self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path, delete=delete)
self.assertFalse(self.shell.exists(remote_path=remote_pth_to_dir))
def test_symlink(self):
for recursive in [True, False]:
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth_to_file = tmpdir.path / "some-file"
self.shell.write_text(remote_path=pth_to_file, text="hello")
pth_to_file_link = tmpdir.path / "some-link-to-file"
self.shell.symlink(source=pth_to_file, destination=pth_to_file_link)
self.assertTrue(self.shell.exists(remote_path=pth_to_file))
self.assertTrue(self.shell.exists(remote_path=pth_to_file_link))
self.shell.remove(remote_path=pth_to_file_link, recursive=recursive)
self.assertTrue(self.shell.exists(remote_path=pth_to_file))
self.assertFalse(self.shell.exists(remote_path=pth_to_file_link))
def test_local_dir_permission_invalid_pth(self): # pylint: disable=invalid-name
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_dir = local_tmpdir.path / "some-nonexisting-dir"
notfounderr = None # type: Optional[FileNotFoundError]
try:
self.shell.mirror_local_permissions(
relative_paths=[pathlib.Path("some-relative-dir")],
local_path=local_dir,
remote_path=remote_tmpdir.path)
except FileNotFoundError as err:
notfounderr = err
self.assertEqual("Local path does not exist: {}".format(local_dir), str(notfounderr))
# file instead of dir as local path
local_file = local_tmpdir.path / "some-file"
local_file.write_text("hello")
local_direrr = None # type: Optional[NotADirectoryError]
try:
self.shell.mirror_local_permissions(
relative_paths=[pathlib.Path("some-relative-dir")],
local_path=local_file,
remote_path=remote_tmpdir.path)
except NotADirectoryError as err:
local_direrr = err
self.assertEqual("Local path is not a directory: {}".format(local_file), str(local_direrr))
# file instead of dir as remote path
remote_file = remote_tmpdir.path / "some-file"
self.shell.write_text(remote_path=remote_file, text="hello")
remote_direrr = None # type: Optional[NotADirectoryError]
try:
self.shell.mirror_local_permissions(
relative_paths=[pathlib.Path("some-relative-dir")],
local_path=local_tmpdir.path,
remote_path=remote_file)
except NotADirectoryError as err:
remote_direrr = err
self.assertEqual("Remote path is not a directory: {}".format(remote_file), str(remote_direrr))
def test_posix_rename_file(self) -> None:
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth_to_file = tmpdir.path / "some-file"
new_pth_to_file = tmpdir.path / "renamed-file"
self.shell.write_text(remote_path=pth_to_file, text="hello")
self.assertTrue(self.shell.exists(remote_path=pth_to_file))
self.assertFalse(self.shell.exists(remote_path=new_pth_to_file))
self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access
self.reconnecting_sftp.posix_rename(
oldpath=pth_to_file.as_posix(),
newpath=new_pth_to_file.as_posix())
self.assertFalse(self.shell.exists(remote_path=pth_to_file))
self.assertTrue(self.shell.exists(remote_path=new_pth_to_file))
def test_non_empty_dir(self):
with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir:
pth_to_some_dir = tmpdir.path / "some-dir"
pth_to_subdir = tmpdir.path / "some-dir/some-subdir"
self.shell.mkdir(remote_path=pth_to_subdir, parents=True)
os_err = None # type: Optional[OSError]
try:
self.shell.remove(remote_path=pth_to_some_dir)
except OSError as err:
os_err = err
self.assertIsNotNone(os_err)
self.assertEqual("The remote directory is not empty and the recursive flag was not set: {}".format(
pth_to_some_dir.as_posix()), str(os_err))
def test_local_only_link(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_pth_to_file = local_tmpdir.path / "some-file"
local_pth_to_link = local_tmpdir.path / "some-link"
local_pth_to_file.write_text("hello")
local_pth_to_link.symlink_to(local_pth_to_file)
self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path)
# Check that the symlinks are copied as files, not as links
remote_pth_to_link = remote_tmpdir.path / "some-link"
self.assertFalse(self.shell.is_symlink(remote_path=remote_pth_to_link))
self.assertTrue(self.shell.exists(remote_path=remote_pth_to_link))
self.assertEqual("hello", self.shell.read_text(remote_path=remote_pth_to_link))
def test_local_permissions(self):
with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \
temppathlib.TemporaryDirectory() as local_tmpdir:
local_pth = local_tmpdir.path / "some-dir/some-file"
local_pth.parent.mkdir()
local_pth.write_text("hello")
local_pth.chmod(0o612)
remote_pth = remote_tmpdir.path / "some-dir/some-file"
self.shell.mkdir(remote_path=remote_pth.parent)
self.shell.write_text(remote_path=remote_pth, text="hello")
self.shell.mirror_local_permissions(
relative_paths=[pathlib.Path("some-dir/some-file")],
local_path=local_tmpdir.path,
remote_path=remote_tmpdir.path)
