def test_tmpdir(self): parent = pathlib.Path('/tmp') / str(uuid.uuid4()) self.shell.mkdir(remote_path=parent) try: with spurplus.TemporaryDirectory( shell=self.shell, prefix='pre pre', suffix='suf suf', tmpdir=parent) as tmpdir: pth = tmpdir.path self.assertTrue(tmpdir.path.name.startswith('pre pre')) self.assertTrue(tmpdir.path.name.endswith('suf suf')) self.assertIn(parent, list(tmpdir.path.parents)) self.assertTrue(self.shell.exists(remote_path=pth)) self.assertFalse(self.shell.exists(remote_path=pth)) # with default parameters with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth = tmpdir.path self.assertTrue(self.shell.exists(remote_path=pth)) self.assertFalse(self.shell.exists(remote_path=pth)) finally: self.shell.run(command=['rm', '-rf', parent.as_posix()])
def test_put(self) -> None: with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_pth = local_tmpdir.path / 'file.txt' local_pth.write_text("hello") remote_pth = remote_tmpdir.path / 'file.txt' # Diff dir_diff = self.shell.directory_diff( local_path=local_tmpdir.path, remote_path=remote_tmpdir.path) self.assertListEqual([pathlib.Path('file.txt')], dir_diff.local_only_files) self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access self.shell.put(local_path=local_pth, remote_path=remote_pth) # Diff self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access dir_diff = self.shell.directory_diff( local_path=local_tmpdir.path, remote_path=remote_tmpdir.path) self.assertListEqual([pathlib.Path('file.txt')], dir_diff.identical_files)
def test_file_instead_of_dir(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_file = local_tmpdir.path / "local-file" local_file.write_text("hello") remote_file = remote_tmpdir.path / "remote-file" self.shell.write_text(remote_path=remote_file, text="hello") local_direrr = None # type: Optional[NotADirectoryError] remote_direrr = None # type: Optional[NotADirectoryError] try: self.shell.directory_diff(local_path=local_file, remote_path=remote_tmpdir.path) except NotADirectoryError as err: local_direrr = err self.assertEqual("Local path is not a directory: {}".format(local_file), str(local_direrr)) try: self.shell.directory_diff(local_path=local_tmpdir.path, remote_path=remote_file) except NotADirectoryError as err: remote_direrr = err a_stat = self.shell.stat(remote_path=remote_file) self.assertEqual("Remote path is not a directory: {} (mode: {})".format(remote_file, a_stat.st_mode), str(remote_direrr))
def test_is_dir(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth_to_dir = tmpdir.path / "some-dir" pth_to_file = tmpdir.path / "some-dir/some-file" pth_to_file_link = tmpdir.path / "some-dir/some-link-to-file" pth_to_dir_link = tmpdir.path / "some-link-to-dir" pth_to_nonexisting = tmpdir.path / "some-non-existing-file" self.shell.mkdir(remote_path=pth_to_dir) self.shell.write_text(remote_path=pth_to_file, text="hello") self.shell.symlink(source=pth_to_file, destination=pth_to_file_link) self.shell.symlink(source=pth_to_dir, destination=pth_to_dir_link) self.assertTrue(self.shell.is_dir(pth_to_dir)) self.assertTrue(self.shell.is_dir(pth_to_dir_link)) self.assertFalse(self.shell.is_dir(pth_to_file)) self.assertFalse(self.shell.is_dir(pth_to_file_link)) os_err = None # type: Optional[OSError] try: self.shell.is_dir(remote_path=pth_to_nonexisting) except OSError as err: os_err = err self.assertIsNotNone(os_err) self.assertEqual("Remote file does not exist: {}".format(pth_to_nonexisting.as_posix()), str(os_err))
def test_that_reusing_sftp_is_faster_for_big_files(self): # pylint: disable=invalid-name # Spurplus is slower at copying many small files than Spur due to the added safety overhead. with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: # open/close spur_shell = self.shell.as_spur() start = time.time() number_of_files = 4 size = 1024 * 1024 * 2 content = size * "hello" for i in range(0, number_of_files): pth = tmpdir.path / '{}.txt'.format(i) with spur_shell.open(name=pth.as_posix(), mode='wt') as fid: fid.write(content) their_duration = time.time() - start # re-use sftp client start = time.time() for i in range(0, number_of_files): pth = tmpdir.path / '{}.txt'.format(i) self.shell.write_text(remote_path=pth, text=content, consistent=False, create_directories=False) our_duration = time.time() - start speedup = their_duration / our_duration self.assertGreater(speedup, 10.0)
def test_is_symlink(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth_to_dir = tmpdir.path / "some-dir" pth_to_file = tmpdir.path / "some-dir/some-file" pth_to_file_link = tmpdir.path / "some-dir/some-link-to-file" pth_to_dir_link = tmpdir.path / "some-link-to-dir" pth_to_nonexisting = tmpdir.path / "some-non-existing-file" self.shell.mkdir(remote_path=pth_to_dir) self.shell.write_text(remote_path=pth_to_file, text="hello") self.shell.run(command=['ln', '-s', pth_to_file.as_posix(), pth_to_file_link.as_posix()]) self.shell.run(command=['ln', '-s', pth_to_dir.as_posix(), pth_to_dir_link.as_posix()]) self.assertFalse(self.shell.is_symlink(pth_to_dir)) self.assertTrue(self.shell.is_symlink(pth_to_dir_link)) pth_to_dir.is_symlink() self.assertFalse(self.shell.is_symlink(pth_to_file)) self.assertTrue(self.shell.is_symlink(pth_to_file_link)) self.assertFalse(self.shell.is_symlink(pth_to_dir)) self.assertTrue(self.shell.is_symlink(pth_to_dir_link)) notfounderr = None # type: Optional[FileNotFoundError] try: self.shell.is_symlink(remote_path=pth_to_nonexisting) except FileNotFoundError as err: notfounderr = err self.assertEqual("Remote file does not exist: {}".format(pth_to_nonexisting.as_posix()), str(notfounderr))
def test_symlink(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth_to_dir = tmpdir.path / "some-dir" pth_to_file = tmpdir.path / "some-file" pth_to_file_link = tmpdir.path / "some-link-to-file" pth_to_dir_link = tmpdir.path / "some-link-to-dir" pth_to_another_file = tmpdir.path / "another-file" self.shell.mkdir(remote_path=pth_to_dir) self.shell.write_text(remote_path=pth_to_file, text="hello") self.shell.write_text(remote_path=pth_to_another_file, text="zdravo") self.shell.symlink(source=pth_to_file, destination=pth_to_file_link) self.shell.symlink(source=pth_to_dir, destination=pth_to_dir_link) self.assertFalse(self.shell.is_symlink(pth_to_file)) self.assertTrue(self.shell.is_symlink(pth_to_file_link)) self.assertFalse(self.shell.is_symlink(pth_to_dir)) self.assertTrue(self.shell.is_symlink(pth_to_dir_link)) self.assertEqual("hello", self.shell.read_text(remote_path=pth_to_file_link)) # Overwriting a link is not possible. file_exists_err = None # type: Optional[FileExistsError] try: self.shell.symlink(source=pth_to_another_file, destination=pth_to_file_link) except FileExistsError as err: file_exists_err = err self.assertIsNotNone(file_exists_err) self.assertEqual("The destination of the symbolic link already exists: {}".format( pth_to_file_link.as_posix()), str(file_exists_err))
def test_put_no_permission(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir: with temppathlib.TemporaryDirectory() as local_tmpdir: local_pth = local_tmpdir.path / 'file.txt' local_pth.write_text("hello") remote_pth = remote_tmpdir.path / 'file.txt' self.shell.put(local_path=local_pth, remote_path=remote_pth) self.shell.chmod(remote_path=remote_pth, mode=0o444) # consistent put succeeds even though the remote path has read-only permissions. self.shell.put(local_path=local_pth, remote_path=remote_pth, consistent=True) a_stat = self.shell.stat(remote_path=remote_pth.as_posix()) self.assertEqual(0o100444, a_stat.st_mode) # direct put fails since we can not write to the file. with self.assertRaises(PermissionError): self.shell.put(local_path=local_pth, remote_path=remote_pth, consistent=False) # consistent put fails if we don't have write permissions to the directory try: self.shell.chmod(remote_path=remote_tmpdir.path, mode=0o444) with self.assertRaises(PermissionError): self.shell.put(local_path=local_pth, remote_path=remote_pth, consistent=True) finally: self.shell.chmod(remote_path=remote_tmpdir.path, mode=0o777)
def test_md5_versus_md5s(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: remote_pths = [] # type: List[pathlib.Path] for i in range(0, 128): pth = tmpdir.path / "{}.txt".format(i) remote_pths.append(pth) if i % 2 == 0: self.shell.write_text(remote_path=pth, text="hello") start = time.time() md5s = self.shell.md5s(remote_paths=remote_pths) md5s_duration = time.time() - start # now benchmark the manual implementation start = time.time() result = [] # type: List[Optional[str]] for remote_pth in remote_pths: if self.shell.exists(remote_path=remote_pth): result.append(self.shell.md5(remote_path=remote_pth)) else: result.append(None) manual_duration = time.time() - start self.assertListEqual(md5s, result) speedup = manual_duration / md5s_duration self.assertGreaterEqual(speedup, 10.0)
def test_exists(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth = tmpdir.path / "some-dir" / "oi" self.assertFalse(self.shell.exists(remote_path=pth)) self.shell.write_text(remote_path=pth, text="hello") self.assertTrue(self.shell.exists(remote_path=pth))
def test_mkdir(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth = tmpdir.path / "some-dir" / "oi" self.shell.mkdir(remote_path=pth, mode=0o700, parents=True) self.assertTrue(self.shell.exists(remote_path=pth)) # check with SFTP a_stat = self.shell.stat(pth.as_posix()) self.assertEqual(0o40700, a_stat.st_mode) # does nothing self.shell.mkdir(remote_path=pth, mode=0o700, parents=True, exist_ok=True) # existing directory raises an error on exist_ok=False existserr = None # type: Optional[FileExistsError] try: self.shell.mkdir(remote_path=pth, mode=0o700, parents=True, exist_ok=False) except FileExistsError as err: existserr = err self.assertEqual("The remote directory already exists: {}".format(pth), str(existserr)) # existing directory does not raise an error on exist_ok=True self.shell.mkdir(remote_path=pth, mode=0o700, exist_ok=True)
def test_write_read_text(self): for consistent in [True, False]: with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth = tmpdir.path / "some-dir" / str(consistent) self.shell.write_text(remote_path=pth, text="hello", consistent=consistent) text = self.shell.read_text(remote_path=pth) self.assertEqual("hello", text)
def test_md5_with_space(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth = tmpdir.path / "oi hoi" self.shell.write_text(remote_path=pth, text="hello") md5digest = self.shell.md5(remote_path=pth) expected = hashlib.md5("hello".encode()).hexdigest() self.assertEqual(expected, md5digest)
def test_mkdir(self) -> None: with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth_to_folder = tmpdir.path / "some-folder" self.assertFalse(self.shell.exists(remote_path=pth_to_folder)) self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access self.reconnecting_sftp.mkdir(path=pth_to_folder.as_posix()) self.assertTrue(self.shell.exists(remote_path=pth_to_folder))
def test_local_only_file(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_pth_to_file = local_tmpdir.path / "some-file" local_pth_to_file.write_text("hello") self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path) remote_pth_to_file = remote_tmpdir.path / "some-file" self.assertTrue(self.shell.exists(remote_path=remote_pth_to_file)) self.assertEqual("hello", self.shell.read_text(remote_path=remote_pth_to_file))
def test_local_only_directory(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_dir = local_tmpdir.path / "some-dir" local_dir.mkdir() self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path) remote_dir = remote_tmpdir.path / "some-dir" self.assertTrue(self.shell.exists(remote_path=remote_dir)) self.assertTrue(self.shell.is_dir(remote_path=remote_dir))
def test_read_bytes_no_permission(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir: pth = remote_tmpdir.path / "some-file" self.shell.write_bytes(remote_path=pth, data=b"hello") try: self.shell.chmod(remote_path=pth, mode=0o111) with self.assertRaises(PermissionError): self.shell.read_bytes(remote_path=pth) finally: self.shell.chmod(remote_path=pth, mode=0o777)
def test_read_bytes_not_found(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir: pth = remote_tmpdir.path / "some-file" notfounderr = None # type: Optional[FileNotFoundError] try: self.shell.read_bytes(remote_path=pth) except FileNotFoundError as err: notfounderr = err self.assertEqual("The remote path was not found: {}".format(pth.as_posix()), str(notfounderr))
def test_remote_only_file_is_deleted(self): # pylint: disable=invalid-name for delete in [spurplus.Delete.BEFORE, spurplus.Delete.AFTER]: with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: remote_pth_to_file = remote_tmpdir.path / "some-file" self.shell.write_text(remote_path=remote_pth_to_file, text="hello") self.assertTrue(self.shell.exists(remote_path=remote_pth_to_file)) self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path, delete=delete) self.assertFalse(self.shell.exists(remote_path=remote_pth_to_file))
def test_mkdir_with_permission_error(self): # pylint: disable=invalid-name with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: try: self.shell.chmod(remote_path=tmpdir.path, mode=0o444) pth = tmpdir.path / "some-dir" / "oi" with self.assertRaises(PermissionError): self.shell.mkdir(remote_path=pth, mode=0o700, parents=True) finally: self.shell.chmod(remote_path=tmpdir.path, mode=0o777)
def test_nonexisting_local_path(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_pth = local_tmpdir.path / "some-dir" notfounderr = None # type: Optional[FileNotFoundError] try: self.shell.sync_to_remote(local_path=local_pth, remote_path=remote_tmpdir.path) except FileNotFoundError as err: notfounderr = err self.assertEqual("Local path does not exist: {}".format(local_pth), str(notfounderr))
def test_local_path_is_not_a_dir(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_pth_to_file = local_tmpdir.path / "some-file" local_pth_to_file.write_text("hello") direrr = None # type: Optional[NotADirectoryError] try: self.shell.sync_to_remote(local_path=local_pth_to_file, remote_path=remote_tmpdir.path) except NotADirectoryError as err: direrr = err self.assertEqual("Local path is not a directory: {}".format(local_pth_to_file), str(direrr))
def test_no_local_and_remote_dir(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir: local_tmpdir = pathlib.Path("nonexisting-local-dir") remote_tmpdir = remote_tmpdir.path / "nonexisting-remote-dir" notfounderr = None # type: Optional[FileNotFoundError] try: self.shell.directory_diff(local_path=local_tmpdir, remote_path=remote_tmpdir) except FileNotFoundError as err: notfounderr = err self.assertEqual("Both the local and the remote path do not exist: {} and {}".format( local_tmpdir, remote_tmpdir), str(notfounderr))
def test_remote_only_dir_is_deleted(self): for delete in [spurplus.Delete.BEFORE, spurplus.Delete.AFTER]: with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: remote_pth_to_dir = remote_tmpdir.path / "some-dir" self.shell.mkdir(remote_path=remote_pth_to_dir) self.shell.write_text(remote_path=remote_pth_to_dir / "some-file", text="hello") self.assertTrue(self.shell.exists(remote_path=remote_pth_to_dir)) self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path, delete=delete) self.assertFalse(self.shell.exists(remote_path=remote_pth_to_dir))
def test_symlink(self): for recursive in [True, False]: with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth_to_file = tmpdir.path / "some-file" self.shell.write_text(remote_path=pth_to_file, text="hello") pth_to_file_link = tmpdir.path / "some-link-to-file" self.shell.symlink(source=pth_to_file, destination=pth_to_file_link) self.assertTrue(self.shell.exists(remote_path=pth_to_file)) self.assertTrue(self.shell.exists(remote_path=pth_to_file_link)) self.shell.remove(remote_path=pth_to_file_link, recursive=recursive) self.assertTrue(self.shell.exists(remote_path=pth_to_file)) self.assertFalse(self.shell.exists(remote_path=pth_to_file_link))
def test_local_dir_permission_invalid_pth(self): # pylint: disable=invalid-name with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_dir = local_tmpdir.path / "some-nonexisting-dir" notfounderr = None # type: Optional[FileNotFoundError] try: self.shell.mirror_local_permissions( relative_paths=[pathlib.Path("some-relative-dir")], local_path=local_dir, remote_path=remote_tmpdir.path) except FileNotFoundError as err: notfounderr = err self.assertEqual("Local path does not exist: {}".format(local_dir), str(notfounderr)) # file instead of dir as local path local_file = local_tmpdir.path / "some-file" local_file.write_text("hello") local_direrr = None # type: Optional[NotADirectoryError] try: self.shell.mirror_local_permissions( relative_paths=[pathlib.Path("some-relative-dir")], local_path=local_file, remote_path=remote_tmpdir.path) except NotADirectoryError as err: local_direrr = err self.assertEqual("Local path is not a directory: {}".format(local_file), str(local_direrr)) # file instead of dir as remote path remote_file = remote_tmpdir.path / "some-file" self.shell.write_text(remote_path=remote_file, text="hello") remote_direrr = None # type: Optional[NotADirectoryError] try: self.shell.mirror_local_permissions( relative_paths=[pathlib.Path("some-relative-dir")], local_path=local_tmpdir.path, remote_path=remote_file) except NotADirectoryError as err: remote_direrr = err self.assertEqual("Remote path is not a directory: {}".format(remote_file), str(remote_direrr))
def test_posix_rename_file(self) -> None: with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth_to_file = tmpdir.path / "some-file" new_pth_to_file = tmpdir.path / "renamed-file" self.shell.write_text(remote_path=pth_to_file, text="hello") self.assertTrue(self.shell.exists(remote_path=pth_to_file)) self.assertFalse(self.shell.exists(remote_path=new_pth_to_file)) self.reconnecting_sftp._sftp.sock.close() # pylint: disable=protected-access self.reconnecting_sftp.posix_rename( oldpath=pth_to_file.as_posix(), newpath=new_pth_to_file.as_posix()) self.assertFalse(self.shell.exists(remote_path=pth_to_file)) self.assertTrue(self.shell.exists(remote_path=new_pth_to_file))
def test_non_empty_dir(self): with spurplus.TemporaryDirectory(shell=self.shell) as tmpdir: pth_to_some_dir = tmpdir.path / "some-dir" pth_to_subdir = tmpdir.path / "some-dir/some-subdir" self.shell.mkdir(remote_path=pth_to_subdir, parents=True) os_err = None # type: Optional[OSError] try: self.shell.remove(remote_path=pth_to_some_dir) except OSError as err: os_err = err self.assertIsNotNone(os_err) self.assertEqual("The remote directory is not empty and the recursive flag was not set: {}".format( pth_to_some_dir.as_posix()), str(os_err))
def test_local_only_link(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_pth_to_file = local_tmpdir.path / "some-file" local_pth_to_link = local_tmpdir.path / "some-link" local_pth_to_file.write_text("hello") local_pth_to_link.symlink_to(local_pth_to_file) self.shell.sync_to_remote(local_path=local_tmpdir.path, remote_path=remote_tmpdir.path) # Check that the symlinks are copied as files, not as links remote_pth_to_link = remote_tmpdir.path / "some-link" self.assertFalse(self.shell.is_symlink(remote_path=remote_pth_to_link)) self.assertTrue(self.shell.exists(remote_path=remote_pth_to_link)) self.assertEqual("hello", self.shell.read_text(remote_path=remote_pth_to_link))
def test_local_permissions(self): with spurplus.TemporaryDirectory(shell=self.shell) as remote_tmpdir, \ temppathlib.TemporaryDirectory() as local_tmpdir: local_pth = local_tmpdir.path / "some-dir/some-file" local_pth.parent.mkdir() local_pth.write_text("hello") local_pth.chmod(0o612) remote_pth = remote_tmpdir.path / "some-dir/some-file" self.shell.mkdir(remote_path=remote_pth.parent) self.shell.write_text(remote_path=remote_pth, text="hello") self.shell.mirror_local_permissions( relative_paths=[pathlib.Path("some-dir/some-file")], local_path=local_tmpdir.path, remote_path=remote_tmpdir.path)