def _open_fs(self, user_context): props = self._serialization_props(user_context) path = props.pop('path') handle = SSHFS(**props) if path: handle = handle.opendir(path) return handle
def filename_to_issue(filename): """ Given a filename, load it from GPFS and parse it into an Issue """ fs = SSHFS(host=DATA_STORE_HOST, user=username) stream = fs.open(filename, 'r', encoding='latin_1') issue = Issue(stream) stream.close() fs.close() return issue
def test_sshconfig_notfound(self): ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port, config_path='zzzz') self.assertFunctional(ssh_fs)
def test_sshconfig_override(self): with open(self.config_file, 'w') as conf: conf.writelines([ "Host test_host\n", " Hostname localhost\n", " User no_one\n", " Port {}\n".format(self.port), " IdentityFile {}\n".format(self.key_file), " IdentitiesOnly yes\n" ]) ssh_fs = SSHFS('test_host', self.user, config_path=self.config_file) self.assertFunctional(ssh_fs)
def get_lxplus_fs(args): assert args.user is not None, "Need username for lxplus" assert args.pwd is not None, "Neew password for lxplus" with Spinner(text="Connecting to lxplus", persist=False): return SSHFS( host="lxplus.cern.ch", user=args.user, passwd=args.pwd, allow_agent=False, look_for_keys=False, )
def test_publickey_file(self): ssh_fs = SSHFS('localhost', self.user, port=self.port, pkey=self.key_file) self.assertFunctional(ssh_fs)
def test_password(self): ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port) self.assertFunctional(ssh_fs)
def make_fs(self): self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port) self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex) self.ssh_fs.makedir(self.test_folder, recreate=True) return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS)
class TestSSHFS(fs.test.FSTestCases, unittest.TestCase): user = "******" pasw = "pass" port = 2222 @classmethod def setUpClass(cls): super(TestSSHFS, cls).setUpClass() cls.sftp_container = utils.startServer(utils.docker_client, cls.user, cls.pasw, cls.port) @classmethod def tearDownClass(cls): utils.stopServer(cls.sftp_container) super(TestSSHFS, cls).tearDownClass() @staticmethod def destroy_fs(fs): fs.close() del fs def make_fs(self): self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port) self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex) self.ssh_fs.makedir(self.test_folder, recreate=True) return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS) @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_0(self): super(TestSSHFS, self).test_download_0() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_1(self): super(TestSSHFS, self).test_download_1() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_2(self): super(TestSSHFS, self).test_download_2() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_4(self): super(TestSSHFS, self).test_download_4() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_0(self): super(TestSSHFS, self).test_upload_0() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_1(self): super(TestSSHFS, self).test_upload_1() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_2(self): super(TestSSHFS, self).test_upload_2() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_4(self): super(TestSSHFS, self).test_upload_4() def test_chmod(self): self.fs.touch("test.txt") remote_path = fs.path.join(self.test_folder, "test.txt") # Initial permissions info = self.fs.getinfo("test.txt", ["access"]) self.assertEqual(info.permissions.mode, 0o644) st = self.fs.delegate_fs()._sftp.stat(remote_path) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) # Change permissions with SSHFS._chown self.fs.delegate_fs()._chmod(remote_path, 0o744) info = self.fs.getinfo("test.txt", ["access"]) self.assertEqual(info.permissions.mode, 0o744) st = self.fs.delegate_fs()._sftp.stat(remote_path) self.assertEqual(stat.S_IMODE(st.st_mode), 0o744) # Change permissions with SSHFS.setinfo self.fs.setinfo("test.txt", {"access": { "permissions": Permissions(mode=0o600) }}) info = self.fs.getinfo("test.txt", ["access"]) self.assertEqual(info.permissions.mode, 0o600) st = self.fs.delegate_fs()._sftp.stat(remote_path) self.assertEqual(stat.S_IMODE(st.st_mode), 0o600) with self.assertRaises(fs.errors.PermissionDenied): self.fs.delegate_fs().setinfo( "/", {"access": { "permissions": Permissions(mode=0o777) }}) def test_chown(self): self.fs.touch("test.txt") remote_path = fs.path.join(self.test_folder, "test.txt") info = self.fs.getinfo("test.txt", namespaces=["access"]) gid, uid = info.get('access', 'uid'), info.get('access', 'gid') with utils.mock.patch.object(self.fs.delegate_fs()._sftp, 'chown') as chown: self.fs.setinfo("test.txt", {'access': {'uid': None}}) chown.assert_called_with(remote_path, uid, gid) self.fs.setinfo("test.txt", {'access': {'gid': None}}) chown.assert_called_with(remote_path, uid, gid) self.fs.setinfo("test.txt", {'access': {'gid': 8000}}) chown.assert_called_with(remote_path, uid, 8000) self.fs.setinfo("test.txt", {'access': {'uid': 1001, 'gid': 1002}}) chown.assert_called_with(remote_path, 1001, 1002) def test_utime(self): def get_accessed(f): return f.getdetails("test.txt").get('details', 'accessed') def get_modified(f): return f.getdetails("test.txt").get('details', 'modified') self.fs.touch("test.txt") self.fs.setinfo("test.txt", {'details': { 'accessed': None, 'modified': None }}) self.assertLessEqual(time.time() - get_accessed(self.fs), 2) self.assertLessEqual(time.time() - get_modified(self.fs), 2) self.fs.setinfo("test.txt", {'details': {'accessed': 0}}) self.assertEqual(get_accessed(self.fs), 0) self.assertEqual(get_modified(self.fs), 0) self.fs.setinfo("test.txt", {'details': {'modified': 100}}) self.assertEqual(get_accessed(self.fs), 100) self.assertEqual(get_modified(self.fs), 100) self.fs.setinfo("test.txt", {'details': { 'modified': 100, 'accessed': 200 }}) self.assertEqual(get_accessed(self.fs), 200) self.assertEqual(get_modified(self.fs), 100)
def _open_fs(self, user_context): props = self._serialization_props(user_context) handle = SSHFS(**props) return handle
class TestSSHFS(fs.test.FSTestCases, unittest.TestCase): user = "******" pasw = "pass" port = 2222 @classmethod def setUpClass(cls): super(TestSSHFS, cls).setUpClass() cls.sftp_container = utils.startServer(utils.docker_client, cls.user, cls.pasw, cls.port) @classmethod def tearDownClass(cls): utils.stopServer(cls.sftp_container) super(TestSSHFS, cls).tearDownClass() @staticmethod def destroy_fs(fs): fs.close() del fs def make_fs(self): self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port) self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex) self.ssh_fs.makedir(self.test_folder, recreate=True) return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS) @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_0(self): super(TestSSHFS, self).test_download_0() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_1(self): super(TestSSHFS, self).test_download_1() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_2(self): super(TestSSHFS, self).test_download_2() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_download_4(self): super(TestSSHFS, self).test_download_4() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_0(self): super(TestSSHFS, self).test_upload_0() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_1(self): super(TestSSHFS, self).test_upload_1() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_2(self): super(TestSSHFS, self).test_upload_2() @unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4') def test_upload_4(self): super(TestSSHFS, self).test_upload_4() def test_chmod(self): self.fs.touch("test.txt") remote_path = fs.path.join(self.test_folder, "test.txt") # Initial permissions info = self.fs.getinfo("test.txt", ["access"]) self.assertEqual(info.permissions.mode, 0o644) st = self.fs.delegate_fs()._sftp.stat(remote_path) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) # Change permissions with SSHFS._chown self.fs.delegate_fs()._chmod(remote_path, 0o744) info = self.fs.getinfo("test.txt", ["access"]) self.assertEqual(info.permissions.mode, 0o744) st = self.fs.delegate_fs()._sftp.stat(remote_path) self.assertEqual(stat.S_IMODE(st.st_mode), 0o744) # Change permissions with SSHFS.setinfo self.fs.setinfo("test.txt", {"access": { "permissions": Permissions(mode=0o600) }}) info = self.fs.getinfo("test.txt", ["access"]) self.assertEqual(info.permissions.mode, 0o600) st = self.fs.delegate_fs()._sftp.stat(remote_path) self.assertEqual(stat.S_IMODE(st.st_mode), 0o600) with self.assertRaises(fs.errors.PermissionDenied): self.fs.delegate_fs().setinfo( "/", {"access": { "permissions": Permissions(mode=0o777) }}) def test_chown(self): self.fs.touch("test.txt") remote_path = fs.path.join(self.test_folder, "test.txt") info = self.fs.getinfo("test.txt", namespaces=["access"]) gid, uid = info.get('access', 'uid'), info.get('access', 'gid') with utils.mock.patch.object(self.fs.delegate_fs()._sftp, 'chown') as chown: self.fs.setinfo("test.txt", {'access': {'uid': None}}) chown.assert_called_with(remote_path, uid, gid) self.fs.setinfo("test.txt", {'access': {'gid': None}}) chown.assert_called_with(remote_path, uid, gid) self.fs.setinfo("test.txt", {'access': {'gid': 8000}}) chown.assert_called_with(remote_path, uid, 8000) self.fs.setinfo("test.txt", {'access': {'uid': 1001, 'gid': 1002}}) chown.assert_called_with(remote_path, 1001, 1002) def test_exec_command_exception(self): ssh = self.fs.delegate_fs() # make sure to invalidate the cache ssh.platform del ssh.platform # pretend we get an error while the platform is guessed with utils.mock.patch.object(ssh, '_exec_command', side_effect=paramiko.ssh_exception. SSHException()) as _exec_command: self.assertEquals(ssh.platform, "unknown") if sys.version_info[:2] != (3, 5): _exec_command.assert_called() def test_utime(self): def get_accessed(f): return f.getdetails("test.txt").get('details', 'accessed') def get_modified(f): return f.getdetails("test.txt").get('details', 'modified') self.fs.touch("test.txt") self.fs.setinfo("test.txt", {'details': { 'accessed': None, 'modified': None }}) self.assertLessEqual(time.time() - get_accessed(self.fs), 2) self.assertLessEqual(time.time() - get_modified(self.fs), 2) self.fs.setinfo("test.txt", {'details': {'accessed': 0}}) self.assertEqual(get_accessed(self.fs), 0) self.assertEqual(get_modified(self.fs), 0) self.fs.setinfo("test.txt", {'details': {'modified': 100}}) self.assertEqual(get_accessed(self.fs), 100) self.assertEqual(get_modified(self.fs), 100) self.fs.setinfo("test.txt", {'details': { 'modified': 100, 'accessed': 200 }}) self.assertEqual(get_accessed(self.fs), 200) self.assertEqual(get_modified(self.fs), 100) def test_symlinks(self): with self.fs.openbin("foo", "wb") as f: f.write(b"foobar") self.fs.delegate_fs()._sftp.symlink( fs.path.join(self.test_folder, "foo"), fs.path.join(self.test_folder, "bar")) # os.symlink(self._get_real_path("foo"), self._get_real_path("bar")) self.assertFalse(self.fs.islink("foo")) self.assertFalse(self.fs.getinfo("foo", namespaces=["link"]).is_link) self.assertTrue(self.fs.islink("bar")) self.assertTrue(self.fs.getinfo("bar", namespaces=["link"]).is_link) foo_info = self.fs.getinfo("foo", namespaces=["link", "lstat"]) self.assertIn("link", foo_info.raw) self.assertIn("lstat", foo_info.raw) self.assertEqual(foo_info.get("link", "target"), None) self.assertEqual(foo_info.target, foo_info.raw["link"]["target"]) bar_info = self.fs.getinfo("bar", namespaces=["link", "lstat"]) self.assertIn("link", bar_info.raw) self.assertIn("lstat", bar_info.raw)
def make_ssfs(url): """Construct SSHFS from url.""" if url.scheme != "sftp": raise ValueError("Scheme must be == 'sftp'") return SSHFS(host=url.hostname, user=url.username, passwd=url.password, port=(url.port or 22))
def _process(self, data): # find already processed files logging.info('Processing data at ' + self.__class__.__name__) done = {} retries = self.__retries + 1 while retries > 0: retries -= 1 if self.__register and os.path.exists(self.__register): try: with open(self.__register, 'r') as f: for line in csv.reader(f): if len(line) == 2: done[line[1]] = int(line[0]) else: logging.warning("Bad line in register file %s : %s" % (self.__register, str(line))) except Exception as x: logging.error(x) path = '' if self.__url: home_fs = open_fs(self.__url) elif self.__con: c = self.__con if c['type'] == 'ssh': from fs.sshfs import SSHFS if 'port' not in c: c['port'] = 22 if 'passwd' not in c: c['passwd'] = None if 'pkey' not in c: c['pkey'] = None if 'path' in c: path = c['path'] home_fs = SSHFS(host=c['host'], user=c['user'], passwd=c['passwd'], pkey=c['pkey'], port=c['port']) else: raise ValueError("Unsupported type of connection") with home_fs: if self.__walk: for _p, f in home_fs.walk.info(path, filter=[self.__filter], namespaces=['details']): if f.is_dir: continue fn = os.path.join(path, _p) modified = int(f.modified.timestamp()) if fn in done: if done[fn] >= modified: continue self.__ls[fn] = modified else: # list files in the specified path iter = home_fs.filterdir(path, files=[self.__filter], namespaces=['details']) for f in filter(lambda f: not f.is_dir, iter): fn = os.path.join(path, f.name) modified = int(f.modified.timestamp()) if fn in done: if done[fn] >= modified: continue self.__ls[fn] = modified if len(self.__ls) > 0: logging.info("Found %d files." % len(self.__ls)) for fn, ts in self.__ls.items(): try: logging.info("Opening file %s." % fn) with home_fs.open(fn) as _file: meta = {'ts': ts, 'name': fn} yield meta, _file # add to the already done files if self.__register: with open(self.__register, "a") as regf: regf.write("%s,%s\n" % (str(ts), fn)) except Exception as x: logging.error("Error processing file %s. %s" % (fn, str(x))) else: logging.info("No new file found !") if retries == 0: break else: logging.info("Retrying after %d seconds." % self.__retry_delay) time.sleep(self.__retry_delay)