def _open_fs(self, user_context):
props = self._serialization_props(user_context)
path = props.pop('path')
handle = SSHFS(**props)
if path:
handle = handle.opendir(path)
return handle
def filename_to_issue(filename):
"""
Given a filename, load it from GPFS and parse it into an Issue
"""
fs = SSHFS(host=DATA_STORE_HOST, user=username)
stream = fs.open(filename, 'r', encoding='latin_1')
issue = Issue(stream)
stream.close()
fs.close()
return issue
def test_sshconfig_notfound(self):
ssh_fs = SSHFS('localhost',
self.user,
self.pasw,
port=self.port,
config_path='zzzz')
self.assertFunctional(ssh_fs)
def test_sshconfig_override(self):
with open(self.config_file, 'w') as conf:
conf.writelines([
"Host test_host\n", " Hostname localhost\n",
" User no_one\n",
" Port {}\n".format(self.port),
" IdentityFile {}\n".format(self.key_file),
" IdentitiesOnly yes\n"
])
ssh_fs = SSHFS('test_host', self.user, config_path=self.config_file)
self.assertFunctional(ssh_fs)
def get_lxplus_fs(args):
assert args.user is not None, "Need username for lxplus"
assert args.pwd is not None, "Neew password for lxplus"
with Spinner(text="Connecting to lxplus", persist=False):
return SSHFS(
host="lxplus.cern.ch",
user=args.user,
passwd=args.pwd,
allow_agent=False,
look_for_keys=False,
)
def test_publickey_file(self):
ssh_fs = SSHFS('localhost',
self.user,
port=self.port,
pkey=self.key_file)
self.assertFunctional(ssh_fs)
def test_password(self):
ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port)
self.assertFunctional(ssh_fs)
def make_fs(self):
self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port)
self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex)
self.ssh_fs.makedir(self.test_folder, recreate=True)
return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS)
class TestSSHFS(fs.test.FSTestCases, unittest.TestCase):
user = "******"
pasw = "pass"
port = 2222
@classmethod
def setUpClass(cls):
super(TestSSHFS, cls).setUpClass()
cls.sftp_container = utils.startServer(utils.docker_client, cls.user,
cls.pasw, cls.port)
@classmethod
def tearDownClass(cls):
utils.stopServer(cls.sftp_container)
super(TestSSHFS, cls).tearDownClass()
@staticmethod
def destroy_fs(fs):
fs.close()
del fs
def make_fs(self):
self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port)
self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex)
self.ssh_fs.makedir(self.test_folder, recreate=True)
return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS)
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_0(self):
super(TestSSHFS, self).test_download_0()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_1(self):
super(TestSSHFS, self).test_download_1()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_2(self):
super(TestSSHFS, self).test_download_2()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_4(self):
super(TestSSHFS, self).test_download_4()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_0(self):
super(TestSSHFS, self).test_upload_0()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_1(self):
super(TestSSHFS, self).test_upload_1()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_2(self):
super(TestSSHFS, self).test_upload_2()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_4(self):
super(TestSSHFS, self).test_upload_4()
def test_chmod(self):
self.fs.touch("test.txt")
remote_path = fs.path.join(self.test_folder, "test.txt")
# Initial permissions
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o644)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o644)
# Change permissions with SSHFS._chown
self.fs.delegate_fs()._chmod(remote_path, 0o744)
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o744)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o744)
# Change permissions with SSHFS.setinfo
self.fs.setinfo("test.txt",
{"access": {
"permissions": Permissions(mode=0o600)
}})
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o600)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o600)
with self.assertRaises(fs.errors.PermissionDenied):
self.fs.delegate_fs().setinfo(
"/", {"access": {
"permissions": Permissions(mode=0o777)
}})
def test_chown(self):
self.fs.touch("test.txt")
remote_path = fs.path.join(self.test_folder, "test.txt")
info = self.fs.getinfo("test.txt", namespaces=["access"])
gid, uid = info.get('access', 'uid'), info.get('access', 'gid')
with utils.mock.patch.object(self.fs.delegate_fs()._sftp,
'chown') as chown:
self.fs.setinfo("test.txt", {'access': {'uid': None}})
chown.assert_called_with(remote_path, uid, gid)
self.fs.setinfo("test.txt", {'access': {'gid': None}})
chown.assert_called_with(remote_path, uid, gid)
self.fs.setinfo("test.txt", {'access': {'gid': 8000}})
chown.assert_called_with(remote_path, uid, 8000)
self.fs.setinfo("test.txt", {'access': {'uid': 1001, 'gid': 1002}})
chown.assert_called_with(remote_path, 1001, 1002)
def test_utime(self):
def get_accessed(f):
return f.getdetails("test.txt").get('details', 'accessed')
def get_modified(f):
return f.getdetails("test.txt").get('details', 'modified')
self.fs.touch("test.txt")
self.fs.setinfo("test.txt",
{'details': {
'accessed': None,
'modified': None
}})
self.assertLessEqual(time.time() - get_accessed(self.fs), 2)
self.assertLessEqual(time.time() - get_modified(self.fs), 2)
self.fs.setinfo("test.txt", {'details': {'accessed': 0}})
self.assertEqual(get_accessed(self.fs), 0)
self.assertEqual(get_modified(self.fs), 0)
self.fs.setinfo("test.txt", {'details': {'modified': 100}})
self.assertEqual(get_accessed(self.fs), 100)
self.assertEqual(get_modified(self.fs), 100)
self.fs.setinfo("test.txt",
{'details': {
'modified': 100,
'accessed': 200
}})
self.assertEqual(get_accessed(self.fs), 200)
self.assertEqual(get_modified(self.fs), 100)
def _open_fs(self, user_context):
props = self._serialization_props(user_context)
handle = SSHFS(**props)
return handle
class TestSSHFS(fs.test.FSTestCases, unittest.TestCase):
user = "******"
pasw = "pass"
port = 2222
@classmethod
def setUpClass(cls):
super(TestSSHFS, cls).setUpClass()
cls.sftp_container = utils.startServer(utils.docker_client, cls.user,
cls.pasw, cls.port)
@classmethod
def tearDownClass(cls):
utils.stopServer(cls.sftp_container)
super(TestSSHFS, cls).tearDownClass()
@staticmethod
def destroy_fs(fs):
fs.close()
del fs
def make_fs(self):
self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port)
self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex)
self.ssh_fs.makedir(self.test_folder, recreate=True)
return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS)
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_0(self):
super(TestSSHFS, self).test_download_0()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_1(self):
super(TestSSHFS, self).test_download_1()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_2(self):
super(TestSSHFS, self).test_download_2()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_4(self):
super(TestSSHFS, self).test_download_4()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_0(self):
super(TestSSHFS, self).test_upload_0()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_1(self):
super(TestSSHFS, self).test_upload_1()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_2(self):
super(TestSSHFS, self).test_upload_2()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_4(self):
super(TestSSHFS, self).test_upload_4()
def test_chmod(self):
self.fs.touch("test.txt")
remote_path = fs.path.join(self.test_folder, "test.txt")
# Initial permissions
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o644)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o644)
# Change permissions with SSHFS._chown
self.fs.delegate_fs()._chmod(remote_path, 0o744)
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o744)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o744)
# Change permissions with SSHFS.setinfo
self.fs.setinfo("test.txt",
{"access": {
"permissions": Permissions(mode=0o600)
}})
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o600)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o600)
with self.assertRaises(fs.errors.PermissionDenied):
self.fs.delegate_fs().setinfo(
"/", {"access": {
"permissions": Permissions(mode=0o777)
}})
def test_chown(self):
self.fs.touch("test.txt")
remote_path = fs.path.join(self.test_folder, "test.txt")
info = self.fs.getinfo("test.txt", namespaces=["access"])
gid, uid = info.get('access', 'uid'), info.get('access', 'gid')
with utils.mock.patch.object(self.fs.delegate_fs()._sftp,
'chown') as chown:
self.fs.setinfo("test.txt", {'access': {'uid': None}})
chown.assert_called_with(remote_path, uid, gid)
self.fs.setinfo("test.txt", {'access': {'gid': None}})
chown.assert_called_with(remote_path, uid, gid)
self.fs.setinfo("test.txt", {'access': {'gid': 8000}})
chown.assert_called_with(remote_path, uid, 8000)
self.fs.setinfo("test.txt", {'access': {'uid': 1001, 'gid': 1002}})
chown.assert_called_with(remote_path, 1001, 1002)
def test_exec_command_exception(self):
ssh = self.fs.delegate_fs()
# make sure to invalidate the cache
ssh.platform
del ssh.platform
# pretend we get an error while the platform is guessed
with utils.mock.patch.object(ssh,
'_exec_command',
side_effect=paramiko.ssh_exception.
SSHException()) as _exec_command:
self.assertEquals(ssh.platform, "unknown")
if sys.version_info[:2] != (3, 5):
_exec_command.assert_called()
def test_utime(self):
def get_accessed(f):
return f.getdetails("test.txt").get('details', 'accessed')
def get_modified(f):
return f.getdetails("test.txt").get('details', 'modified')
self.fs.touch("test.txt")
self.fs.setinfo("test.txt",
{'details': {
'accessed': None,
'modified': None
}})
self.assertLessEqual(time.time() - get_accessed(self.fs), 2)
self.assertLessEqual(time.time() - get_modified(self.fs), 2)
self.fs.setinfo("test.txt", {'details': {'accessed': 0}})
self.assertEqual(get_accessed(self.fs), 0)
self.assertEqual(get_modified(self.fs), 0)
self.fs.setinfo("test.txt", {'details': {'modified': 100}})
self.assertEqual(get_accessed(self.fs), 100)
self.assertEqual(get_modified(self.fs), 100)
self.fs.setinfo("test.txt",
{'details': {
'modified': 100,
'accessed': 200
}})
self.assertEqual(get_accessed(self.fs), 200)
self.assertEqual(get_modified(self.fs), 100)
def test_symlinks(self):
with self.fs.openbin("foo", "wb") as f:
f.write(b"foobar")
self.fs.delegate_fs()._sftp.symlink(
fs.path.join(self.test_folder, "foo"),
fs.path.join(self.test_folder, "bar"))
# os.symlink(self._get_real_path("foo"), self._get_real_path("bar"))
self.assertFalse(self.fs.islink("foo"))
self.assertFalse(self.fs.getinfo("foo", namespaces=["link"]).is_link)
self.assertTrue(self.fs.islink("bar"))
self.assertTrue(self.fs.getinfo("bar", namespaces=["link"]).is_link)
foo_info = self.fs.getinfo("foo", namespaces=["link", "lstat"])
self.assertIn("link", foo_info.raw)
self.assertIn("lstat", foo_info.raw)
self.assertEqual(foo_info.get("link", "target"), None)
self.assertEqual(foo_info.target, foo_info.raw["link"]["target"])
bar_info = self.fs.getinfo("bar", namespaces=["link", "lstat"])
self.assertIn("link", bar_info.raw)
self.assertIn("lstat", bar_info.raw)
def make_ssfs(url):
"""Construct SSHFS from url."""
if url.scheme != "sftp":
raise ValueError("Scheme must be == 'sftp'")
return SSHFS(host=url.hostname, user=url.username, passwd=url.password, port=(url.port or 22))
def _process(self, data):
# find already processed files
logging.info('Processing data at ' + self.__class__.__name__)
done = {}
retries = self.__retries + 1
while retries > 0:
retries -= 1
if self.__register and os.path.exists(self.__register):
try:
with open(self.__register, 'r') as f:
for line in csv.reader(f):
if len(line) == 2:
done[line[1]] = int(line[0])
else:
logging.warning("Bad line in register file %s : %s" % (self.__register, str(line)))
except Exception as x:
logging.error(x)
path = ''
if self.__url:
home_fs = open_fs(self.__url)
elif self.__con:
c = self.__con
if c['type'] == 'ssh':
from fs.sshfs import SSHFS
if 'port' not in c: c['port'] = 22
if 'passwd' not in c: c['passwd'] = None
if 'pkey' not in c: c['pkey'] = None
if 'path' in c: path = c['path']
home_fs = SSHFS(host=c['host'],
user=c['user'], passwd=c['passwd'],
pkey=c['pkey'], port=c['port'])
else:
raise ValueError("Unsupported type of connection")
with home_fs:
if self.__walk:
for _p, f in home_fs.walk.info(path, filter=[self.__filter], namespaces=['details']):
if f.is_dir:
continue
fn = os.path.join(path, _p)
modified = int(f.modified.timestamp())
if fn in done:
if done[fn] >= modified:
continue
self.__ls[fn] = modified
else:
# list files in the specified path
iter = home_fs.filterdir(path, files=[self.__filter], namespaces=['details'])
for f in filter(lambda f: not f.is_dir, iter):
fn = os.path.join(path, f.name)
modified = int(f.modified.timestamp())
if fn in done:
if done[fn] >= modified:
continue
self.__ls[fn] = modified
if len(self.__ls) > 0:
logging.info("Found %d files." % len(self.__ls))
for fn, ts in self.__ls.items():
try:
logging.info("Opening file %s." % fn)
with home_fs.open(fn) as _file:
meta = {'ts': ts, 'name': fn}
yield meta, _file
# add to the already done files
if self.__register:
with open(self.__register, "a") as regf:
regf.write("%s,%s\n" % (str(ts), fn))
except Exception as x:
logging.error("Error processing file %s. %s" % (fn, str(x)))
else:
logging.info("No new file found !")
if retries == 0:
break
else:
logging.info("Retrying after %d seconds." % self.__retry_delay)
time.sleep(self.__retry_delay)
