def test_sshconfig_notfound(self):
ssh_fs = SSHFS('localhost',
self.user,
self.pasw,
port=self.port,
config_path='zzzz')
self.assertFunctional(ssh_fs)
def test_sshconfig_override(self):
with open(self.config_file, 'w') as conf:
conf.writelines([
"Host test_host\n", " Hostname localhost\n",
" User no_one\n",
" Port {}\n".format(self.port),
" IdentityFile {}\n".format(self.key_file),
" IdentitiesOnly yes\n"
])
ssh_fs = SSHFS('test_host', self.user, config_path=self.config_file)
self.assertFunctional(ssh_fs)
def main():
p = argparse.ArgumentParser()
p.add_argument("--deploy-user", default="atsjenkins")
p.add_argument("--deploy-pwd", default=os.getenv("ATSJENKINS_PASSWORD"))
p.add_argument("--coverage-source", required=True)
p.add_argument("--commit-hash", default=os.getenv("CI_COMMIT_SHA"))
p.add_argument("--coverage-commit-limit",
default=int(os.getenv("COVERAGE_COMMIT_LIMIT", 10)),
type=int)
p.add_argument("--coverage-root",
default=os.getenv(
"COVERAGE_WEBSITE_ROOT",
"/eos/user/a/atsjenkins/www/ACTS/coverage"))
p.add_argument("--website-public-url",
default=os.getenv(
"COVERAGE_WEBSITE_URL",
"https://acts.web.cern.ch/ACTS/coverage/"))
p.add_argument("--project-id", default=3031, type=int)
p.add_argument("--dry-run", "-s", action="store_true")
args = p.parse_args()
try:
www_fs = SSHFS(host="lxplus.cern.ch",
user=args.deploy_user,
passwd=args.deploy_pwd).opendir(args.coverage_root)
# www_fs = OSFS("www")
listdir = www_fs.listdir(".")
except:
print("Unable to establish SSH connection to lxplus")
print("This might indicate a problem with the credentials")
print("or a temporary connection / configuration problem")
raise
sys.exit(1)
gl = gitlab.Gitlab("https://gitlab.cern.ch/")
project = gl.projects.get(args.project_id)
commit_slug = args.commit_hash[:7]
coverage_dest = os.path.join(args.coverage_root, commit_slug)
print("Going to deploy coverage for", commit_slug, "to", coverage_dest)
print("Will be publicly available under",
urljoin(args.website_public_url, commit_slug))
src_fs = OSFS(args.coverage_source)
if not args.dry_run:
fs.copy.copy_dir(src_fs, ".", www_fs, commit_slug)
# cleanup
# get all deployed commits
deployed_commits = set(filter(www_fs.isdir, www_fs.listdir(".")))
with ThreadPoolExecutor(max_workers=8) as tp:
# deployed_commit_info = p.map(project.commits.get, deployed_commits)
futures = [tp.submit(project.commits.get, c) for c in deployed_commits]
wait(futures)
deployed_commits_with_time = []
for commit, future in zip(deployed_commits, futures):
try:
info = future.result()
date = parse(info.committed_date)
deployed_commits_with_time.append((commit, date))
except gitlab.exceptions.GitlabGetError as e:
print("Commit", commit, "not found, will remove")
deployed_commits_with_time = list(
reversed(sorted(deployed_commits_with_time, key=lambda i: i[1])))
# take the n newest commits
commits_to_keep = set(
h for h, _ in deployed_commits_with_time[:args.coverage_commit_limit])
print("Currently deployed commits:")
for idx, (h, t) in enumerate(deployed_commits_with_time):
if idx < args.coverage_commit_limit:
print(" o", h, "-", t)
else:
print(" x", h, "-", t)
print("Keeping commits:", ", ".join(commits_to_keep))
commits_to_delete = deployed_commits - commits_to_keep
if len(commits_to_delete) > 0:
print("Removing:", ", ".join(commits_to_delete))
if not args.dry_run:
for commit in commits_to_delete:
www_fs.removetree(commit)
# install / update indexfile
latest_commit = deployed_commits_with_time[0][0]
latest_coverage_url = urljoin(args.website_public_url, latest_commit)
index_content = """
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="refresh" content="0; url={0}" />
</head>
<body>
Redirecting to <a href"{0}">{0}</a>
</body>
</html>
""".format(latest_coverage_url)
with www_fs.open("index.html", "w") as f:
print("Writing index file redirecting to", latest_coverage_url)
if not args.dry_run:
f.write(index_content)
def test_publickey_file(self):
ssh_fs = SSHFS('localhost',
self.user,
port=self.port,
pkey=self.key_file)
self.assertFunctional(ssh_fs)
def test_password(self):
ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port)
self.assertFunctional(ssh_fs)
def make_fs(self):
self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port)
self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex)
self.ssh_fs.makedir(self.test_folder, recreate=True)
return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS)
class TestSSHFS(fs.test.FSTestCases, unittest.TestCase):
user = "******"
pasw = "pass"
port = 2222
@classmethod
def setUpClass(cls):
super(TestSSHFS, cls).setUpClass()
cls.sftp_container = utils.startServer(utils.docker_client, cls.user,
cls.pasw, cls.port)
@classmethod
def tearDownClass(cls):
utils.stopServer(cls.sftp_container)
super(TestSSHFS, cls).tearDownClass()
@staticmethod
def destroy_fs(fs):
fs.close()
del fs
def make_fs(self):
self.ssh_fs = SSHFS('localhost', self.user, self.pasw, port=self.port)
self.test_folder = fs.path.join('/home', self.user, uuid.uuid4().hex)
self.ssh_fs.makedir(self.test_folder, recreate=True)
return self.ssh_fs.opendir(self.test_folder, factory=ClosingSubFS)
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_0(self):
super(TestSSHFS, self).test_download_0()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_1(self):
super(TestSSHFS, self).test_download_1()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_2(self):
super(TestSSHFS, self).test_download_2()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_download_4(self):
super(TestSSHFS, self).test_download_4()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_0(self):
super(TestSSHFS, self).test_upload_0()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_1(self):
super(TestSSHFS, self).test_upload_1()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_2(self):
super(TestSSHFS, self).test_upload_2()
@unittest.skipIf(sys.version_info[:2] == (3, 4), 'hangs in Python 3.4')
def test_upload_4(self):
super(TestSSHFS, self).test_upload_4()
def test_chmod(self):
self.fs.touch("test.txt")
remote_path = fs.path.join(self.test_folder, "test.txt")
# Initial permissions
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o644)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o644)
# Change permissions with SSHFS._chown
self.fs.delegate_fs()._chmod(remote_path, 0o744)
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o744)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o744)
# Change permissions with SSHFS.setinfo
self.fs.setinfo("test.txt",
{"access": {
"permissions": Permissions(mode=0o600)
}})
info = self.fs.getinfo("test.txt", ["access"])
self.assertEqual(info.permissions.mode, 0o600)
st = self.fs.delegate_fs()._sftp.stat(remote_path)
self.assertEqual(stat.S_IMODE(st.st_mode), 0o600)
with self.assertRaises(fs.errors.PermissionDenied):
self.fs.delegate_fs().setinfo(
"/", {"access": {
"permissions": Permissions(mode=0o777)
}})
def test_chown(self):
self.fs.touch("test.txt")
remote_path = fs.path.join(self.test_folder, "test.txt")
info = self.fs.getinfo("test.txt", namespaces=["access"])
gid, uid = info.get('access', 'uid'), info.get('access', 'gid')
with utils.mock.patch.object(self.fs.delegate_fs()._sftp,
'chown') as chown:
self.fs.setinfo("test.txt", {'access': {'uid': None}})
chown.assert_called_with(remote_path, uid, gid)
self.fs.setinfo("test.txt", {'access': {'gid': None}})
chown.assert_called_with(remote_path, uid, gid)
self.fs.setinfo("test.txt", {'access': {'gid': 8000}})
chown.assert_called_with(remote_path, uid, 8000)
self.fs.setinfo("test.txt", {'access': {'uid': 1001, 'gid': 1002}})
chown.assert_called_with(remote_path, 1001, 1002)
def test_utime(self):
def get_accessed(f):
return f.getdetails("test.txt").get('details', 'accessed')
def get_modified(f):
return f.getdetails("test.txt").get('details', 'modified')
self.fs.touch("test.txt")
self.fs.setinfo("test.txt",
{'details': {
'accessed': None,
'modified': None
}})
self.assertLessEqual(time.time() - get_accessed(self.fs), 2)
self.assertLessEqual(time.time() - get_modified(self.fs), 2)
self.fs.setinfo("test.txt", {'details': {'accessed': 0}})
self.assertEqual(get_accessed(self.fs), 0)
self.assertEqual(get_modified(self.fs), 0)
self.fs.setinfo("test.txt", {'details': {'modified': 100}})
self.assertEqual(get_accessed(self.fs), 100)
self.assertEqual(get_modified(self.fs), 100)
self.fs.setinfo("test.txt",
{'details': {
'modified': 100,
'accessed': 200
}})
self.assertEqual(get_accessed(self.fs), 200)
self.assertEqual(get_modified(self.fs), 100)