def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
self.assertTrue(m.writable())
self.pool.apply(fuseRenameTestHelper, (self.mounttmp,))
# Starting manifest
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual(["testdir"], d1)
d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual(["testdir2"], sorted(d1))
d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
self.assertEqual(["file1.txt"], d1)
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
def runTest(self):
self.make_mount(fuse.ProjectDirectory,
project_object=self.api.users().current().execute())
d1 = llfuse.listdir(self.mounttmp)
self.assertIn('Unrestricted public data', d1)
d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
public_project = run_test_server.fixture('groups')[
'anonymously_accessible_project']
found_in = 0
found_not_in = 0
for name, item in run_test_server.fixture('collections').iteritems():
if 'name' not in item:
pass
elif item['owner_uuid'] == public_project['uuid']:
self.assertIn(item['name'], d2)
found_in += 1
else:
# Artificial assumption here: there is no public
# collection fixture with the same name as a
# non-public collection.
self.assertNotIn(item['name'], d2)
found_not_in += 1
self.assertNotEqual(0, found_in)
self.assertNotEqual(0, found_not_in)
d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
def runTest(self):
aproject = self.api.groups().create(body={
"name": "aproject",
"group_class": "project"
}).execute()
bproject = self.api.groups().create(body={
"name": "bproject",
"group_class": "project",
"owner_uuid": aproject["uuid"]
}).execute()
self.make_mount(fuse.ProjectDirectory,
project_object=self.api.users().current().execute())
self.operations.listen_for_events()
d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject"))
self.assertEqual(["bproject"], sorted(d1))
self.api.groups().delete(uuid=bproject["uuid"]).execute()
for attempt in AssertWithTimeout(10):
attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
def runTest(self):
self.make_mount(fuse.SharedDirectory,
exclude=self.api.users().current().execute()['uuid'])
# shared_dirs is a list of the directories exposed
# by fuse.SharedDirectory (i.e. any object visible
# to the current user)
shared_dirs = llfuse.listdir(self.mounttmp)
shared_dirs.sort()
self.assertIn('FUSE User', shared_dirs)
# fuse_user_objs is a list of the objects owned by the FUSE
# test user (which present as files in the 'FUSE User'
# directory)
fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
fuse_user_objs.sort()
self.assertEqual(['FUSE Test Project', # project owned by user
'collection #1 owned by FUSE', # collection owned by user
'collection #2 owned by FUSE', # collection owned by user
'pipeline instance owned by FUSE.pipelineInstance', # pipeline instance owned by user
], fuse_user_objs)
# test_proj_files is a list of the files in the FUSE Test Project.
test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
test_proj_files.sort()
self.assertEqual(['collection in FUSE project',
'pipeline instance in FUSE project.pipelineInstance',
'pipeline template in FUSE project.pipelineTemplate'
], test_proj_files)
# Double check that we can open and read objects in this folder as a file,
# and that its contents are what we expect.
pipeline_template_path = os.path.join(
self.mounttmp,
'FUSE User',
'FUSE Test Project',
'pipeline template in FUSE project.pipelineTemplate')
with open(pipeline_template_path) as f:
j = json.load(f)
self.assertEqual("pipeline template in FUSE project", j['name'])
# check mtime on template
st = os.stat(pipeline_template_path)
try:
mtime = st.st_mtime_ns / 1000000000
except AttributeError:
mtime = st.st_mtime
self.assertEqual(mtime, 1397493304)
# check mtime on collection
st = os.stat(os.path.join(
self.mounttmp,
'FUSE User',
'collection #1 owned by FUSE'))
try:
mtime = st.st_mtime_ns / 1000000000
except AttributeError:
mtime = st.st_mtime
self.assertEqual(mtime, 1391448174)
def listContentsInProjectWithManyCollections(self, project_contents):
project_contents = llfuse.listdir(self.mounttmp)
self.assertEqual(201, len(project_contents))
self.assertIn('Collection_1', project_contents)
for collection_name in project_contents:
collection_contents = llfuse.listdir(os.path.join(self.mounttmp, collection_name))
self.assertIn('baz', collection_contents)
def runTest(self):
self.make_mount(fuse.MagicDirectory)
self.operations.inodes.inode_cache.cap = 1
self.operations.inodes.inode_cache.min_entries = 2
with self.assertRaises(OSError):
llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
def tst_bug382(self):
dirname = self.newname()
fullname = self.mnt_dir + "/" + dirname
os.mkdir(fullname)
assert stat.S_ISDIR(os.stat(fullname).st_mode)
assert dirname in llfuse.listdir(self.mnt_dir)
cmd = ('(%d, %r)' % (llfuse.ROOT_INODE, path2bytes(dirname))).encode()
llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME), 'rmtree', cmd)
assert_raises(FileNotFoundError, os.stat, fullname)
assert dirname not in llfuse.listdir(self.mnt_dir)
def moveFileFromCollectionWithMultipleBlocks(self):
d1 = llfuse.listdir(os.path.join(mounttmp, stream))
self.assertIn(filename, d1)
os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename))
d1 = llfuse.listdir(os.path.join(mounttmp))
self.assertIn('moved_from_'+stream+'_'+filename, d1)
d1 = llfuse.listdir(os.path.join(mounttmp, stream))
self.assertNotIn(filename, d1)
def tst_bug382(self):
dirname = self.newname()
fullname = self.mnt_dir + "/" + dirname
os.mkdir(fullname)
self.assertTrue(stat.S_ISDIR(os.stat(fullname).st_mode))
self.assertTrue(dirname in llfuse.listdir(self.mnt_dir))
llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME),
'rmtree', pickle.dumps((llfuse.ROOT_INODE, dirname),
pickle.HIGHEST_PROTOCOL))
self.assertRaises(OSError, os.stat, fullname)
self.assertTrue(dirname not in llfuse.listdir(self.mnt_dir))
def tst_bug382(self):
dirname = self.newname()
fullname = self.mnt_dir + "/" + dirname
os.mkdir(fullname)
assert stat.S_ISDIR(os.stat(fullname).st_mode)
assert dirname in llfuse.listdir(self.mnt_dir)
llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME),
'rmtree', pickle.dumps((llfuse.ROOT_INODE, path2bytes(dirname)),
PICKLE_PROTOCOL))
assert_raises(FileNotFoundError, os.stat, fullname)
assert dirname not in llfuse.listdir(self.mnt_dir)
def tst_mknod(self):
filename = os.path.join(self.mnt_dir, self.newname())
src = self.src
shutil.copyfile(src, filename)
fstat = os.lstat(filename)
self.assertTrue(stat.S_ISREG(fstat.st_mode))
self.assertEquals(fstat.st_nlink, 1)
self.assertTrue(basename(filename) in llfuse.listdir(self.mnt_dir))
self.assertTrue(filecmp.cmp(src, filename, False))
os.unlink(filename)
self.assertRaises(OSError, os.stat, filename)
self.assertTrue(basename(filename) not in llfuse.listdir(self.mnt_dir))
def runTest(self):
d1 = llfuse.listdir(os.path.join(mounttmp))
self.assertEqual(["testdir"], d1)
d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
d1 = llfuse.listdir(os.path.join(mounttmp))
self.assertEqual(["file1.txt", "testdir"], sorted(d1))
d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual([], d1)
def tst_mkdir(self):
dirname = self.newname()
fullname = self.mnt_dir + "/" + dirname
os.mkdir(fullname)
fstat = os.stat(fullname)
self.assertTrue(stat.S_ISDIR(fstat.st_mode))
self.assertEquals(llfuse.listdir(fullname), [])
self.assertEquals(fstat.st_nlink, 1)
self.assertTrue(dirname in llfuse.listdir(self.mnt_dir))
os.rmdir(fullname)
self.assertRaises(OSError, os.stat, fullname)
self.assertTrue(dirname not in llfuse.listdir(self.mnt_dir))
def tst_symlink(self):
linkname = self.newname()
fullname = self.mnt_dir + "/" + linkname
os.symlink("/imaginary/dest", fullname)
fstat = os.lstat(fullname)
assert stat.S_ISLNK(fstat.st_mode)
assert os.readlink(fullname) == "/imaginary/dest"
assert fstat.st_nlink == 1
assert linkname in llfuse.listdir(self.mnt_dir)
os.unlink(fullname)
assert_raises(FileNotFoundError, os.lstat, fullname)
assert linkname not in llfuse.listdir(self.mnt_dir)
def tst_mknod(self):
filename = os.path.join(self.mnt_dir, self.newname())
src = self.src
shutil.copyfile(src, filename)
fstat = os.lstat(filename)
assert stat.S_ISREG(fstat.st_mode)
assert fstat.st_nlink == 1
assert basename(filename) in llfuse.listdir(self.mnt_dir)
assert filecmp.cmp(src, filename, False)
os.unlink(filename)
assert_raises(FileNotFoundError, os.stat, filename)
assert basename(filename) not in llfuse.listdir(self.mnt_dir)
def tst_symlink(self):
linkname = self.newname()
fullname = self.mnt_dir + "/" + linkname
os.symlink("/imaginary/dest", fullname)
fstat = os.lstat(fullname)
self.assertTrue(stat.S_ISLNK(fstat.st_mode))
self.assertEquals(os.readlink(fullname), "/imaginary/dest")
self.assertEquals(fstat.st_nlink, 1)
self.assertTrue(linkname in llfuse.listdir(self.mnt_dir))
os.unlink(fullname)
self.assertRaises(OSError, os.lstat, fullname)
self.assertTrue(linkname not in llfuse.listdir(self.mnt_dir))
def tst_mkdir(self):
dirname = self.newname()
fullname = self.mnt_dir + "/" + dirname
os.mkdir(fullname)
fstat = os.stat(fullname)
assert stat.S_ISDIR(fstat.st_mode)
assert llfuse.listdir(fullname) == []
assert fstat.st_nlink == 1
assert dirname in llfuse.listdir(self.mnt_dir)
os.rmdir(fullname)
assert_raises(FileNotFoundError, os.stat, fullname)
assert dirname not in llfuse.listdir(self.mnt_dir)
def runTest(self):
self.make_mount(fuse.TagsDirectory)
d1 = llfuse.listdir(self.mounttmp)
d1.sort()
self.assertEqual(['foo_tag'], d1)
d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
d2.sort()
self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
d3.sort()
self.assertEqual(['foo'], d3)
def runTest(self):
self.make_mount(fuse.MagicDirectory)
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
arvados.util.uuid_pattern.match(fn)
for fn in mount_ls),
"new FUSE MagicDirectory has no collections or projects")
self.assertDirContents(self.testcollection, ['thing1.txt'])
self.assertDirContents(os.path.join('by_id', self.testcollection),
['thing1.txt'])
self.assertIn(self.collection_in_test_project,
llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
self.assertIn(self.collection_in_test_project,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertIn(self.testcollection, mount_ls)
self.assertIn(self.testcollection,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
self.assertIn(self.test_project, mount_ls)
self.assertIn(self.test_project,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
with self.assertRaises(OSError):
llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
self.assertEqual(v, f.read())
def main(args=None):
'''Control a mounted S3QL File System.'''
if args is None:
args = sys.argv[1:]
options = parse_args(args)
setup_logging(options)
path = options.mountpoint
if not os.path.exists(path):
raise QuietError('Mountpoint %r does not exist' % path)
ctrlfile = os.path.join(path, CTRL_NAME)
if not (CTRL_NAME not in llfuse.listdir(path)
and os.path.exists(ctrlfile)):
raise QuietError('Mountpoint is not an S3QL file system')
if os.stat(ctrlfile).st_uid != os.geteuid() and os.geteuid() != 0:
raise QuietError('Only root and the mounting user may run s3qlctrl.')
if options.action == 'flushcache':
llfuse.setxattr(ctrlfile, 's3ql_flushcache!', 'dummy')
if options.action == 'upload-meta':
llfuse.setxattr(ctrlfile, 'upload-meta', 'dummy')
elif options.action == 'log':
llfuse.setxattr(ctrlfile, 'logging',
pickle.dumps((options.level, options.modules),
pickle.HIGHEST_PROTOCOL))
elif options.action == 'cachesize':
llfuse.setxattr(ctrlfile, 'cachesize', pickle.dumps(options.cachesize*1024))
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertEqual(collection2["manifest_text"], "")
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
self.assertTrue(m.writable())
self.assertNotIn("file1.txt", collection)
self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,))
self.assertIn("file1.txt", collection)
d1 = llfuse.listdir(self.mounttmp)
self.assertEqual(["file1.txt"], d1)
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
def tst_lock_rm(self):
# Extract tar
tempdir = os.path.join(self.mnt_dir, 'lock_dir')
filename = os.path.join(tempdir, 'myfile')
os.mkdir(tempdir)
with open(filename, 'w') as fh:
fh.write('Hello, world')
# copy
try:
s3ql.cli.lock.main([tempdir])
except:
sys.excepthook(*sys.exc_info())
self.fail("s3qllock raised exception")
# Try to delete
with self.assertRaises(OSError) as cm:
os.unlink(filename)
self.assertEqual(cm.exception[0], errno.EPERM)
# Try to write
with self.assertRaises(IOError) as cm:
open(filename, 'w+').write('Hello')
self.assertEqual(cm.exception[0], errno.EPERM)
# delete properly
try:
s3ql.cli.remove.main([tempdir])
except:
sys.excepthook(*sys.exc_info())
self.fail("s3qlrm raised exception")
self.assertTrue('lock_dir' not in llfuse.listdir(self.mnt_dir))
def tst_lock_rm(self):
# Extract tar
tempdir = os.path.join(self.mnt_dir, 'lock_dir')
filename = os.path.join(tempdir, 'myfile')
os.mkdir(tempdir)
with open(filename, 'w') as fh:
fh.write('Hello, world')
# copy
try:
s3ql.lock.main([tempdir])
except:
sys.excepthook(*sys.exc_info())
pytest.fail("s3qllock raised exception")
# Try to delete
assert_raises(PermissionError, os.unlink, filename)
# Try to write
with pytest.raises(PermissionError):
open(filename, 'w+').write('Hello')
# delete properly
try:
s3ql.remove.main([tempdir])
except:
sys.excepthook(*sys.exc_info())
pytest.fail("s3qlrm raised exception")
assert 'lock_dir' not in llfuse.listdir(self.mnt_dir)
def main(args=None):
'''Recursively delete files and directories in an S3QL file system'''
if args is None:
args = sys.argv[1:]
options = parse_args(args)
setup_logging(options)
for name in options.path:
if not os.path.exists(name):
raise QuietError('%r does not exist' % name)
parent = os.path.dirname(os.path.abspath(name))
fstat_p = os.stat(parent)
fstat = os.stat(name)
if fstat_p.st_dev != fstat.st_dev:
raise QuietError('%s is a mount point itself.' % name)
ctrlfile = os.path.join(parent, CTRL_NAME)
if not (CTRL_NAME not in llfuse.listdir(parent) and os.path.exists(ctrlfile)):
raise QuietError('%s is not on an S3QL file system' % name)
if os.stat(ctrlfile).st_uid != os.geteuid():
raise QuietError('Only root and the mounting user may run s3qlrm.')
llfuse.setxattr(ctrlfile, 'rmtree', pickle.dumps((fstat_p.st_ino,
os.path.basename(name)),
pickle.HIGHEST_PROTOCOL))
def runTest(self):
self.make_mount(fuse.ProjectDirectory,
project_object=self.api.users().current().execute())
d1 = llfuse.listdir(self.mounttmp)
self.assertNotIn('testcollection', d1)
self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
d1 = llfuse.listdir(self.mounttmp)
self.assertIn('testcollection', d1)
self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
d1 = llfuse.listdir(self.mounttmp)
self.assertNotIn('testcollection', d1)
def main(args=None):
'''Umount S3QL file system
This function writes to stdout/stderr and calls `system.exit()` instead
of returning.
'''
if args is None:
args = sys.argv[1:]
options = parse_args(args)
setup_logging(options)
mountpoint = options.mountpoint
# Check if it's a mount point
if not posixpath.ismount(mountpoint):
print('Not a mount point.', file=sys.stderr)
sys.exit(1)
# Check if it's an S3QL mountpoint
ctrlfile = os.path.join(mountpoint, CTRL_NAME)
if not (CTRL_NAME not in llfuse.listdir(mountpoint)
and os.path.exists(ctrlfile)):
print('Not an S3QL file system.', file=sys.stderr)
sys.exit(1)
if options.lazy:
lazy_umount(mountpoint)
else:
blocking_umount(mountpoint)
def runTest(self):
# Can't delete because it's not empty
with self.assertRaises(OSError):
os.rmdir(os.path.join(mounttmp, "testdir"))
d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
# Delete file
os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
# Make sure it's empty
d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual([], d1)
# Try to delete it again
with self.assertRaises(OSError):
os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
def runTest(self):
with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
f.write("foo")
d1 = llfuse.listdir(os.path.join(mounttmp))
self.assertEqual(["file1.txt"], sorted(d1))
os.remove(os.path.join(mounttmp, "file1.txt"))
d1 = llfuse.listdir(os.path.join(mounttmp))
self.assertEqual([], sorted(d1))
f.seek(0)
self.assertEqual(f.read(), "foo")
f.write("bar")
f.seek(0)
self.assertEqual(f.read(), "foobar")
def runTest(self):
with self.assertRaises(IOError):
with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
f.write("Hello world!")
os.mkdir(os.path.join(mounttmp, "testdir"))
with self.assertRaises(OSError):
os.mkdir(os.path.join(mounttmp, "testdir"))
d1 = llfuse.listdir(mounttmp)
self.assertEqual(["testdir"], d1)
with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
f.write("Hello world!")
d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
def runTest(self):
d1 = llfuse.listdir(mounttmp)
self.assertNotIn('testcollection', d1)
os.mkdir(os.path.join(mounttmp, "testcollection"))
d1 = llfuse.listdir(mounttmp)
self.assertIn('testcollection', d1)
with self.assertRaises(OSError):
os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data'))
os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
d1 = llfuse.listdir(mounttmp)
self.assertNotIn('testcollection', d1)
d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
self.assertIn('testcollection', d1)
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
self.operations.listen_for_events()
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual([], sorted(d1))
with arvados.collection.Collection(collection.manifest_locator(),
api_client=self.api) as collection2:
with collection2.open("file1.txt", "w") as f:
f.write("foo")
for attempt in AssertWithTimeout(10):
attempt(self.assertEqual, ["file1.txt"],
llfuse.listdir(os.path.join(self.mounttmp)))
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
with collection.open("file1.txt", "w") as f:
f.write("blub")
with collection.open("file2.txt", "w") as f:
f.write("plnp")
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
d1 = llfuse.listdir(self.mounttmp)
self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
collection.remove("file2.txt")
d1 = llfuse.listdir(self.mounttmp)
self.assertEqual(["file1.txt"], d1)
def runTest(self):
self.make_mount(fuse.TagsDirectory, poll_time=1)
self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
self.tag_collection(bar_uuid, 'fuse_test_tag')
for attempt in AssertWithTimeout(10):
attempt(self.assertIn, 'fuse_test_tag',
llfuse.listdir(self.mounttmp))
self.assertDirContents('fuse_test_tag', [bar_uuid])
baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
l = self.tag_collection(baz_uuid, 'fuse_test_tag')
for attempt in AssertWithTimeout(10):
attempt(self.assertDirContents, 'fuse_test_tag',
[bar_uuid, baz_uuid])
self.api.links().delete(uuid=l['uuid']).execute()
for attempt in AssertWithTimeout(10):
attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual([], sorted(d1))
# See note in MountTestBase.setUp
self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False):
if skip_pdh_only is True:
self.make_mount(fuse.MagicDirectory) # in this case, the default by_id applies
else:
self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
arvados.util.uuid_pattern.match(fn)
for fn in mount_ls),
"new FUSE MagicDirectory lists Collection")
# look up using pdh should succeed in all cases
self.assertDirContents(self.testcollection, ['thing1.txt'])
self.assertDirContents(os.path.join('by_id', self.testcollection),
['thing1.txt'])
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertIn(self.testcollection, mount_ls)
self.assertIn(self.testcollection,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
self.assertEqual(v, f.read())
# look up using uuid should fail when pdh_only is set
if pdh_only is True:
with self.assertRaises(OSError):
self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
['thing1.txt'])
else:
self.assertDirContents(os.path.join('by_id', self.testcollectionuuid),
['thing1.txt'])
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
self.assertTrue(m.writable())
self.pool.apply(fuseRenameTestHelper, (self.mounttmp, ))
# Starting manifest
collection2 = self.api.collections().get(
uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(
collection2["manifest_text"],
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$'
)
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual(["testdir"], d1)
d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
os.rename(os.path.join(self.mounttmp, "testdir"),
os.path.join(self.mounttmp, "testdir2"))
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual(["testdir2"], sorted(d1))
d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
self.assertEqual(["file1.txt"], d1)
collection2 = self.api.collections().get(
uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(
collection2["manifest_text"],
r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$'
)
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
self.operations.listen_for_events(self.api)
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual([], sorted(d1))
with arvados.collection.Collection(collection.manifest_locator(),
api_client=self.api) as collection2:
with collection2.open("file1.txt", "w") as f:
f.write("foo")
time.sleep(1)
# should show up via event bus notify
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual(["file1.txt"], sorted(d1))
def runTest(self):
self.make_mount(fuse.MagicDirectory)
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
arvados.util.uuid_pattern.match(fn)
for fn in mount_ls),
"new FUSE MagicDirectory lists Collection")
self.assertDirContents(self.testcollection, ['thing1.txt'])
self.assertDirContents(os.path.join('by_id', self.testcollection),
['thing1.txt'])
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertIn(self.testcollection, mount_ls)
self.assertIn(self.testcollection,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
self.assertEqual(v, f.read())
def runTest(self):
with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
f.write("bar")
d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
self.assertEqual(len(d1), 2)
with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
self.assertEqual(f.read(), "bar")
self.assertRegexpMatches(
d1[1], r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
with open(os.path.join(mounttmp, d1[1]), "r") as f:
self.assertEqual(f.read(), "foo")
def runTest(self):
d1 = llfuse.listdir(mounttmp)
self.assertNotIn('testcollection', d1)
os.mkdir(os.path.join(mounttmp, "testcollection"))
d1 = llfuse.listdir(mounttmp)
self.assertIn('testcollection', d1)
with self.assertRaises(OSError):
os.rename(os.path.join(mounttmp, "testcollection"),
os.path.join(mounttmp, 'Unrestricted public data'))
os.rename(
os.path.join(mounttmp, "testcollection"),
os.path.join(mounttmp, 'Unrestricted public data',
'testcollection'))
d1 = llfuse.listdir(mounttmp)
self.assertNotIn('testcollection', d1)
d1 = llfuse.listdir(
os.path.join(mounttmp, 'Unrestricted public data'))
self.assertIn('testcollection', d1)
def runTest(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual([], sorted(d1))
with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
with collection2.open("file1.txt", "w") as f:
f.write("foo")
# See note in MountTestBase.setUp
self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
def runTest(self):
self.make_mount(fuse.MagicDirectory)
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertFalse(
any(
arvados.util.keep_locator_pattern.match(fn)
or arvados.util.uuid_pattern.match(fn) for fn in mount_ls),
"new FUSE MagicDirectory has no collections or projects")
self.assertDirContents(self.testcollection, ['thing1.txt'])
self.assertDirContents(os.path.join('by_id', self.testcollection),
['thing1.txt'])
self.assertIn(
self.collection_in_test_project,
llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
self.assertIn(
self.collection_in_test_project,
llfuse.listdir(
os.path.join(self.mounttmp, 'by_id', self.test_project)))
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertIn(self.testcollection, mount_ls)
self.assertIn(self.testcollection,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
self.assertIn(self.test_project, mount_ls)
self.assertIn(self.test_project,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
with self.assertRaises(OSError):
llfuse.listdir(
os.path.join(self.mounttmp, 'by_id', self.non_project_group))
files = {}
files[os.path.join(self.mounttmp, self.testcollection,
'thing1.txt')] = 'data 1'
for k, v in viewitems(files):
with open(os.path.join(self.mounttmp, k), 'rb') as f:
self.assertEqual(v, f.read().decode())
def tst_link(self):
name1 = os.path.join(self.mnt_dir, self.newname())
name2 = os.path.join(self.mnt_dir, self.newname())
src = self.src
shutil.copyfile(src, name1)
assert filecmp.cmp(name1, src, False)
os.link(name1, name2)
fstat1 = os.lstat(name1)
fstat2 = os.lstat(name2)
assert fstat1 == fstat2
assert fstat1.st_nlink == 2
assert basename(name2) in llfuse.listdir(self.mnt_dir)
assert filecmp.cmp(name1, name2, False)
os.unlink(name2)
fstat1 = os.lstat(name1)
assert fstat1.st_nlink == 1
os.unlink(name1)
def tst_link(self):
name1 = os.path.join(self.mnt_dir, self.newname())
name2 = os.path.join(self.mnt_dir, self.newname())
src = self.src
shutil.copyfile(src, name1)
self.assertTrue(filecmp.cmp(name1, src, False))
os.link(name1, name2)
fstat1 = os.lstat(name1)
fstat2 = os.lstat(name2)
self.assertEqual(fstat1, fstat2)
self.assertEqual(fstat1.st_nlink, 2)
self.assertTrue(basename(name2) in llfuse.listdir(self.mnt_dir))
self.assertTrue(filecmp.cmp(name1, name2, False))
os.unlink(name2)
fstat1 = os.lstat(name1)
self.assertEqual(fstat1.st_nlink, 1)
os.unlink(name1)
def runTest(self):
os.environ['KEEP_LOCAL_STORE'] = keeptmp
with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2:
with collection2.open("file1.txt", "w") as f2:
f2.write("foo")
f.write("bar")
d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
self.assertEqual(len(d1), 2)
with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
self.assertEqual(f.read(), "bar")
assertRegex(self, d1[1],
r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
with open(os.path.join(mounttmp, d1[1]), "r") as f:
self.assertEqual(f.read(), "foo")
def assert_s3ql_fs(path):
'''Raise `QuietError` if *path* is not on an S3QL file system
Returns name of the S3QL control file.
'''
try:
os.stat(path)
except FileNotFoundError:
raise QuietError('%s does not exist' % path)
except OSError as exc:
if exc.errno is errno.ENOTCONN:
raise QuietError('File system appears to have crashed.')
raise
ctrlfile = os.path.join(path, CTRL_NAME)
if not (CTRL_NAME not in llfuse.listdir(path)
and os.path.exists(ctrlfile)):
raise QuietError('%s is not on an S3QL file system' % path)
return ctrlfile
def tst_chown(self):
filename = os.path.join(self.mnt_dir, self.newname())
os.mkdir(filename)
fstat = os.lstat(filename)
uid = fstat.st_uid
gid = fstat.st_gid
uid_new = uid + 1
os.chown(filename, uid_new, -1)
fstat = os.lstat(filename)
self.assertEqual(fstat.st_uid, uid_new)
self.assertEqual(fstat.st_gid, gid)
gid_new = gid + 1
os.chown(filename, -1, gid_new)
fstat = os.lstat(filename)
self.assertEqual(fstat.st_uid, uid_new)
self.assertEqual(fstat.st_gid, gid_new)
os.rmdir(filename)
self.assertRaises(FileNotFoundError, os.stat, filename)
self.assertTrue(basename(filename) not in llfuse.listdir(self.mnt_dir))
def tst_readdir(self):
dir_ = os.path.join(self.mnt_dir, self.newname())
file_ = dir_ + "/" + self.newname()
subdir = dir_ + "/" + self.newname()
subfile = subdir + "/" + self.newname()
src = self.src
os.mkdir(dir_)
shutil.copyfile(src, file_)
os.mkdir(subdir)
shutil.copyfile(src, subfile)
listdir_is = llfuse.listdir(dir_)
listdir_is.sort()
listdir_should = [basename(file_), basename(subdir)]
listdir_should.sort()
assert listdir_is == listdir_should
os.unlink(file_)
os.unlink(subfile)
os.rmdir(subdir)
os.rmdir(dir_)
def tst_chown(self):
filename = os.path.join(self.mnt_dir, self.newname())
os.mkdir(filename)
fstat = os.lstat(filename)
uid = fstat.st_uid
gid = fstat.st_gid
uid_new = uid + 1
os.chown(filename, uid_new, -1)
fstat = os.lstat(filename)
assert fstat.st_uid == uid_new
assert fstat.st_gid == gid
gid_new = gid + 1
os.chown(filename, -1, gid_new)
fstat = os.lstat(filename)
assert fstat.st_uid == uid_new
assert fstat.st_gid == gid_new
os.rmdir(filename)
assert_raises(FileNotFoundError, os.stat, filename)
assert basename(filename) not in llfuse.listdir(self.mnt_dir)
def readContentsFromCollectionWithMultipleBlocks(self):
for i in range(0, streams):
d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i)))
for j in range(0, files_per_stream):
with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f:
self.assertEqual(data, f.read())
def runTest(self):
d1 = llfuse.listdir(mounttmp)
self.assertEqual(["file1.txt"], d1)
with open(os.path.join(mounttmp, "file1.txt")) as f:
self.assertEqual("plnp", f.read())
def test_listdir():
# There is a race-condition here if /usr/bin is modified while the test
# runs - but hopefully this is sufficiently rare.
list1 = set(os.listdir('/usr/bin'))
list2 = set(llfuse.listdir('/usr/bin'))
assert list1 == list2
def assertDirContents(self, subdir, expect_content):
path = self.mounttmp
if subdir:
path = os.path.join(path, subdir)
self.assertEqual(sorted(expect_content), sorted(llfuse.listdir(path)))
def getProjectWithManyCollections(self):
project_contents = llfuse.listdir(self.mounttmp)
self.assertEqual(201, len(project_contents))
self.assertIn('Collection_1', project_contents)
return project_contents
def runTest(self):
self.make_mount(fuse.SharedDirectory,
exclude=self.api.users().current().execute()['uuid'])
# shared_dirs is a list of the directories exposed
# by fuse.SharedDirectory (i.e. any object visible
# to the current user)
shared_dirs = llfuse.listdir(self.mounttmp)
shared_dirs.sort()
self.assertIn('FUSE User', shared_dirs)
# fuse_user_objs is a list of the objects owned by the FUSE
# test user (which present as files in the 'FUSE User'
# directory)
fuse_user_objs = llfuse.listdir(
os.path.join(self.mounttmp, 'FUSE User'))
fuse_user_objs.sort()
self.assertEqual(
[
'FUSE Test Project', # project owned by user
'collection #1 owned by FUSE', # collection owned by user
'collection #2 owned by FUSE', # collection owned by user
'pipeline instance owned by FUSE.pipelineInstance', # pipeline instance owned by user
],
fuse_user_objs)
# test_proj_files is a list of the files in the FUSE Test Project.
test_proj_files = llfuse.listdir(
os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
test_proj_files.sort()
self.assertEqual([
'collection in FUSE project',
'pipeline instance in FUSE project.pipelineInstance',
'pipeline template in FUSE project.pipelineTemplate'
], test_proj_files)
# Double check that we can open and read objects in this folder as a file,
# and that its contents are what we expect.
pipeline_template_path = os.path.join(
self.mounttmp, 'FUSE User', 'FUSE Test Project',
'pipeline template in FUSE project.pipelineTemplate')
with open(pipeline_template_path) as f:
j = json.load(f)
self.assertEqual("pipeline template in FUSE project", j['name'])
# check mtime on template
st = os.stat(pipeline_template_path)
try:
mtime = st.st_mtime_ns / 1000000000
except AttributeError:
mtime = st.st_mtime
self.assertEqual(mtime, 1397493304)
# check mtime on collection
st = os.stat(
os.path.join(self.mounttmp, 'FUSE User',
'collection #1 owned by FUSE'))
try:
mtime = st.st_mtime_ns / 1000000000
except AttributeError:
mtime = st.st_mtime
self.assertEqual(mtime, 1391448174)
