def runTest(self): collection = arvados.collection.Collection(api_client=self.api) collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) self.assertTrue(m.writable()) self.pool.apply(fuseRenameTestHelper, (self.mounttmp,)) # Starting manifest collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() self.assertRegexpMatches(collection2["manifest_text"], r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$') d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual(["testdir"], d1) d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir")) self.assertEqual(["file1.txt"], d1) os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2")) d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual(["testdir2"], sorted(d1)) d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2")) self.assertEqual(["file1.txt"], d1) collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() self.assertRegexpMatches(collection2["manifest_text"], r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
def runTest(self): self.make_mount(fuse.ProjectDirectory, project_object=self.api.users().current().execute()) d1 = llfuse.listdir(self.mounttmp) self.assertIn('Unrestricted public data', d1) d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data')) public_project = run_test_server.fixture('groups')[ 'anonymously_accessible_project'] found_in = 0 found_not_in = 0 for name, item in run_test_server.fixture('collections').iteritems(): if 'name' not in item: pass elif item['owner_uuid'] == public_project['uuid']: self.assertIn(item['name'], d2) found_in += 1 else: # Artificial assumption here: there is no public # collection fixture with the same name as a # non-public collection. self.assertNotIn(item['name'], d2) found_not_in += 1 self.assertNotEqual(0, found_in) self.assertNotEqual(0, found_not_in) d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3')) self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
def runTest(self): aproject = self.api.groups().create(body={ "name": "aproject", "group_class": "project" }).execute() bproject = self.api.groups().create(body={ "name": "bproject", "group_class": "project", "owner_uuid": aproject["uuid"] }).execute() self.make_mount(fuse.ProjectDirectory, project_object=self.api.users().current().execute()) self.operations.listen_for_events() d1 = llfuse.listdir(os.path.join(self.mounttmp, "aproject")) self.assertEqual(["bproject"], sorted(d1)) self.api.groups().delete(uuid=bproject["uuid"]).execute() for attempt in AssertWithTimeout(10): attempt(self.assertEqual, [], llfuse.listdir(os.path.join(self.mounttmp, "aproject")))
def runTest(self): self.make_mount(fuse.SharedDirectory, exclude=self.api.users().current().execute()['uuid']) # shared_dirs is a list of the directories exposed # by fuse.SharedDirectory (i.e. any object visible # to the current user) shared_dirs = llfuse.listdir(self.mounttmp) shared_dirs.sort() self.assertIn('FUSE User', shared_dirs) # fuse_user_objs is a list of the objects owned by the FUSE # test user (which present as files in the 'FUSE User' # directory) fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User')) fuse_user_objs.sort() self.assertEqual(['FUSE Test Project', # project owned by user 'collection #1 owned by FUSE', # collection owned by user 'collection #2 owned by FUSE', # collection owned by user 'pipeline instance owned by FUSE.pipelineInstance', # pipeline instance owned by user ], fuse_user_objs) # test_proj_files is a list of the files in the FUSE Test Project. test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project')) test_proj_files.sort() self.assertEqual(['collection in FUSE project', 'pipeline instance in FUSE project.pipelineInstance', 'pipeline template in FUSE project.pipelineTemplate' ], test_proj_files) # Double check that we can open and read objects in this folder as a file, # and that its contents are what we expect. pipeline_template_path = os.path.join( self.mounttmp, 'FUSE User', 'FUSE Test Project', 'pipeline template in FUSE project.pipelineTemplate') with open(pipeline_template_path) as f: j = json.load(f) self.assertEqual("pipeline template in FUSE project", j['name']) # check mtime on template st = os.stat(pipeline_template_path) try: mtime = st.st_mtime_ns / 1000000000 except AttributeError: mtime = st.st_mtime self.assertEqual(mtime, 1397493304) # check mtime on collection st = os.stat(os.path.join( self.mounttmp, 'FUSE User', 'collection #1 owned by FUSE')) try: mtime = st.st_mtime_ns / 1000000000 except AttributeError: mtime = st.st_mtime self.assertEqual(mtime, 1391448174)
def listContentsInProjectWithManyCollections(self, project_contents): project_contents = llfuse.listdir(self.mounttmp) self.assertEqual(201, len(project_contents)) self.assertIn('Collection_1', project_contents) for collection_name in project_contents: collection_contents = llfuse.listdir(os.path.join(self.mounttmp, collection_name)) self.assertIn('baz', collection_contents)
def runTest(self): self.make_mount(fuse.MagicDirectory) self.operations.inodes.inode_cache.cap = 1 self.operations.inodes.inode_cache.min_entries = 2 with self.assertRaises(OSError): llfuse.listdir(os.path.join(self.mounttmp, self.testcollection)) llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
def tst_bug382(self): dirname = self.newname() fullname = self.mnt_dir + "/" + dirname os.mkdir(fullname) assert stat.S_ISDIR(os.stat(fullname).st_mode) assert dirname in llfuse.listdir(self.mnt_dir) cmd = ('(%d, %r)' % (llfuse.ROOT_INODE, path2bytes(dirname))).encode() llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME), 'rmtree', cmd) assert_raises(FileNotFoundError, os.stat, fullname) assert dirname not in llfuse.listdir(self.mnt_dir)
def moveFileFromCollectionWithMultipleBlocks(self): d1 = llfuse.listdir(os.path.join(mounttmp, stream)) self.assertIn(filename, d1) os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename)) d1 = llfuse.listdir(os.path.join(mounttmp)) self.assertIn('moved_from_'+stream+'_'+filename, d1) d1 = llfuse.listdir(os.path.join(mounttmp, stream)) self.assertNotIn(filename, d1)
def tst_bug382(self): dirname = self.newname() fullname = self.mnt_dir + "/" + dirname os.mkdir(fullname) self.assertTrue(stat.S_ISDIR(os.stat(fullname).st_mode)) self.assertTrue(dirname in llfuse.listdir(self.mnt_dir)) llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME), 'rmtree', pickle.dumps((llfuse.ROOT_INODE, dirname), pickle.HIGHEST_PROTOCOL)) self.assertRaises(OSError, os.stat, fullname) self.assertTrue(dirname not in llfuse.listdir(self.mnt_dir))
def tst_bug382(self): dirname = self.newname() fullname = self.mnt_dir + "/" + dirname os.mkdir(fullname) assert stat.S_ISDIR(os.stat(fullname).st_mode) assert dirname in llfuse.listdir(self.mnt_dir) llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME), 'rmtree', pickle.dumps((llfuse.ROOT_INODE, path2bytes(dirname)), PICKLE_PROTOCOL)) assert_raises(FileNotFoundError, os.stat, fullname) assert dirname not in llfuse.listdir(self.mnt_dir)
def tst_mknod(self): filename = os.path.join(self.mnt_dir, self.newname()) src = self.src shutil.copyfile(src, filename) fstat = os.lstat(filename) self.assertTrue(stat.S_ISREG(fstat.st_mode)) self.assertEquals(fstat.st_nlink, 1) self.assertTrue(basename(filename) in llfuse.listdir(self.mnt_dir)) self.assertTrue(filecmp.cmp(src, filename, False)) os.unlink(filename) self.assertRaises(OSError, os.stat, filename) self.assertTrue(basename(filename) not in llfuse.listdir(self.mnt_dir))
def runTest(self): d1 = llfuse.listdir(os.path.join(mounttmp)) self.assertEqual(["testdir"], d1) d1 = llfuse.listdir(os.path.join(mounttmp, "testdir")) self.assertEqual(["file1.txt"], d1) os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt")) d1 = llfuse.listdir(os.path.join(mounttmp)) self.assertEqual(["file1.txt", "testdir"], sorted(d1)) d1 = llfuse.listdir(os.path.join(mounttmp, "testdir")) self.assertEqual([], d1)
def tst_mkdir(self): dirname = self.newname() fullname = self.mnt_dir + "/" + dirname os.mkdir(fullname) fstat = os.stat(fullname) self.assertTrue(stat.S_ISDIR(fstat.st_mode)) self.assertEquals(llfuse.listdir(fullname), []) self.assertEquals(fstat.st_nlink, 1) self.assertTrue(dirname in llfuse.listdir(self.mnt_dir)) os.rmdir(fullname) self.assertRaises(OSError, os.stat, fullname) self.assertTrue(dirname not in llfuse.listdir(self.mnt_dir))
def tst_symlink(self): linkname = self.newname() fullname = self.mnt_dir + "/" + linkname os.symlink("/imaginary/dest", fullname) fstat = os.lstat(fullname) assert stat.S_ISLNK(fstat.st_mode) assert os.readlink(fullname) == "/imaginary/dest" assert fstat.st_nlink == 1 assert linkname in llfuse.listdir(self.mnt_dir) os.unlink(fullname) assert_raises(FileNotFoundError, os.lstat, fullname) assert linkname not in llfuse.listdir(self.mnt_dir)
def tst_mknod(self): filename = os.path.join(self.mnt_dir, self.newname()) src = self.src shutil.copyfile(src, filename) fstat = os.lstat(filename) assert stat.S_ISREG(fstat.st_mode) assert fstat.st_nlink == 1 assert basename(filename) in llfuse.listdir(self.mnt_dir) assert filecmp.cmp(src, filename, False) os.unlink(filename) assert_raises(FileNotFoundError, os.stat, filename) assert basename(filename) not in llfuse.listdir(self.mnt_dir)
def tst_symlink(self): linkname = self.newname() fullname = self.mnt_dir + "/" + linkname os.symlink("/imaginary/dest", fullname) fstat = os.lstat(fullname) self.assertTrue(stat.S_ISLNK(fstat.st_mode)) self.assertEquals(os.readlink(fullname), "/imaginary/dest") self.assertEquals(fstat.st_nlink, 1) self.assertTrue(linkname in llfuse.listdir(self.mnt_dir)) os.unlink(fullname) self.assertRaises(OSError, os.lstat, fullname) self.assertTrue(linkname not in llfuse.listdir(self.mnt_dir))
def tst_mkdir(self): dirname = self.newname() fullname = self.mnt_dir + "/" + dirname os.mkdir(fullname) fstat = os.stat(fullname) assert stat.S_ISDIR(fstat.st_mode) assert llfuse.listdir(fullname) == [] assert fstat.st_nlink == 1 assert dirname in llfuse.listdir(self.mnt_dir) os.rmdir(fullname) assert_raises(FileNotFoundError, os.stat, fullname) assert dirname not in llfuse.listdir(self.mnt_dir)
def runTest(self): self.make_mount(fuse.TagsDirectory) d1 = llfuse.listdir(self.mounttmp) d1.sort() self.assertEqual(['foo_tag'], d1) d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag')) d2.sort() self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2) d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7')) d3.sort() self.assertEqual(['foo'], d3)
def runTest(self): self.make_mount(fuse.MagicDirectory) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or arvados.util.uuid_pattern.match(fn) for fn in mount_ls), "new FUSE MagicDirectory has no collections or projects") self.assertDirContents(self.testcollection, ['thing1.txt']) self.assertDirContents(os.path.join('by_id', self.testcollection), ['thing1.txt']) self.assertIn(self.collection_in_test_project, llfuse.listdir(os.path.join(self.mounttmp, self.test_project))) self.assertIn(self.collection_in_test_project, llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project))) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertIn(self.testcollection, mount_ls) self.assertIn(self.testcollection, llfuse.listdir(os.path.join(self.mounttmp, 'by_id'))) self.assertIn(self.test_project, mount_ls) self.assertIn(self.test_project, llfuse.listdir(os.path.join(self.mounttmp, 'by_id'))) with self.assertRaises(OSError): llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group)) files = {} files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1' for k, v in files.items(): with open(os.path.join(self.mounttmp, k)) as f: self.assertEqual(v, f.read())
def main(args=None): '''Control a mounted S3QL File System.''' if args is None: args = sys.argv[1:] options = parse_args(args) setup_logging(options) path = options.mountpoint if not os.path.exists(path): raise QuietError('Mountpoint %r does not exist' % path) ctrlfile = os.path.join(path, CTRL_NAME) if not (CTRL_NAME not in llfuse.listdir(path) and os.path.exists(ctrlfile)): raise QuietError('Mountpoint is not an S3QL file system') if os.stat(ctrlfile).st_uid != os.geteuid() and os.geteuid() != 0: raise QuietError('Only root and the mounting user may run s3qlctrl.') if options.action == 'flushcache': llfuse.setxattr(ctrlfile, 's3ql_flushcache!', 'dummy') if options.action == 'upload-meta': llfuse.setxattr(ctrlfile, 'upload-meta', 'dummy') elif options.action == 'log': llfuse.setxattr(ctrlfile, 'logging', pickle.dumps((options.level, options.modules), pickle.HIGHEST_PROTOCOL)) elif options.action == 'cachesize': llfuse.setxattr(ctrlfile, 'cachesize', pickle.dumps(options.cachesize*1024))
def runTest(self): collection = arvados.collection.Collection(api_client=self.api) collection.save_new() collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() self.assertEqual(collection2["manifest_text"], "") collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) self.assertTrue(m.writable()) self.assertNotIn("file1.txt", collection) self.pool.apply(fuseCreateFileTestHelper, (self.mounttmp,)) self.assertIn("file1.txt", collection) d1 = llfuse.listdir(self.mounttmp) self.assertEqual(["file1.txt"], d1) collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() self.assertRegexpMatches(collection2["manifest_text"], r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
def tst_lock_rm(self): # Extract tar tempdir = os.path.join(self.mnt_dir, 'lock_dir') filename = os.path.join(tempdir, 'myfile') os.mkdir(tempdir) with open(filename, 'w') as fh: fh.write('Hello, world') # copy try: s3ql.cli.lock.main([tempdir]) except: sys.excepthook(*sys.exc_info()) self.fail("s3qllock raised exception") # Try to delete with self.assertRaises(OSError) as cm: os.unlink(filename) self.assertEqual(cm.exception[0], errno.EPERM) # Try to write with self.assertRaises(IOError) as cm: open(filename, 'w+').write('Hello') self.assertEqual(cm.exception[0], errno.EPERM) # delete properly try: s3ql.cli.remove.main([tempdir]) except: sys.excepthook(*sys.exc_info()) self.fail("s3qlrm raised exception") self.assertTrue('lock_dir' not in llfuse.listdir(self.mnt_dir))
def tst_lock_rm(self): # Extract tar tempdir = os.path.join(self.mnt_dir, 'lock_dir') filename = os.path.join(tempdir, 'myfile') os.mkdir(tempdir) with open(filename, 'w') as fh: fh.write('Hello, world') # copy try: s3ql.lock.main([tempdir]) except: sys.excepthook(*sys.exc_info()) pytest.fail("s3qllock raised exception") # Try to delete assert_raises(PermissionError, os.unlink, filename) # Try to write with pytest.raises(PermissionError): open(filename, 'w+').write('Hello') # delete properly try: s3ql.remove.main([tempdir]) except: sys.excepthook(*sys.exc_info()) pytest.fail("s3qlrm raised exception") assert 'lock_dir' not in llfuse.listdir(self.mnt_dir)
def main(args=None): '''Recursively delete files and directories in an S3QL file system''' if args is None: args = sys.argv[1:] options = parse_args(args) setup_logging(options) for name in options.path: if not os.path.exists(name): raise QuietError('%r does not exist' % name) parent = os.path.dirname(os.path.abspath(name)) fstat_p = os.stat(parent) fstat = os.stat(name) if fstat_p.st_dev != fstat.st_dev: raise QuietError('%s is a mount point itself.' % name) ctrlfile = os.path.join(parent, CTRL_NAME) if not (CTRL_NAME not in llfuse.listdir(parent) and os.path.exists(ctrlfile)): raise QuietError('%s is not on an S3QL file system' % name) if os.stat(ctrlfile).st_uid != os.geteuid(): raise QuietError('Only root and the mounting user may run s3qlrm.') llfuse.setxattr(ctrlfile, 'rmtree', pickle.dumps((fstat_p.st_ino, os.path.basename(name)), pickle.HIGHEST_PROTOCOL))
def runTest(self): self.make_mount(fuse.ProjectDirectory, project_object=self.api.users().current().execute()) d1 = llfuse.listdir(self.mounttmp) self.assertNotIn('testcollection', d1) self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,)) d1 = llfuse.listdir(self.mounttmp) self.assertIn('testcollection', d1) self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,)) d1 = llfuse.listdir(self.mounttmp) self.assertNotIn('testcollection', d1)
def main(args=None): '''Umount S3QL file system This function writes to stdout/stderr and calls `system.exit()` instead of returning. ''' if args is None: args = sys.argv[1:] options = parse_args(args) setup_logging(options) mountpoint = options.mountpoint # Check if it's a mount point if not posixpath.ismount(mountpoint): print('Not a mount point.', file=sys.stderr) sys.exit(1) # Check if it's an S3QL mountpoint ctrlfile = os.path.join(mountpoint, CTRL_NAME) if not (CTRL_NAME not in llfuse.listdir(mountpoint) and os.path.exists(ctrlfile)): print('Not an S3QL file system.', file=sys.stderr) sys.exit(1) if options.lazy: lazy_umount(mountpoint) else: blocking_umount(mountpoint)
def runTest(self): # Can't delete because it's not empty with self.assertRaises(OSError): os.rmdir(os.path.join(mounttmp, "testdir")) d1 = llfuse.listdir(os.path.join(mounttmp, "testdir")) self.assertEqual(["file1.txt"], d1) # Delete file os.remove(os.path.join(mounttmp, "testdir", "file1.txt")) # Make sure it's empty d1 = llfuse.listdir(os.path.join(mounttmp, "testdir")) self.assertEqual([], d1) # Try to delete it again with self.assertRaises(OSError): os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
def runTest(self): with open(os.path.join(mounttmp, "file1.txt"), "w+") as f: f.write("foo") d1 = llfuse.listdir(os.path.join(mounttmp)) self.assertEqual(["file1.txt"], sorted(d1)) os.remove(os.path.join(mounttmp, "file1.txt")) d1 = llfuse.listdir(os.path.join(mounttmp)) self.assertEqual([], sorted(d1)) f.seek(0) self.assertEqual(f.read(), "foo") f.write("bar") f.seek(0) self.assertEqual(f.read(), "foobar")
def runTest(self): with self.assertRaises(IOError): with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f: f.write("Hello world!") os.mkdir(os.path.join(mounttmp, "testdir")) with self.assertRaises(OSError): os.mkdir(os.path.join(mounttmp, "testdir")) d1 = llfuse.listdir(mounttmp) self.assertEqual(["testdir"], d1) with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f: f.write("Hello world!") d1 = llfuse.listdir(os.path.join(mounttmp, "testdir")) self.assertEqual(["file1.txt"], d1)
def runTest(self): d1 = llfuse.listdir(mounttmp) self.assertNotIn('testcollection', d1) os.mkdir(os.path.join(mounttmp, "testcollection")) d1 = llfuse.listdir(mounttmp) self.assertIn('testcollection', d1) with self.assertRaises(OSError): os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data')) os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection')) d1 = llfuse.listdir(mounttmp) self.assertNotIn('testcollection', d1) d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data')) self.assertIn('testcollection', d1)
def runTest(self): collection = arvados.collection.Collection(api_client=self.api) collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) self.operations.listen_for_events() d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual([], sorted(d1)) with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2: with collection2.open("file1.txt", "w") as f: f.write("foo") for attempt in AssertWithTimeout(10): attempt(self.assertEqual, ["file1.txt"], llfuse.listdir(os.path.join(self.mounttmp)))
def runTest(self): collection = arvados.collection.Collection(api_client=self.api) with collection.open("file1.txt", "w") as f: f.write("blub") with collection.open("file2.txt", "w") as f: f.write("plnp") collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) d1 = llfuse.listdir(self.mounttmp) self.assertEqual(["file1.txt", "file2.txt"], sorted(d1)) collection.remove("file2.txt") d1 = llfuse.listdir(self.mounttmp) self.assertEqual(["file1.txt"], d1)
def runTest(self): self.make_mount(fuse.TagsDirectory, poll_time=1) self.assertIn('foo_tag', llfuse.listdir(self.mounttmp)) bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid'] self.tag_collection(bar_uuid, 'fuse_test_tag') for attempt in AssertWithTimeout(10): attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp)) self.assertDirContents('fuse_test_tag', [bar_uuid]) baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid'] l = self.tag_collection(baz_uuid, 'fuse_test_tag') for attempt in AssertWithTimeout(10): attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid]) self.api.links().delete(uuid=l['uuid']).execute() for attempt in AssertWithTimeout(10): attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
def runTest(self): collection = arvados.collection.Collection(api_client=self.api) collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual([], sorted(d1)) # See note in MountTestBase.setUp self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp, collection.manifest_locator(), self.keeptmp, arvados.config.settings()))
def verify_pdh_only(self, pdh_only=False, skip_pdh_only=False): if skip_pdh_only is True: self.make_mount(fuse.MagicDirectory) # in this case, the default by_id applies else: self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or arvados.util.uuid_pattern.match(fn) for fn in mount_ls), "new FUSE MagicDirectory lists Collection") # look up using pdh should succeed in all cases self.assertDirContents(self.testcollection, ['thing1.txt']) self.assertDirContents(os.path.join('by_id', self.testcollection), ['thing1.txt']) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertIn(self.testcollection, mount_ls) self.assertIn(self.testcollection, llfuse.listdir(os.path.join(self.mounttmp, 'by_id'))) files = {} files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1' for k, v in files.items(): with open(os.path.join(self.mounttmp, k)) as f: self.assertEqual(v, f.read()) # look up using uuid should fail when pdh_only is set if pdh_only is True: with self.assertRaises(OSError): self.assertDirContents(os.path.join('by_id', self.testcollectionuuid), ['thing1.txt']) else: self.assertDirContents(os.path.join('by_id', self.testcollectionuuid), ['thing1.txt'])
def runTest(self): collection = arvados.collection.Collection(api_client=self.api) collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) self.assertTrue(m.writable()) self.pool.apply(fuseRenameTestHelper, (self.mounttmp, )) # Starting manifest collection2 = self.api.collections().get( uuid=collection.manifest_locator()).execute() self.assertRegexpMatches( collection2["manifest_text"], r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$' ) d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual(["testdir"], d1) d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir")) self.assertEqual(["file1.txt"], d1) os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2")) d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual(["testdir2"], sorted(d1)) d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2")) self.assertEqual(["file1.txt"], d1) collection2 = self.api.collections().get( uuid=collection.manifest_locator()).execute() self.assertRegexpMatches( collection2["manifest_text"], r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$' )
def runTest(self): collection = arvados.collection.Collection(api_client=self.api) collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) self.operations.listen_for_events(self.api) d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual([], sorted(d1)) with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2: with collection2.open("file1.txt", "w") as f: f.write("foo") time.sleep(1) # should show up via event bus notify d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual(["file1.txt"], sorted(d1))
def runTest(self): self.make_mount(fuse.MagicDirectory) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or arvados.util.uuid_pattern.match(fn) for fn in mount_ls), "new FUSE MagicDirectory lists Collection") self.assertDirContents(self.testcollection, ['thing1.txt']) self.assertDirContents(os.path.join('by_id', self.testcollection), ['thing1.txt']) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertIn(self.testcollection, mount_ls) self.assertIn(self.testcollection, llfuse.listdir(os.path.join(self.mounttmp, 'by_id'))) files = {} files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1' for k, v in files.items(): with open(os.path.join(self.mounttmp, k)) as f: self.assertEqual(v, f.read())
def runTest(self): with open(os.path.join(mounttmp, "file1.txt"), "w") as f: f.write("bar") d1 = sorted(llfuse.listdir(os.path.join(mounttmp))) self.assertEqual(len(d1), 2) with open(os.path.join(mounttmp, "file1.txt"), "r") as f: self.assertEqual(f.read(), "bar") self.assertRegexpMatches( d1[1], r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~') with open(os.path.join(mounttmp, d1[1]), "r") as f: self.assertEqual(f.read(), "foo")
def runTest(self): d1 = llfuse.listdir(mounttmp) self.assertNotIn('testcollection', d1) os.mkdir(os.path.join(mounttmp, "testcollection")) d1 = llfuse.listdir(mounttmp) self.assertIn('testcollection', d1) with self.assertRaises(OSError): os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data')) os.rename( os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection')) d1 = llfuse.listdir(mounttmp) self.assertNotIn('testcollection', d1) d1 = llfuse.listdir( os.path.join(mounttmp, 'Unrestricted public data')) self.assertIn('testcollection', d1)
def runTest(self): collection = arvados.collection.Collection(api_client=self.api) collection.save_new() m = self.make_mount(fuse.CollectionDirectory) with llfuse.lock: m.new_collection(collection.api_response(), collection) d1 = llfuse.listdir(os.path.join(self.mounttmp)) self.assertEqual([], sorted(d1)) with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2: with collection2.open("file1.txt", "w") as f: f.write("foo") # See note in MountTestBase.setUp self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
def runTest(self): self.make_mount(fuse.MagicDirectory) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertFalse( any( arvados.util.keep_locator_pattern.match(fn) or arvados.util.uuid_pattern.match(fn) for fn in mount_ls), "new FUSE MagicDirectory has no collections or projects") self.assertDirContents(self.testcollection, ['thing1.txt']) self.assertDirContents(os.path.join('by_id', self.testcollection), ['thing1.txt']) self.assertIn( self.collection_in_test_project, llfuse.listdir(os.path.join(self.mounttmp, self.test_project))) self.assertIn( self.collection_in_test_project, llfuse.listdir( os.path.join(self.mounttmp, 'by_id', self.test_project))) mount_ls = llfuse.listdir(self.mounttmp) self.assertIn('README', mount_ls) self.assertIn(self.testcollection, mount_ls) self.assertIn(self.testcollection, llfuse.listdir(os.path.join(self.mounttmp, 'by_id'))) self.assertIn(self.test_project, mount_ls) self.assertIn(self.test_project, llfuse.listdir(os.path.join(self.mounttmp, 'by_id'))) with self.assertRaises(OSError): llfuse.listdir( os.path.join(self.mounttmp, 'by_id', self.non_project_group)) files = {} files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1' for k, v in viewitems(files): with open(os.path.join(self.mounttmp, k), 'rb') as f: self.assertEqual(v, f.read().decode())
def tst_link(self): name1 = os.path.join(self.mnt_dir, self.newname()) name2 = os.path.join(self.mnt_dir, self.newname()) src = self.src shutil.copyfile(src, name1) assert filecmp.cmp(name1, src, False) os.link(name1, name2) fstat1 = os.lstat(name1) fstat2 = os.lstat(name2) assert fstat1 == fstat2 assert fstat1.st_nlink == 2 assert basename(name2) in llfuse.listdir(self.mnt_dir) assert filecmp.cmp(name1, name2, False) os.unlink(name2) fstat1 = os.lstat(name1) assert fstat1.st_nlink == 1 os.unlink(name1)
def tst_link(self): name1 = os.path.join(self.mnt_dir, self.newname()) name2 = os.path.join(self.mnt_dir, self.newname()) src = self.src shutil.copyfile(src, name1) self.assertTrue(filecmp.cmp(name1, src, False)) os.link(name1, name2) fstat1 = os.lstat(name1) fstat2 = os.lstat(name2) self.assertEqual(fstat1, fstat2) self.assertEqual(fstat1.st_nlink, 2) self.assertTrue(basename(name2) in llfuse.listdir(self.mnt_dir)) self.assertTrue(filecmp.cmp(name1, name2, False)) os.unlink(name2) fstat1 = os.lstat(name1) self.assertEqual(fstat1.st_nlink, 1) os.unlink(name1)
def runTest(self): os.environ['KEEP_LOCAL_STORE'] = keeptmp with open(os.path.join(mounttmp, "file1.txt"), "w") as f: with arvados.collection.Collection(uuid, api_client=arvados.api_from_config('v1', apiconfig=settings)) as collection2: with collection2.open("file1.txt", "w") as f2: f2.write("foo") f.write("bar") d1 = sorted(llfuse.listdir(os.path.join(mounttmp))) self.assertEqual(len(d1), 2) with open(os.path.join(mounttmp, "file1.txt"), "r") as f: self.assertEqual(f.read(), "bar") assertRegex(self, d1[1], r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~') with open(os.path.join(mounttmp, d1[1]), "r") as f: self.assertEqual(f.read(), "foo")
def assert_s3ql_fs(path): '''Raise `QuietError` if *path* is not on an S3QL file system Returns name of the S3QL control file. ''' try: os.stat(path) except FileNotFoundError: raise QuietError('%s does not exist' % path) except OSError as exc: if exc.errno is errno.ENOTCONN: raise QuietError('File system appears to have crashed.') raise ctrlfile = os.path.join(path, CTRL_NAME) if not (CTRL_NAME not in llfuse.listdir(path) and os.path.exists(ctrlfile)): raise QuietError('%s is not on an S3QL file system' % path) return ctrlfile
def tst_chown(self): filename = os.path.join(self.mnt_dir, self.newname()) os.mkdir(filename) fstat = os.lstat(filename) uid = fstat.st_uid gid = fstat.st_gid uid_new = uid + 1 os.chown(filename, uid_new, -1) fstat = os.lstat(filename) self.assertEqual(fstat.st_uid, uid_new) self.assertEqual(fstat.st_gid, gid) gid_new = gid + 1 os.chown(filename, -1, gid_new) fstat = os.lstat(filename) self.assertEqual(fstat.st_uid, uid_new) self.assertEqual(fstat.st_gid, gid_new) os.rmdir(filename) self.assertRaises(FileNotFoundError, os.stat, filename) self.assertTrue(basename(filename) not in llfuse.listdir(self.mnt_dir))
def tst_readdir(self): dir_ = os.path.join(self.mnt_dir, self.newname()) file_ = dir_ + "/" + self.newname() subdir = dir_ + "/" + self.newname() subfile = subdir + "/" + self.newname() src = self.src os.mkdir(dir_) shutil.copyfile(src, file_) os.mkdir(subdir) shutil.copyfile(src, subfile) listdir_is = llfuse.listdir(dir_) listdir_is.sort() listdir_should = [basename(file_), basename(subdir)] listdir_should.sort() assert listdir_is == listdir_should os.unlink(file_) os.unlink(subfile) os.rmdir(subdir) os.rmdir(dir_)
def tst_chown(self): filename = os.path.join(self.mnt_dir, self.newname()) os.mkdir(filename) fstat = os.lstat(filename) uid = fstat.st_uid gid = fstat.st_gid uid_new = uid + 1 os.chown(filename, uid_new, -1) fstat = os.lstat(filename) assert fstat.st_uid == uid_new assert fstat.st_gid == gid gid_new = gid + 1 os.chown(filename, -1, gid_new) fstat = os.lstat(filename) assert fstat.st_uid == uid_new assert fstat.st_gid == gid_new os.rmdir(filename) assert_raises(FileNotFoundError, os.stat, filename) assert basename(filename) not in llfuse.listdir(self.mnt_dir)
def readContentsFromCollectionWithMultipleBlocks(self): for i in range(0, streams): d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i))) for j in range(0, files_per_stream): with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f: self.assertEqual(data, f.read())
def runTest(self): d1 = llfuse.listdir(mounttmp) self.assertEqual(["file1.txt"], d1) with open(os.path.join(mounttmp, "file1.txt")) as f: self.assertEqual("plnp", f.read())
def test_listdir(): # There is a race-condition here if /usr/bin is modified while the test # runs - but hopefully this is sufficiently rare. list1 = set(os.listdir('/usr/bin')) list2 = set(llfuse.listdir('/usr/bin')) assert list1 == list2
def assertDirContents(self, subdir, expect_content): path = self.mounttmp if subdir: path = os.path.join(path, subdir) self.assertEqual(sorted(expect_content), sorted(llfuse.listdir(path)))
def getProjectWithManyCollections(self): project_contents = llfuse.listdir(self.mounttmp) self.assertEqual(201, len(project_contents)) self.assertIn('Collection_1', project_contents) return project_contents
def runTest(self): self.make_mount(fuse.SharedDirectory, exclude=self.api.users().current().execute()['uuid']) # shared_dirs is a list of the directories exposed # by fuse.SharedDirectory (i.e. any object visible # to the current user) shared_dirs = llfuse.listdir(self.mounttmp) shared_dirs.sort() self.assertIn('FUSE User', shared_dirs) # fuse_user_objs is a list of the objects owned by the FUSE # test user (which present as files in the 'FUSE User' # directory) fuse_user_objs = llfuse.listdir( os.path.join(self.mounttmp, 'FUSE User')) fuse_user_objs.sort() self.assertEqual( [ 'FUSE Test Project', # project owned by user 'collection #1 owned by FUSE', # collection owned by user 'collection #2 owned by FUSE', # collection owned by user 'pipeline instance owned by FUSE.pipelineInstance', # pipeline instance owned by user ], fuse_user_objs) # test_proj_files is a list of the files in the FUSE Test Project. test_proj_files = llfuse.listdir( os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project')) test_proj_files.sort() self.assertEqual([ 'collection in FUSE project', 'pipeline instance in FUSE project.pipelineInstance', 'pipeline template in FUSE project.pipelineTemplate' ], test_proj_files) # Double check that we can open and read objects in this folder as a file, # and that its contents are what we expect. pipeline_template_path = os.path.join( self.mounttmp, 'FUSE User', 'FUSE Test Project', 'pipeline template in FUSE project.pipelineTemplate') with open(pipeline_template_path) as f: j = json.load(f) self.assertEqual("pipeline template in FUSE project", j['name']) # check mtime on template st = os.stat(pipeline_template_path) try: mtime = st.st_mtime_ns / 1000000000 except AttributeError: mtime = st.st_mtime self.assertEqual(mtime, 1397493304) # check mtime on collection st = os.stat( os.path.join(self.mounttmp, 'FUSE User', 'collection #1 owned by FUSE')) try: mtime = st.st_mtime_ns / 1000000000 except AttributeError: mtime = st.st_mtime self.assertEqual(mtime, 1391448174)