def backend(): backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') be = local.Backend('local://' + backend_dir, None, None) try: yield be finally: be.close() shutil.rmtree(backend_dir)
def backend(): backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') plain_backend = local.Backend( Namespace(storage_url='local://' + backend_dir)) backend = ComprencBackend(b'schnorz', ('zlib', 6), plain_backend) try: yield backend finally: backend.close() shutil.rmtree(backend_dir)
def ctx(): ctx = Namespace() ctx.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') ctx.backend_pool = BackendPool(lambda: local.Backend( Namespace(storage_url='local://' + ctx.backend_dir))) ctx.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') ctx.max_obj_size = 1024 # Destructors are not guaranteed to run, and we can't unlink # the file immediately because apsw refers to it by name. # Therefore, we unlink the file manually in tearDown() ctx.dbfile = tempfile.NamedTemporaryFile(delete=False) ctx.db = Connection(ctx.dbfile.name) create_tables(ctx.db) init_tables(ctx.db) # Create an inode we can work with ctx.inode = 42 now_ns = time_ns() ctx.db.execute( "INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " "VALUES (?,?,?,?,?,?,?,?,?)", (ctx.inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, os.getuid(), os.getgid(), now_ns, now_ns, now_ns, 1, 32)) cache = BlockCache(ctx.backend_pool, ctx.db, ctx.cachedir + "/cache", ctx.max_obj_size * 100) ctx.cache = cache # Monkeypatch around the need for removal and upload threads cache.to_remove = DummyQueue(cache) class DummyDistributor: def put(ctx, arg, timeout=None): cache._do_upload(*arg) return True cache.to_upload = DummyDistributor() # Tested methods assume that they are called from # file system request handler s3ql.block_cache.lock = MockLock() s3ql.block_cache.lock_released = MockLock() try: yield ctx finally: ctx.cache.backend_pool = ctx.backend_pool ctx.cache.destroy() shutil.rmtree(ctx.cachedir) shutil.rmtree(ctx.backend_dir) ctx.dbfile.close() os.unlink(ctx.dbfile.name)
def ctx(): ctx = Namespace() ctx.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') plain_backend = local.Backend(Namespace( storage_url='local://' + ctx.backend_dir)) ctx.backend_pool = BackendPool(lambda: ComprencBackend(b'schwubl', ('zlib', 6), plain_backend)) ctx.backend = ctx.backend_pool.pop_conn() ctx.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') ctx.max_obj_size = 1024 # Destructors are not guaranteed to run, and we can't unlink # the file immediately because apsw refers to it by name. # Therefore, we unlink the file manually in tearDown() ctx.dbfile = tempfile.NamedTemporaryFile(delete=False) ctx.db = Connection(ctx.dbfile.name) create_tables(ctx.db) init_tables(ctx.db) # Tested methods assume that they are called from # file system request handler llfuse.lock.acquire() cache = BlockCache(ctx.backend_pool, ctx.db, ctx.cachedir + "/cache", ctx.max_obj_size * 5) ctx.block_cache = cache ctx.server = fs.Operations(cache, ctx.db, ctx.max_obj_size, InodeCache(ctx.db, 0)) ctx.server.init() # Monkeypatch around the need for removal and upload threads cache.to_remove = DummyQueue(cache) class DummyDistributor: def put(ctx, arg, timeout=None): cache._do_upload(*arg) return True cache.to_upload = DummyDistributor() # Keep track of unused filenames ctx.name_cnt = 0 yield ctx ctx.server.inodes.destroy() llfuse.lock.release() ctx.block_cache.destroy() shutil.rmtree(ctx.cachedir) shutil.rmtree(ctx.backend_dir) os.unlink(ctx.dbfile.name) ctx.dbfile.close()
async def ctx(): ctx = Namespace() ctx.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') ctx.backend_pool = BackendPool(lambda: local.Backend( Namespace(storage_url='local://' + ctx.backend_dir))) ctx.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') ctx.max_obj_size = 1024 # Destructors are not guaranteed to run, and we can't unlink # the file immediately because apsw refers to it by name. # Therefore, we unlink the file manually in tearDown() ctx.dbfile = tempfile.NamedTemporaryFile(delete=False) ctx.db = Connection(ctx.dbfile.name) create_tables(ctx.db) init_tables(ctx.db) # Create an inode we can work with ctx.inode = 42 now_ns = time_ns() ctx.db.execute( "INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " "VALUES (?,?,?,?,?,?,?,?,?)", (ctx.inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, os.getuid(), os.getgid(), now_ns, now_ns, now_ns, 1, 32)) cache = BlockCache(ctx.backend_pool, ctx.db, ctx.cachedir + "/cache", ctx.max_obj_size * 100) cache.portal = trio.BlockingTrioPortal() ctx.cache = cache # Monkeypatch around the need for removal and upload threads cache.to_remove = DummyQueue(cache) class DummyChannel: async def send(self, arg): await trio.run_sync_in_worker_thread(cache._do_upload, *arg) cache.to_upload = (DummyChannel(), None) try: yield ctx finally: ctx.cache.backend_pool = ctx.backend_pool if ctx.cache.destroy is not None: await ctx.cache.destroy() shutil.rmtree(ctx.cachedir) shutil.rmtree(ctx.backend_dir) ctx.dbfile.close() os.unlink(ctx.dbfile.name)
def setUp(self): self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') self.backend = local.Backend('local://' + self.backend_dir, None, None) self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') self.max_obj_size = 1024 self.dbfile = tempfile.NamedTemporaryFile() self.db = Connection(self.dbfile.name) create_tables(self.db) init_tables(self.db) self.fsck = Fsck(self.cachedir, self.backend, {'max_obj_size': self.max_obj_size}, self.db) self.fsck.expect_errors = True
def test_clear(self): self.mkfs() proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + [ '--quiet', '--log', 'none', '--authfile', '/dev/null', 'clear', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) print('yes', file=proc.stdin) proc.stdin.close() self.assertEqual(proc.wait(), 0) plain_backend = local.Backend(Namespace(storage_url=self.storage_url)) assert list(plain_backend.list()) == []
async def ctx(): ctx = Namespace() ctx.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') plain_backend = local.Backend( Namespace(storage_url='local://' + ctx.backend_dir)) ctx.backend_pool = BackendPool( lambda: ComprencBackend(b'schwubl', ('zlib', 6), plain_backend)) ctx.backend = ctx.backend_pool.pop_conn() ctx.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') ctx.max_obj_size = 1024 # Destructors are not guaranteed to run, and we can't unlink # the file immediately because apsw refers to it by name. # Therefore, we unlink the file manually in tearDown() ctx.dbfile = tempfile.NamedTemporaryFile(delete=False) ctx.db = Connection(ctx.dbfile.name) create_tables(ctx.db) init_tables(ctx.db) cache = BlockCache(ctx.backend_pool, ctx.db, ctx.cachedir + "/cache", ctx.max_obj_size * 5) cache.portal = trio.BlockingTrioPortal() ctx.cache = cache ctx.server = fs.Operations(cache, ctx.db, ctx.max_obj_size, InodeCache(ctx.db, 0)) ctx.server.init() # Monkeypatch around the need for removal and upload threads cache.to_remove = DummyQueue(cache) class DummyChannel: async def send(self, arg): await trio.run_sync_in_worker_thread(cache._do_upload, *arg) cache.to_upload = (DummyChannel(), None) # Keep track of unused filenames ctx.name_cnt = 0 yield ctx ctx.server.inodes.destroy() await ctx.cache.destroy() shutil.rmtree(ctx.cachedir) shutil.rmtree(ctx.backend_dir) os.unlink(ctx.dbfile.name) ctx.dbfile.close()
def setUp(self): self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') self.backend_pool = BackendPool( lambda: local.Backend('local://' + self.backend_dir, None, None)) self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') self.max_obj_size = 1024 # Destructors are not guaranteed to run, and we can't unlink # the file immediately because apsw refers to it by name. # Therefore, we unlink the file manually in tearDown() self.dbfile = tempfile.NamedTemporaryFile(delete=False) self.db = Connection(self.dbfile.name) create_tables(self.db) init_tables(self.db) # Create an inode we can work with self.inode = 42 self.db.execute( "INSERT INTO inodes (id,mode,uid,gid,mtime,atime,ctime,refcount,size) " "VALUES (?,?,?,?,?,?,?,?,?)", (self.inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1, 32)) cache = BlockCache(self.backend_pool, self.db, self.cachedir + "/cache", self.max_obj_size * 100) self.cache = cache # Monkeypatch around the need for removal and upload threads cache.to_remove = DummyQueue(cache) class DummyDistributor: def put(self, arg, timeout=None): cache._do_upload(*arg) return True cache.to_upload = DummyDistributor() # Tested methods assume that they are called from # file system request handler llfuse.lock.acquire()
def setUp(self): self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') plain_backend = local.Backend('local://' + self.backend_dir, None, None) self.backend_pool = BackendPool( lambda: ComprencBackend(b'schwubl', ('zlib', 6), plain_backend)) self.backend = self.backend_pool.pop_conn() self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') self.max_obj_size = 1024 # Destructors are not guaranteed to run, and we can't unlink # the file immediately because apsw refers to it by name. # Therefore, we unlink the file manually in tearDown() self.dbfile = tempfile.NamedTemporaryFile(delete=False) self.db = Connection(self.dbfile.name) create_tables(self.db) init_tables(self.db) # Tested methods assume that they are called from # file system request handler llfuse.lock.acquire() cache = BlockCache(self.backend_pool, self.db, self.cachedir + "/cache", self.max_obj_size * 5) self.block_cache = cache self.server = fs.Operations(cache, self.db, self.max_obj_size, InodeCache(self.db, 0)) self.server.init() # Monkeypatch around the need for removal and upload threads cache.to_remove = DummyQueue(cache) class DummyDistributor: def put(self, arg, timeout=None): cache._do_upload(*arg) return True cache.to_upload = DummyDistributor() # Keep track of unused filenames self.name_cnt = 0
def test_passphrase(self): self.mkfs() passphrase_new = 'sd982jhd' proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + [ '--quiet', '--fatal-warnings', '--log', 'none', '--authfile', '/dev/null', 'passphrase', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) print(self.passphrase, file=proc.stdin) print(passphrase_new, file=proc.stdin) print(passphrase_new, file=proc.stdin) proc.stdin.close() self.assertEqual(proc.wait(), 0) plain_backend = local.Backend(self.storage_url, None, None) backend = ComprencBackend(passphrase_new.encode(), ('zlib', 6), plain_backend) backend.fetch('s3ql_passphrase') # will fail with wrong pw
def test_key_recovery(self): mkfs_output = self.mkfs() hit = re.search( r'^---BEGIN MASTER KEY---\n' r'(.+)\n' r'---END MASTER KEY---$', mkfs_output, re.MULTILINE) assert hit master_key = hit.group(1) plain_backend = local.Backend(Namespace(storage_url=self.storage_url)) del plain_backend['s3ql_passphrase'] # Oops backend = ComprencBackend(self.passphrase.encode(), ('zlib', 6), plain_backend) with pytest.raises(CorruptedObjectError): backend.fetch('s3ql_metadata') passphrase_new = 'sd982jhd' proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + [ '--quiet', '--log', 'none', '--authfile', '/dev/null', 'recover-key', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) print(master_key, file=proc.stdin) print(passphrase_new, file=proc.stdin) print(passphrase_new, file=proc.stdin) proc.stdin.close() self.assertEqual(proc.wait(), 0) backend = ComprencBackend(passphrase_new.encode(), ('zlib', 6), plain_backend) backend.fetch('s3ql_passphrase') # will fail with wrong pw
def setUp(self): self.name_cnt = 0 self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') self.plain_backend = local.Backend('local://' + self.backend_dir, None, None) self.backend = self._wrap_backend() self.retries = 0