def test_changing_version_should_not_affect_other_links(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1) data.seek(0) storage.store('world.txt', data, version=2) self.assertEqual(storage.stored_version('hello.txt'), 1) self.assertEqual(storage.stored_version('world.txt'), 2)
def test_store_should_respect_given_data_size(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1, size=2) storage_path = os.path.join(self.temp_dir, 'links', 'hello.txt') with gzip.open(storage_path, 'rb') as f: self.assertEqual(f.read(), b'he')
def test_store_should_add_file_to_storage(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1) storage_path = os.path.join(self.temp_dir, 'links', 'hello.txt') with gzip.open(storage_path, 'rb') as f: self.assertEqual(f.read(), b'hello')
def test_deleting_older_version_should_have_no_effect(self): storage = FileStorage(self.temp_dir) storage.store('hello.txt', BytesIO(b'world'), version=2) storage.delete('hello.txt', version=1) storage_path = os.path.join(self.temp_dir, 'links', 'hello.txt') with gzip.open(storage_path, 'rb') as f: self.assertEqual(f.read(), b'world')
def test_store_should_set_modified_time_to_version(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1) storage_path = os.path.join(self.temp_dir, 'links', 'hello.txt') storage_version = os.lstat(storage_path).st_mtime self.assertEqual(1, storage_version)
def test_blob_should_be_deleted_with_the_last_reference(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1) storage.delete('hello.txt', version=1) # check that blobs directory has no files in it # (empty directories are allowed) for _, _, files in os.walk(storage.blobs_dir): self.assertEqual(len(files), 0)
def test_store_should_not_overwrite_newer_files(self): storage = FileStorage(self.temp_dir) old_data = BytesIO(b'hello') new_data = BytesIO(b'world') storage.store('hello.txt', new_data, version=2) storage.store('hello.txt', old_data, version=1) storage_path = os.path.join(self.temp_dir, 'links', 'hello.txt') with gzip.open(storage_path, 'rb') as f: self.assertEqual(f.read(), b'world')
def test_store_should_reuse_blobs(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1) data.seek(0) storage.store('world.txt', data, version=1) storage_path_a = os.path.join(self.temp_dir, 'links', 'hello.txt') storage_path_b = os.path.join(self.temp_dir, 'links', 'world.txt') self.assertEqual(os.readlink(storage_path_a), os.readlink(storage_path_b))
def test_store_should_reuse_blobs(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1) data.seek(0) storage.store('world.txt', data, version=1) storage_path_a = os.path.join(self.temp_dir, 'links', 'hello.txt') storage_path_b = os.path.join(self.temp_dir, 'links', 'world.txt') self.assertEqual(os.readlink(storage_path_a), os.readlink(storage_path_b))
def test_removing_one_reference_should_not_break_others(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') storage.store('hello.txt', data, version=1) data.seek(0) storage.store('world.txt', data, version=1) storage.delete('hello.txt', version=1) path = os.path.join(self.temp_dir, 'links', 'world.txt') with gzip.open(path, 'rb') as f: self.assertEqual(f.read(), b'hello')
def test_store_should_accept_digest_hints(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') digest = hashlib.sha256(b'hello').hexdigest() storage.store('hello.txt', data, version=1) data.seek(0) storage.store('world.txt', data, version=1, digest=digest) storage_path_a = os.path.join(self.temp_dir, 'links', 'hello.txt') storage_path_b = os.path.join(self.temp_dir, 'links', 'world.txt') self.assertEqual(os.readlink(storage_path_a), os.readlink(storage_path_b))
def test_store_should_accept_digest_hints(self): storage = FileStorage(self.temp_dir) data = BytesIO(b'hello') digest = hashlib.sha256(b'hello').hexdigest() storage.store('hello.txt', data, version=1) data.seek(0) storage.store('world.txt', data, version=1, digest=digest) storage_path_a = os.path.join(self.temp_dir, 'links', 'hello.txt') storage_path_b = os.path.join(self.temp_dir, 'links', 'world.txt') self.assertEqual(os.readlink(storage_path_a), os.readlink(storage_path_b))
def test_store_should_add_compressed_file_to_storage_as_is(self): storage = FileStorage(self.temp_dir) raw_data = BytesIO(b'hello') gz_data = BytesIO() with gzip.GzipFile(fileobj=gz_data, mode='wb') as dst: shutil.copyfileobj(raw_data, dst) gz_data.seek(0) storage.store('hello.txt', gz_data, version=1, compressed=True) storage_path = os.path.join(self.temp_dir, 'links', 'hello.txt') with gzip.open(storage_path, 'rb') as f: self.assertEqual(f.read(), b'hello')
class FiletrackerServer(base.Server): """A WSGI application providing a filetracker server. Note that this wouldn't work as standalone server: a "manager" process should handle DB initialization and recovery, refer to ``filetracker.servers.run`` for more details. """ def __init__(self, dir=None): if dir is None: if 'FILETRACKER_DIR' not in os.environ: raise AssertionError( "LocalFileServer must have its working " "directory specified either as a constructor argument " "or passed via FILETRACKER_DIR environment variable.") dir = os.environ['FILETRACKER_DIR'] self.storage = FileStorage(dir) self.dir = self.storage.links_dir def parse_query_params(self, environ): return parse_qs(environ.get('QUERY_STRING', '')) def handle_PUT(self, environ, start_response): endpoint, path = base.get_endpoint_and_path(environ) if endpoint != 'files': raise base.HttpError('400 Bad Request', 'PUT can be only performed on "/files/..."') content_length = int(environ.get('CONTENT_LENGTH')) query_params = self.parse_query_params(environ) last_modified = query_params.get('last_modified', (None, ))[0] if last_modified: last_modified = email.utils.parsedate_tz(last_modified) last_modified = email.utils.mktime_tz(last_modified) else: raise base.HttpError('400 Bad Request', '"?last-modified=" is required') compressed = environ.get('HTTP_CONTENT_ENCODING', None) == 'gzip' digest = environ.get('HTTP_SHA256_CHECKSUM', None) logical_size = environ.get('HTTP_LOGICAL_SIZE', None) if compressed and digest and logical_size: logger.debug('Handling PUT %s.', path) else: logger.info( 'Handling PUT %s with unusual headers: ' 'compressed=%s, digest=%s, logical_size=%s', path, compressed, digest, logical_size) version = self.storage.store(name=path, data=environ['wsgi.input'], version=last_modified, size=content_length, compressed=compressed, digest=digest, logical_size=logical_size) start_response('200 OK', [ ('Content-Type', 'text/plain'), ('Last-Modified', email.utils.formatdate(version)), ]) return [] def _file_headers(self, name): link_st = os.lstat(os.path.join(self.dir, name)) blob_st = os.stat(os.path.join(self.dir, name)) logical_size = self.storage.logical_size(name) return [ ('Content-Type', 'application/octet-stream'), ('Content-Length', str(blob_st.st_size)), ('Content-Encoding', 'gzip'), ('Last-Modified', email.utils.formatdate(link_st.st_mtime)), ('Logical-Size', str(logical_size)), ] def handle_GET(self, environ, start_response): endpoint, path = base.get_endpoint_and_path(environ) if endpoint == 'list': return self.handle_list(environ, start_response) elif endpoint == 'version': return self.handle_version(environ, start_response) elif endpoint == 'files': full_path = os.path.join(self.dir, path) if not os.path.isfile(full_path): raise base.HttpError('404 Not Found', 'File "{}" not found'.format(full_path)) start_response('200 OK', self._file_headers(path)) return _FileIterator(open(full_path, 'rb')) else: raise base.HttpError( '400 Bad Request', 'Unknown endpoint "{}", expected "files" or "list"'.format( endpoint)) def handle_DELETE(self, environ, start_response): endpoint, path = base.get_endpoint_and_path(environ) if endpoint != 'files': raise HttpError('400 Bad Request', 'DELETE can be only performed on "/files/..."') query_params = self.parse_query_params(environ) last_modified = query_params.get('last_modified', (None, ))[0] if last_modified: last_modified = email.utils.parsedate_tz(last_modified) last_modified = email.utils.mktime_tz(last_modified) else: raise base.HttpError('400 Bad Request', '"?last-modified=" is required') logger.debug('Handling DELETE %s@%d', path, last_modified) try: self.storage.delete(name=path, version=last_modified) except FiletrackerFileNotFoundError: raise base.HttpError('404 Not Found', '') start_response('200 OK', []) return [] def handle_list(self, environ, start_response): _, path = base.get_endpoint_and_path(environ) query_params = self.parse_query_params(environ) last_modified = query_params.get('last_modified', (None, ))[0] if not last_modified: last_modified = int(time.time()) logger.debug('Handling GET /list/%s (@%d)', path, last_modified) root_dir = os.path.join(self.dir, path) if not os.path.isdir(root_dir): raise base.HttpError('400 Bad Request', 'Path doesn\'t exist or is not a directory') start_response('200 OK', []) return _list_files_iterator(root_dir, last_modified) def handle_version(self, environ, start_response): start_response('200 OK', [('Content-Type', 'application/json')]) response = { 'protocol_versions': [2], } return [json.dumps(response).encode('utf8')]
class FiletrackerServer(base.Server): """A WSGI application providing a filetracker server. Note that this wouldn't work as standalone server: a "manager" process should handle DB initialization and recovery, refer to ``filetracker.servers.run`` for more details. """ def __init__(self, dir=None): if dir is None: if 'FILETRACKER_DIR' not in os.environ: raise AssertionError("LocalFileServer must have its working " "directory specified either as a constructor argument " "or passed via FILETRACKER_DIR environment variable.") dir = os.environ['FILETRACKER_DIR'] self.storage = FileStorage(dir) self.dir = self.storage.links_dir def parse_query_params(self, environ): return parse_qs(environ.get('QUERY_STRING', '')) def handle_PUT(self, environ, start_response): endpoint, path = base.get_endpoint_and_path(environ) if endpoint != 'files': raise base.HttpError('400 Bad Request', 'PUT can be only performed on "/files/..."') content_length = int(environ.get('CONTENT_LENGTH')) query_params = self.parse_query_params(environ) last_modified = query_params.get('last_modified', (None,))[0] if last_modified: last_modified = email.utils.parsedate_tz(last_modified) last_modified = email.utils.mktime_tz(last_modified) else: raise base.HttpError('400 Bad Request', '"?last-modified=" is required') compressed = environ.get('HTTP_CONTENT_ENCODING', None) == 'gzip' digest = environ.get('HTTP_SHA256_CHECKSUM', None) logical_size = environ.get('HTTP_LOGICAL_SIZE', None) if compressed and digest and logical_size: logger.debug('Handling PUT %s.', path) else: logger.info('Handling PUT %s with unusual headers: ' 'compressed=%s, digest=%s, logical_size=%s', path, compressed, digest, logical_size) version = self.storage.store(name=path, data=environ['wsgi.input'], version=last_modified, size=content_length, compressed=compressed, digest=digest, logical_size=logical_size) start_response('200 OK', [ ('Content-Type', 'text/plain'), ('Last-Modified', email.utils.formatdate(version)), ]) return [] def _file_headers(self, name): link_st = os.lstat(os.path.join(self.dir, name)) blob_st = os.stat(os.path.join(self.dir, name)) logical_size = self.storage.logical_size(name) return [ ('Content-Type', 'application/octet-stream'), ('Content-Length', str(blob_st.st_size)), ('Content-Encoding', 'gzip'), ('Last-Modified', email.utils.formatdate(link_st.st_mtime)), ('Logical-Size', str(logical_size)), ] def handle_GET(self, environ, start_response): endpoint, path = base.get_endpoint_and_path(environ) if endpoint == 'list': return self.handle_list(environ, start_response) elif endpoint == 'version': return self.handle_version(environ, start_response) elif endpoint == 'files': full_path = os.path.join(self.dir, path) if not os.path.isfile(full_path): raise base.HttpError('404 Not Found', 'File "{}" not found'.format(full_path)) start_response('200 OK', self._file_headers(path)) return _FileIterator(open(full_path, 'rb')) else: raise base.HttpError( '400 Bad Request', 'Unknown endpoint "{}", expected "files" or "list"' .format(endpoint)) def handle_DELETE(self, environ, start_response): endpoint, path = base.get_endpoint_and_path(environ) if endpoint != 'files': raise HttpError('400 Bad Request', 'DELETE can be only performed on "/files/..."') query_params = self.parse_query_params(environ) last_modified = query_params.get('last_modified', (None,))[0] if last_modified: last_modified = email.utils.parsedate_tz(last_modified) last_modified = email.utils.mktime_tz(last_modified) else: raise base.HttpError('400 Bad Request', '"?last-modified=" is required') logger.debug('Handling DELETE %s@%d', path, last_modified) try: self.storage.delete(name=path, version=last_modified) except FiletrackerFileNotFoundError: raise base.HttpError('404 Not Found', '') start_response('200 OK', []) return [] def handle_list(self, environ, start_response): _, path = base.get_endpoint_and_path(environ) query_params = self.parse_query_params(environ) last_modified = query_params.get('last_modified', (None,))[0] if not last_modified: last_modified = int(time.time()) logger.debug('Handling GET /list/%s (@%d)', path, last_modified) root_dir = os.path.join(self.dir, path) if not os.path.isdir(root_dir): raise base.HttpError('400 Bad Request', 'Path doesn\'t exist or is not a directory') start_response('200 OK', []) return _list_files_iterator(root_dir, last_modified) def handle_version(self, environ, start_response): start_response('200 OK', [('Content-Type', 'application/json')]) response = { 'protocol_versions': [2], } return [json.dumps(response).encode('utf8')]