def test_filetree_node_file(): b = BlobStash() b.cleanup() try: b.run() client = FileTreeClient(api_key="123") # Upload the README from the souce tree node = client.put_node("README.md") assert node.name == "README.md" assert node.type == "file" # Ensure we can get it back node2 = client.node(node.ref) assert node == node2 # Try downloading the file back using the `Node` object try: client.get_node(node, "README2.md") assert subprocess.check_call(["diff", "README.md", "README2.md"]) == 0 finally: os.unlink("README2.md") # Now using the string reference try: client.get_node(node.ref, "README3.md") assert subprocess.check_call(["diff", "README.md", "README3.md"]) == 0 finally: os.unlink("README3.md") finally: b.shutdown() b.cleanup()
def test_test_utils(): """Ensure the BlobStash utils can spawn a server.""" b = BlobStash() b.cleanup() try: b.run() c = Client() resp = c.request("GET", "/", raw=True) assert resp.status_code == 404 finally: b.shutdown() b.cleanup()
def test_blobstore_client(): """Ensure the BlobStash utils can spawn a server.""" b = BlobStash() b.cleanup() try: b.run() client = BlobStoreClient(api_key="123") assert len(list(client)) == 0 assert len(list(client.iter())) == 0 # try a blob that does not exist with pytest.raises(BlobNotFoundError): client.get("0" * 64) # (blake2b 256 bits for an empty string) expected_empty_hash = ( "0e5751c026e543b2e8ab2eb06099daa1d1e5df47778f7787faab45cdf12fe3a8" ) empty_blob = Blob.from_data(b"") assert empty_blob.hash == expected_empty_hash client.put(empty_blob) fetched_empty_blob = client.get(expected_empty_hash) assert empty_blob == fetched_empty_blob blobs = [empty_blob] for i in range(1000 - len(blobs)): blob = Blob.from_data(os.urandom(1024 * 8)) client.put(blob) blobs.append(blob) def by_hash(blob): return blob.hash blobs = sorted(blobs, key=by_hash) fetched_blobs = sorted([cblob for cblob in client], key=by_hash) assert len(fetched_blobs) == len(blobs) for i, blob_ref in enumerate(fetched_blobs): assert blob_ref.hash == blobs[i].hash blob = client.get(blob_ref.hash) assert blob.data == blobs[i].data finally: b.shutdown() b.cleanup()
def test_kvstore_client(): """Ensure the BlobStash utils can spawn a server.""" b = BlobStash() b.cleanup() try: b.run() client = KVStoreClient(api_key="123") KV_COUNT = 10 KV_VERSIONS_COUNT = 100 keys = {} for x in range(KV_COUNT): key = "k{}".format(x) if key not in keys: keys[key] = [] for y in range(KV_VERSIONS_COUNT): val = "value.{}.{}".format(x, y) kv = client.put("k{}".format(x), val, version=y + 1) keys[key].append(kv) for key in keys.keys(): kv = client.get(key) assert kv == keys[key][-1] versions = list(client.get_versions(key)) assert len(versions) == len(keys[key]) for i, kv in enumerate(versions): assert kv == keys[key][KV_VERSIONS_COUNT - (1 + i)] b.shutdown() b.run(reindex=True) for key in keys.keys(): kv = client.get(key) assert kv == keys[key][-1] versions = list(client.get_versions(key)) assert len(versions) == KV_VERSIONS_COUNT for i, kv in enumerate(versions): assert kv == keys[key][KV_VERSIONS_COUNT - (1 + i)] rkeys = list(client.iter()) for kv in rkeys: assert kv == keys[kv.key][-1] finally: print("done") b.shutdown() b.cleanup()
def test_docstore(): b = BlobStash() b.cleanup() try: b.run() client = DocStoreClient(api_key="123") col1 = client.col1 for i in range(10): col1.insert({"lol": i + 1}) col2 = client.col2 DOCS_COUNT = 1000 docs = [] for i in range(DOCS_COUNT): doc = dict(hello=i) col2.insert(doc) docs.append(doc) for doc in docs: rdoc = col2.get_by_id(doc["_id"]) assert rdoc == doc rdocs = [] for rdoc in col2.query(): rdocs.append(rdoc) assert rdocs == docs[::-1] col3 = client.col3 for i in range(50): col3.insert({"i": i, "nested": {"i": i}, "l": [True, i]}) assert len(list(col3.query(Q["nested"]["i"] >= 25))) == 25 assert sorted(["col1", "col2", "col3" ]) == sorted([c.name for c in client.collections()]) finally: b.shutdown() b.cleanup()
def test_filetree_fs(): b = BlobStash() b.cleanup() try: b.run() client = FileTreeClient(api_key="123") fs = client.fs("test_fs") # Ensure we can't put a dir with pytest.raises(FileTreeError): fs.put_node("/path", "docs") readme = fs.put_node("/README.md", "README.md") root = fs.node() assert len(root.children) == 1 readme2 = root.children[0] assert readme2.name == "README.md" assert readme2 == readme # Put it twice license = fs.put_node("/path/to/LICENSE", "LICENSE") license = fs.put_node("/path/to/LICENSE", "LICENSE") root = fs.node() assert len(root.children) == 2 subdir = fs.node("/path") assert len(subdir.children) == 1 assert subdir.children[0].name == "to" assert subdir.children[0].type == "dir" subdir2 = fs.node("/path/to") assert len(subdir2.children) == 1 assert subdir2.children[0] == license license2 = fs.node("/path/to/LICENSE") assert license2 == license finally: b.shutdown() b.cleanup()
def test_filetree_node_fileobj(): b = BlobStash() b.cleanup() try: b.run() client = FileTreeClient(api_key="123") # Try to upload a basic fileobj expected = b"Hello world" node = client.fput_node("hello.txt", io.BytesIO(expected)) # Get it back and compare it f = client.fget_node(node) out = f.read() assert out == expected assert node.name == "hello.txt" assert node.size == len(expected) assert node.type == "file" finally: b.shutdown() b.cleanup()
def test_filetree_fs_upload_download(): b = BlobStash() b.cleanup() try: b.run() client = FileTreeClient(api_key="123") for i in ["1", "2"]: fs = client.fs("source_code") if i == "1": with pytest.raises(FileTreeError): # Can't upload a file as a dir fs.upload("README.md") fs.upload("blobstash") try: fs.download("blobstash" + i) assert (subprocess.check_call( ["diff", "blobstash" + i, "blobstash"]) == 0) finally: shutil.rmtree("blobstash" + i) finally: b.shutdown() b.cleanup()
logging_log_level = logging.INFO log_level = 'error' if os.getenv('BLOBSTASH_DEBUG'): logging_log_level = logging.DEBUG log_level = 'debug' logging.basicConfig(level=logging_log_level) logging.info('Running integration tests...') b = BlobStash(config='tests/blobstash.yaml') b.cleanup() c = Client() logging.info('Start BlobStash') b.run(log_level=log_level) logging.info('[STEP 1] Testing the blob store...') # FIXME(tsileo): only GET/POST at / and GET /{hash} logging.info('Insert test blob') blob = Blob.from_data(b'hello') resp = c.put_blob(blob) assert resp.status_code == 200, 'failed to put blob {}'.format(blob.hash) logging.info('Fetch test blob back') blob2 = c.get_blob(blob.hash, to_blob=True) assert blob2.data == blob.data, 'failed to fetch blob {} != {}'.format(blob.data, blob2.data) # TODO(tsileo): test 404 and malformed hash
logging_log_level = logging.INFO log_level = 'error' if os.getenv('BLOBSTASH_DEBUG'): logging_log_level = logging.DEBUG log_level = 'debug' logging.basicConfig(level=logging_log_level) logging.info('Running integration tests...') b = BlobStash(config='tests/blobstash.yaml') b.cleanup() client = DocStoreClient(api_key='123') logging.info('Start BlobStash') b.run(log_level=log_level) col1 = client.col1 for i in range(10): col1.insert({'lol': i+1}) col2 = client.col2 COL = 'hello' DOCS_COUNT = 1000 docs = [] for i in range(DOCS_COUNT): doc = dict(hello=i) resp = col2.insert(doc) docs.append(doc)
MORE_BLOBS = 999 logging_log_level = logging.INFO log_level = 'error' if os.getenv('BLOBSTASH_DEBUG'): logging_log_level = logging.DEBUG log_level = 'debug' logging.basicConfig(level=logging_log_level) logging.info('Running integration tests...') b = BlobStash(config='tests/blobstash.yaml') b.cleanup() c = Client() logging.info('Start BlobStash') b.run(log_level=log_level) logging.info('[STEP 1] Testing the blob store...') # FIXME(tsileo): only GET/POST at / and GET /{hash} logging.info('Insert test blob') blob = Blob.from_data(b'hello') resp = c.put_blob(blob) assert resp.status_code == 200, 'failed to put blob {}'.format(blob.hash) logging.info('Fetch test blob back') blob2 = c.get_blob(blob.hash, to_blob=True) assert blob2.data == blob.data, 'failed to fetch blob {} != {}'.format( blob.data, blob2.data) # TODO(tsileo): test 404 and malformed hash
from blobstash.base.test_utils import BlobStash logging_log_level = logging.INFO log_level = 'error' if os.getenv('BLOBSTASH_DEBUG'): logging_log_level = logging.DEBUG log_level = 'debug' logging.basicConfig(level=logging_log_level) logging.info('Running integration tests...') b = BlobStash(config='tests/blobstash.yaml') b.cleanup() client = DocStoreClient(api_key='123') logging.info('Start BlobStash') b.run(log_level=log_level) col1 = client.col1 for i in range(10): col1.insert({'lol': i + 1}) col2 = client.col2 COL = 'hello' DOCS_COUNT = 1000 docs = [] for i in range(DOCS_COUNT): doc = dict(hello=i) resp = col2.insert(doc) docs.append(doc)