def test_filetree_node_file():
b = BlobStash()
b.cleanup()
try:
b.run()
client = FileTreeClient(api_key="123")
# Upload the README from the souce tree
node = client.put_node("README.md")
assert node.name == "README.md"
assert node.type == "file"
# Ensure we can get it back
node2 = client.node(node.ref)
assert node == node2
# Try downloading the file back using the `Node` object
try:
client.get_node(node, "README2.md")
assert subprocess.check_call(["diff", "README.md",
"README2.md"]) == 0
finally:
os.unlink("README2.md")
# Now using the string reference
try:
client.get_node(node.ref, "README3.md")
assert subprocess.check_call(["diff", "README.md",
"README3.md"]) == 0
finally:
os.unlink("README3.md")
finally:
b.shutdown()
b.cleanup()
def test_test_utils():
"""Ensure the BlobStash utils can spawn a server."""
b = BlobStash()
b.cleanup()
try:
b.run()
c = Client()
resp = c.request("GET", "/", raw=True)
assert resp.status_code == 404
finally:
b.shutdown()
b.cleanup()
def test_blobstore_client():
"""Ensure the BlobStash utils can spawn a server."""
b = BlobStash()
b.cleanup()
try:
b.run()
client = BlobStoreClient(api_key="123")
assert len(list(client)) == 0
assert len(list(client.iter())) == 0
# try a blob that does not exist
with pytest.raises(BlobNotFoundError):
client.get("0" * 64)
# (blake2b 256 bits for an empty string)
expected_empty_hash = (
"0e5751c026e543b2e8ab2eb06099daa1d1e5df47778f7787faab45cdf12fe3a8"
)
empty_blob = Blob.from_data(b"")
assert empty_blob.hash == expected_empty_hash
client.put(empty_blob)
fetched_empty_blob = client.get(expected_empty_hash)
assert empty_blob == fetched_empty_blob
blobs = [empty_blob]
for i in range(1000 - len(blobs)):
blob = Blob.from_data(os.urandom(1024 * 8))
client.put(blob)
blobs.append(blob)
def by_hash(blob):
return blob.hash
blobs = sorted(blobs, key=by_hash)
fetched_blobs = sorted([cblob for cblob in client], key=by_hash)
assert len(fetched_blobs) == len(blobs)
for i, blob_ref in enumerate(fetched_blobs):
assert blob_ref.hash == blobs[i].hash
blob = client.get(blob_ref.hash)
assert blob.data == blobs[i].data
finally:
b.shutdown()
b.cleanup()
def test_kvstore_client():
"""Ensure the BlobStash utils can spawn a server."""
b = BlobStash()
b.cleanup()
try:
b.run()
client = KVStoreClient(api_key="123")
KV_COUNT = 10
KV_VERSIONS_COUNT = 100
keys = {}
for x in range(KV_COUNT):
key = "k{}".format(x)
if key not in keys:
keys[key] = []
for y in range(KV_VERSIONS_COUNT):
val = "value.{}.{}".format(x, y)
kv = client.put("k{}".format(x), val, version=y + 1)
keys[key].append(kv)
for key in keys.keys():
kv = client.get(key)
assert kv == keys[key][-1]
versions = list(client.get_versions(key))
assert len(versions) == len(keys[key])
for i, kv in enumerate(versions):
assert kv == keys[key][KV_VERSIONS_COUNT - (1 + i)]
b.shutdown()
b.run(reindex=True)
for key in keys.keys():
kv = client.get(key)
assert kv == keys[key][-1]
versions = list(client.get_versions(key))
assert len(versions) == KV_VERSIONS_COUNT
for i, kv in enumerate(versions):
assert kv == keys[key][KV_VERSIONS_COUNT - (1 + i)]
rkeys = list(client.iter())
for kv in rkeys:
assert kv == keys[kv.key][-1]
finally:
print("done")
b.shutdown()
b.cleanup()
def test_docstore():
b = BlobStash()
b.cleanup()
try:
b.run()
client = DocStoreClient(api_key="123")
col1 = client.col1
for i in range(10):
col1.insert({"lol": i + 1})
col2 = client.col2
DOCS_COUNT = 1000
docs = []
for i in range(DOCS_COUNT):
doc = dict(hello=i)
col2.insert(doc)
docs.append(doc)
for doc in docs:
rdoc = col2.get_by_id(doc["_id"])
assert rdoc == doc
rdocs = []
for rdoc in col2.query():
rdocs.append(rdoc)
assert rdocs == docs[::-1]
col3 = client.col3
for i in range(50):
col3.insert({"i": i, "nested": {"i": i}, "l": [True, i]})
assert len(list(col3.query(Q["nested"]["i"] >= 25))) == 25
assert sorted(["col1", "col2", "col3"
]) == sorted([c.name for c in client.collections()])
finally:
b.shutdown()
b.cleanup()
def test_filetree_fs():
b = BlobStash()
b.cleanup()
try:
b.run()
client = FileTreeClient(api_key="123")
fs = client.fs("test_fs")
# Ensure we can't put a dir
with pytest.raises(FileTreeError):
fs.put_node("/path", "docs")
readme = fs.put_node("/README.md", "README.md")
root = fs.node()
assert len(root.children) == 1
readme2 = root.children[0]
assert readme2.name == "README.md"
assert readme2 == readme
# Put it twice
license = fs.put_node("/path/to/LICENSE", "LICENSE")
license = fs.put_node("/path/to/LICENSE", "LICENSE")
root = fs.node()
assert len(root.children) == 2
subdir = fs.node("/path")
assert len(subdir.children) == 1
assert subdir.children[0].name == "to"
assert subdir.children[0].type == "dir"
subdir2 = fs.node("/path/to")
assert len(subdir2.children) == 1
assert subdir2.children[0] == license
license2 = fs.node("/path/to/LICENSE")
assert license2 == license
finally:
b.shutdown()
b.cleanup()
def test_filetree_node_fileobj():
b = BlobStash()
b.cleanup()
try:
b.run()
client = FileTreeClient(api_key="123")
# Try to upload a basic fileobj
expected = b"Hello world"
node = client.fput_node("hello.txt", io.BytesIO(expected))
# Get it back and compare it
f = client.fget_node(node)
out = f.read()
assert out == expected
assert node.name == "hello.txt"
assert node.size == len(expected)
assert node.type == "file"
finally:
b.shutdown()
b.cleanup()
def test_filetree_fs_upload_download():
b = BlobStash()
b.cleanup()
try:
b.run()
client = FileTreeClient(api_key="123")
for i in ["1", "2"]:
fs = client.fs("source_code")
if i == "1":
with pytest.raises(FileTreeError):
# Can't upload a file as a dir
fs.upload("README.md")
fs.upload("blobstash")
try:
fs.download("blobstash" + i)
assert (subprocess.check_call(
["diff", "blobstash" + i, "blobstash"]) == 0)
finally:
shutil.rmtree("blobstash" + i)
finally:
b.shutdown()
b.cleanup()
logging_log_level = logging.INFO
log_level = 'error'
if os.getenv('BLOBSTASH_DEBUG'):
logging_log_level = logging.DEBUG
log_level = 'debug'
logging.basicConfig(level=logging_log_level)
logging.info('Running integration tests...')
b = BlobStash(config='tests/blobstash.yaml')
b.cleanup()
c = Client()
logging.info('Start BlobStash')
b.run(log_level=log_level)
logging.info('[STEP 1] Testing the blob store...')
# FIXME(tsileo): only GET/POST at / and GET /{hash}
logging.info('Insert test blob')
blob = Blob.from_data(b'hello')
resp = c.put_blob(blob)
assert resp.status_code == 200, 'failed to put blob {}'.format(blob.hash)
logging.info('Fetch test blob back')
blob2 = c.get_blob(blob.hash, to_blob=True)
assert blob2.data == blob.data, 'failed to fetch blob {} != {}'.format(blob.data, blob2.data)
# TODO(tsileo): test 404 and malformed hash
logging_log_level = logging.INFO
log_level = 'error'
if os.getenv('BLOBSTASH_DEBUG'):
logging_log_level = logging.DEBUG
log_level = 'debug'
logging.basicConfig(level=logging_log_level)
logging.info('Running integration tests...')
b = BlobStash(config='tests/blobstash.yaml')
b.cleanup()
client = DocStoreClient(api_key='123')
logging.info('Start BlobStash')
b.run(log_level=log_level)
col1 = client.col1
for i in range(10):
col1.insert({'lol': i+1})
col2 = client.col2
COL = 'hello'
DOCS_COUNT = 1000
docs = []
for i in range(DOCS_COUNT):
doc = dict(hello=i)
resp = col2.insert(doc)
docs.append(doc)
MORE_BLOBS = 999
logging_log_level = logging.INFO
log_level = 'error'
if os.getenv('BLOBSTASH_DEBUG'):
logging_log_level = logging.DEBUG
log_level = 'debug'
logging.basicConfig(level=logging_log_level)
logging.info('Running integration tests...')
b = BlobStash(config='tests/blobstash.yaml')
b.cleanup()
c = Client()
logging.info('Start BlobStash')
b.run(log_level=log_level)
logging.info('[STEP 1] Testing the blob store...')
# FIXME(tsileo): only GET/POST at / and GET /{hash}
logging.info('Insert test blob')
blob = Blob.from_data(b'hello')
resp = c.put_blob(blob)
assert resp.status_code == 200, 'failed to put blob {}'.format(blob.hash)
logging.info('Fetch test blob back')
blob2 = c.get_blob(blob.hash, to_blob=True)
assert blob2.data == blob.data, 'failed to fetch blob {} != {}'.format(
blob.data, blob2.data)
# TODO(tsileo): test 404 and malformed hash
from blobstash.base.test_utils import BlobStash
logging_log_level = logging.INFO
log_level = 'error'
if os.getenv('BLOBSTASH_DEBUG'):
logging_log_level = logging.DEBUG
log_level = 'debug'
logging.basicConfig(level=logging_log_level)
logging.info('Running integration tests...')
b = BlobStash(config='tests/blobstash.yaml')
b.cleanup()
client = DocStoreClient(api_key='123')
logging.info('Start BlobStash')
b.run(log_level=log_level)
col1 = client.col1
for i in range(10):
col1.insert({'lol': i + 1})
col2 = client.col2
COL = 'hello'
DOCS_COUNT = 1000
docs = []
for i in range(DOCS_COUNT):
doc = dict(hello=i)
resp = col2.insert(doc)
docs.append(doc)