def test_repoversion(self):
"""Check versioning in repos.
This test creates a repo and creates different versions.
"""
# create store object and repo
dbdir = tempfile.mkdtemp()
store = DicedStore(dbdir)
try:
# my initial repo
store.create_repo("myrepo")
myrepo = store.open_repo("myrepo")
# check current version
uuid = myrepo.get_current_version()
uuid2 = store.list_repos()[0][1]
self.assertEqual(uuid, uuid2)
# check lock status
self.assertFalse(myrepo.locked)
# check branch on open node is illegal
try:
myrepo.create_branch("start")
except DicedException:
pass
else:
assert False, "Expected an exception above, but it wasn't raised!"
# lock node
myrepo.lock_node("now done")
self.assertTrue(myrepo.locked)
# create new branch
uuidnew = myrepo.create_branch("start")
self.assertNotEqual(uuid, uuidnew)
# open repo with uuid
myrepov2 = store.open_repo(uuid=uuidnew)
self.assertEqual(myrepov2.get_current_version(), uuidnew)
# check log history
logs = myrepov2.get_commit_log()
self.assertEqual(logs[0][0], uuid)
self.assertEqual(logs[0][1], "now done")
self.assertEqual(logs[1][0], uuidnew)
self.assertEqual(logs[1][1], "(open node)")
# change version
myrepo.change_version(uuidnew)
self.assertFalse(myrepo.locked)
self.assertEqual(myrepo.uuid, uuidnew)
finally:
store._shutdown_store()
shutil.rmtree(dbdir)
def fri_get_image_generator(get_image_args, qq):
if not get_image_args:
return
dvid_server = get_image_args[0][1]
dvid_uuid = get_image_args[0][2]
instance_name = get_image_args[0][7]
if dvid_server[:5] == 'gs://': # DICED google bucket
using_diced = True
if dvid_server.find(',') >= 0:
store = DicedStore(*dvid_server.split(','))
else:
store = DicedStore(dvid_server)
repo = store.open_repo(uuid=dvid_uuid)
dvid_node = repo.get_array(instance_name)
else:
using_diced = False
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
for gg in get_image_args:
while qq.qsize() >= 2:
time.sleep(10)
qq.put(fri_get_image(gg, dvid_node, using_diced, instance_name))
if using_diced:
store._shutdown_store()
def gen_full_tab_roi(filename,
store_name,
repo_uuid,
crop=[None, None, None],
n_splits=1,
step_size=512):
store = DicedStore(store_name)
repo = store.open_repo(uuid=repo_uuid)
imdata = repo.get_array('grayscale')
extents = imdata.get_extents()
store._shutdown_store()
for ii in range(len(extents)):
if crop[ii] is None:
crop[ii] = [extents[ii].start, extents[ii].stop]
file_idx = 0
f_out = open('%s_%02d.txt' % (filename, file_idx), 'w')
count_idx = 0
max_count = int(
np.ceil((np.ceil(
(crop[0][1] - crop[0][0]) / float(step_size)) * np.ceil(
(crop[1][1] - crop[1][0]) / float(step_size)) * np.ceil(
(crop[2][1] - crop[2][0]) / float(step_size))) / n_splits))
for zz in range(crop[0][0], crop[0][1], step_size):
for yy in range(crop[1][0], crop[1][1], step_size):
for xx in range(crop[2][0], crop[2][1], step_size):
f_out.write('%d,%d,%d,%d\n' % (step_size, zz, yy, xx))
count_idx += 1
if count_idx == max_count:
count_idx = 0
file_idx += 1
f_out.close()
f_out = open('%s_%02d.txt' % (filename, file_idx), 'w')
from diced import DicedStore, ArrayDtype
import time
store = DicedStore("/data/repo.diced") # location of diced repo
#repo = store.create_repo("firstrepo", "My first repo")
while True:
time.sleep(.5)
def load_data(data):
store = DicedStore(data['url'])
repo = store.open_repo(uuid=data['repo'])
full_im = repo.get_array(data['array'])
return full_im
def test_initserver(self):
"""Check running of default dvid server and potential errors.
"""
defstore2 = None
dbdir = tempfile.mkdtemp()
dbdir2 = tempfile.mkdtemp()
defstore = DicedStore(dbdir)
try:
# dvid address
self.assertEqual(defstore._server, "127.0.0.1:8000")
# try running DVID on same address
try:
defstore = DicedStore(dbdir)
except DicedException:
pass # Good, should not be able to create duplicate store
else:
assert False, "Expected an exception above, but it wasn't raised!"
# use previous address
reusestore = DicedStore("dvid://127.0.0.1",
port=8000,
rpcport=8001)
self.assertEqual(reusestore._server, "127.0.0.1:8000")
# run a second dvid server
defstore2 = DicedStore(dbdir2, port=9000, rpcport=9001)
# dvid address
self.assertEqual(defstore2._server, "127.0.0.1:9000")
finally:
# shutdown DVID
defstore._shutdown_store()
if defstore2:
defstore2._shutdown_store()
# cleanup dirs
shutil.rmtree(dbdir)
shutil.rmtree(dbdir2)
def test_repos(self):
"""Test the creation, deletion, and querying of repos.
"""
dbdir = tempfile.mkdtemp()
store = DicedStore(dbdir, port=10000, rpcport=10001)
try:
store.create_repo("myrepo")
store.create_repo("myrepo1")
# make sure duplicate repos cannot be added
try:
store.create_repo("myrepo1")
except DicedException:
pass # Good, should not be able to create duplicate repo
else:
assert False, "Expected an exception above, but it wasn't raised!"
# grab repo names
reponames = store.list_repos()
self.assertEqual(len(reponames), 2)
nameonly = []
for (name, uuid) in reponames:
nameonly.append(name)
self.assertTrue("myrepo" in nameonly)
self.assertTrue("myrepo1" in nameonly)
# delete repo and check repo list
store.delete_repo("myrepo")
reponames = store.list_repos()
self.assertEqual(len(reponames), 1)
nameonly = []
for (name, uuid) in reponames:
nameonly.append(name)
self.assertFalse("myrepo" in nameonly)
self.assertTrue("myrepo1" in nameonly)
# test get uuid interface
uuid = store.get_repouuid("myrepo1")
self.assertEqual(uuid, reponames[0][1])
# retrieve DicedRepo object
repo = store.open_repo("myrepo1")
finally:
store._shutdown_store()
shutil.rmtree(dbdir)
import numpy as np
import h5py
import signal
from diced import DicedStore, ArrayDtype
hdf_path = "/home/vleite/Downloads/sample_A_20160501.hdf"
store = DicedStore("~/repo.diced") # location of diced repo
print("store " + str(store))
repoName = "22"
store.create_repo(repoName, "no dataset")
uuid = store.get_repouuid(repoName)
print(uuid)
repo = store.open_repo(repoName, uuid)
print("repo " + str(repo))
arr_raw = repo.create_array("raw", ArrayDtype.uint8) # default 3D array
hdf_file = h5py.File(hdf_path, "r")
lm_ds = hdf_file["volumes/raw"]
print(lm_ds.size)
arr_raw[0:125, 0:1250, 0:1250] = np.array(lm_ds, "uint8")
arr_label = repo.create_array("label", ArrayDtype.uint64) # default 3D array
lm_ds = hdf_file["volumes/labels/neuron_ids"]
arr_label[0:125, 0:1250, 0:1250] = np.array(lm_ds, "uint64")
# print(arr_raw.get_numdims())
# print(arr_raw.get_extents())
def test_files(self):
"""Test file interface in DicedRepo.
This test adds files, retrieves files, and delete files
from the version repo.
"""
# create store object and repo
dbdir = tempfile.mkdtemp()
store = DicedStore(dbdir)
try:
# my initial repo
store.create_repo("myrepo")
myrepo = store.open_repo("myrepo")
# create file
myrepo.upload_filedata("hello.txt", "world text")
res = myrepo.download_filedata("hello.txt")
self.assertEqual(res, b"world text")
# overwrite
myrepo.upload_filedata("hello.txt", "world text2")
res = myrepo.download_filedata("hello.txt")
self.assertEqual(res, b"world text2")
# new file
myrepo.upload_filedata("world.txt", "foobar")
# test file list
allfiles = myrepo.list_files()
self.assertEqual(len(allfiles), 2)
self.assertTrue("hello.txt" in allfiles)
self.assertTrue("world.txt" in allfiles)
# test error handling
try:
res = myrepo.download_filedata("hello2.txt")
except DicedException:
pass
else:
assert False, "Expected an exception above, but it wasn't raised!"
rootuuid = myrepo.get_current_version()
myrepo.lock_node("lock")
# check lock error message
try:
myrepo.upload_filedata("world.txt", "foobar")
except DicedException:
pass
else:
assert False, "Expected an exception above, but it wasn't raised!"
# create new branch and overwrite file data
newuuid = myrepo.create_branch("new branch")
myrepo.change_version(newuuid)
# modify data in child node
myrepo.upload_filedata("world.txt", "newdata")
res = myrepo.download_filedata("world.txt")
self.assertEqual(res, b"newdata")
myrepo.delete_file("hello.txt")
allfiles = myrepo.list_files()
self.assertEqual(len(allfiles), 1)
self.assertTrue("world.txt" in allfiles)
# text not overwritten in root
myrepo.change_version(rootuuid)
res = myrepo.download_filedata("world.txt")
self.assertEqual(res, b"foobar")
# file shouldn't be deleted in root
allfiles = myrepo.list_files()
self.assertEqual(len(allfiles), 2)
finally:
store._shutdown_store()
def test_array(self):
"""Test array creation/loading interface.
Tries creating different types of arrays,
checks proper deletion, and checks error
handling.
"""
# create store object and repo
dbdir = tempfile.mkdtemp()
store = DicedStore(dbdir)
try:
# my initial repo
store.create_repo("myrepo")
myrepo = store.open_repo("myrepo")
# test generic 3D raw array type
arr = myrepo.create_array("myarray", ArrayDtype.uint8)
self.assertEqual(arr.numdims, 3)
self.assertFalse(arr.islabel3D)
self.assertFalse(arr.locked)
arr = myrepo.get_array("myarray")
self.assertEqual(arr.numdims, 3)
self.assertFalse(arr.islabel3D)
self.assertFalse(arr.locked)
# test 2D raw array type
arr = myrepo.create_array("myarray2d", ArrayDtype.uint8, dims=2)
self.assertEqual(arr.numdims, 2)
self.assertFalse(arr.islabel3D)
self.assertFalse(arr.locked)
arr = myrepo.get_array("myarray2d")
self.assertEqual(arr.numdims, 2)
self.assertFalse(arr.islabel3D)
self.assertFalse(arr.locked)
# check 3D labels
arr = myrepo.create_array("mylabelarray",
ArrayDtype.uint64,
islabel3D=True)
self.assertEqual(arr.numdims, 3)
self.assertTrue(arr.islabel3D)
self.assertFalse(arr.locked)
arr = myrepo.get_array("myarray2d")
self.assertEqual(arr.numdims, 2)
# query instances
instances = myrepo.list_instances()
self.assertEqual(len(instances), 3)
allinstances = myrepo.list_instances(showhidden=True)
self.assertEqual(len(allinstances), 5)
# test delete
myrepo.delete_array("myarray2d")
instances = myrepo.list_instances()
self.assertEqual(len(instances), 2)
# reinsert and change meta and lossy
arr = myrepo.create_array("myarray2d",
ArrayDtype.uint8,
dims=3,
lossycompression=True)
self.assertEqual(arr.numdims, 3)
self.assertFalse(arr.islabel3D)
self.assertFalse(arr.locked)
# test hidden files set in meta
ns = DVIDNodeService("127.0.0.1:8000",
myrepo.get_current_version())
ns.create_keyvalue("blahblah")
# reload myrepo
myrepo.change_version(myrepo.get_current_version())
allinstances = myrepo.list_instances(showhidden=True)
self.assertEqual(len(allinstances), 6)
self.assertTrue(("blahblah", "keyvalue") in allinstances)
# test hidden exclusion add
ns.put(".meta", "restrictions",
json.dumps(["myarray2d"]).encode('utf-8'))
myrepo.change_version(myrepo.get_current_version())
allinstances = myrepo.list_instances()
self.assertEqual(len(allinstances), 2)
# test array errors
try:
arr = myrepo.create_array("myarray", ArrayDtype.uint8)
except DicedException:
pass
else:
assert False, "Expected an exception above, but it wasn't raised!"
try:
arr = myrepo.create_array("myarray",
ArrayDtype.uint16,
islabel3D=True)
except DicedException:
pass
else:
assert False, "Expected an exception above, but it wasn't raised!"
try:
arr = myrepo.get_array("myarray3")
except DicedException:
pass
else:
assert False, "Expected an exception above, but it wasn't raised!"
# try to create array on locked node
myrepo.lock_node("blah")
try:
arr = myrepo.create_array("myarray4", ArrayDtype.uint16)
except DicedException:
pass
else:
assert False, "Expected an exception above, but it wasn't raised!"
# check lock status
arr = myrepo.get_array("myarray2d")
self.assertEqual(arr.numdims, 3)
self.assertFalse(arr.islabel3D)
self.assertTrue(arr.locked)
finally:
store._shutdown_store()