def test_array_instantiation_with_channel(self):
"""
Array can be instantiated.
.
"""
boss = BossRemote(_DEFAULT_BOSS_OPTIONS)
_ = array(boss.get_channel("cc", "kasthuri2015", "em"))
def test_array_instantiation_dtype(self):
"""
Array can be instantiated with type.
.
"""
boss = BossRemote(_DEFAULT_BOSS_OPTIONS)
test_array = array(boss.get_channel("cc", "kasthuri2015", "em"))
self.assertEqual(test_array.dtype, "uint8")
def test_can_retrieve_correct_u64_data(self):
data = array("bossdb://kasthuri2015/em/3cylneuron_v1")
boss = BossRemote(_DEFAULT_BOSS_OPTIONS)
get_cutout_data = boss.get_cutout(
boss.get_channel("3cylneuron_v1", "kasthuri2015", "em"),
0,
[6000, 6200],
[12000, 12500],
[923, 924],
)
self.assertTrue(
(get_cutout_data == data[923:924, 12000:12500, 6000:6200]).all()
)
# git pull
# pip install -U .
#You are ready!
#If you would like to get multiple images you can make a while loop or for loop, let me know
#if you want me to do it :)
boss = BossRemote({
"protocol": "https",
"host": "api.theboss.io",
#Remember to change your token here. You can get your own at: https://api.theboss.io/token/
"token": "Token"
})
#Here you will specify form where the data is coming from, the resolution, and the size of your image.
volume = boss.get_cutout(
boss.get_channel("cc", "kasthuri2015", "em"),
0,
[5000, 6000],
[8000, 9000],
[1100, 1200],
)
volume2 = boss.get_cutout(
boss.get_channel("mitochondria", "kasthuri2015", "em"),
0,
[5000, 6000],
[8000, 9000],
[1100, 1200],
)
volume3 = boss.get_cutout(
from intern.remote.boss import BossRemote
from intern.resource.boss.resource import ChannelResource
import matplotlib.pyplot as plt
import numpy as np
boss = BossRemote({
"protocol": "https",
"host": "api.theboss.io",
"token": "db1cec2c865fc84e48772f4f4a5f010c0a180b88"
})
#Here you will specify form where the data is coming from, the resolution, and the size of your image.
volume = boss.get_cutout(
boss.get_channel("em", "pinky40", "v7"),
0,
[10000, 10500],
[10000, 10500],
[500, 550],
)
print(volume)
plt.imshow(volume[1, :, :], cmap="gray")
plt.show()
class bossHandler:
"""
This is a class for interacting with BOSS. It basically is a wrapper for Intern.
"""
def __init__(self, collection_name):
"""
Constructor:
Takes the following argument:
:param collection_name: name of the collection in BOSS
:type collection_name: string
"""
self.collection_name = collection_name
try:
self.rmt = BossRemote()
except:
print('Unexpected Error:', sys.exc_info()[0])
# def get_collection_list(self):
# return self.rmt.list_collections()
def list_experiments(self):
"""
:return: all the experiments available in current collection
"""
exp_list = self.rmt.list_experiments(self.collection_name)
return exp_list
def select_experiment(self, experiment_name):
"""
Select an experiment to be added to this handler
"""
tmp = ExperimentResource(collection_name=self.collection_name,
name=experiment_name)
exp = self.rmt.get_project(tmp)
self.experiment = exp
# return exp
def get_experiment(self):
"""
:return: the currently selected experiment for this handler
"""
if hasattr(self, 'experiment'):
return self.experiment
else:
raise AttributeError(
'No experiment exists. First, select an experiment using select_experiment'
)
def list_channels(self):
"""
:return: all channel in currently selected experiment
"""
return self.rmt.list_channels(self.collection_name,
self.experiment.name)
def select_channel(self, channel_name):
"""
Select a channel to be added to this handler
"""
self.channel = self.rmt.get_channel(chan_name=channel_name,
coll_name=self.collection_name,
exp_name=self.experiment.name)
def get_coordinate_frame(self):
"""
Return: current experiment's coordinate frame
"""
tmp = CoordinateFrameResource(name=self.experiment.coord_frame)
coor = self.rmt.get_project(tmp)
self.coordinate_frame = coor
return coor
def get_all(self):
"""
:return: the entire channel image data at its native resolution
"""
x_rng = [self.coordinate_frame.x_start, self.coordinate_frame.x_stop]
y_rng = [self.coordinate_frame.y_start, self.coordinate_frame.y_stop]
z_rng = [self.coordinate_frame.z_start, self.coordinate_frame.z_stop]
return self.rmt.get_cutout(self.channel, 0, x_rng, y_rng, z_rng)
def get_cutout(self, x_range, y_range, z_range, resolution=0):
"""
:return: a cutout of the image data
"""
return self.rmt.get_cutout(self.channel, resolution, x_range, y_range,
z_range)
})
COLL = "kharris15"
EXP = "apical"
CHAN = "anno"
MESH_ID = 22
res = 0
x_rng = [4500, 4700]
y_rng = [3600, 3900]
z_rng = [0, 162]
###### Use MeshService from remote ######
# Define channel resource
ann_chan = rmt.get_channel(CHAN, COLL, EXP)
# Use resource to generate mesh from the Segmentation ID specified
# and the specified cutout volume ranges
# Save the output in a tuple, raw_data and mesh_data will be returned.
mesh = rmt.mesh(
ann_chan,
res,
x_rng,
y_rng,
z_rng,
id_list=[MESH_ID],
voxel_unit=VoxelUnits.um,
simp_fact=100,
)
from intern.remote.boss import BossRemote
from intern.resource.boss.resource import *
# admin token needed to list all projects
rmt = BossRemote("/home/ben/Documents/travis_user_neurodata.cfg")
with open("public_datasets.csv", "w") as f:
f.write(
"coll,exp,ch,exp_description,num_hierarchy_levels,dtype,x_start,x_stop,y_start,y_stop,z_start,z_stop\n"
)
colls = rmt.list_collections()
colls.remove("ben_dev")
colls.remove("ZBrain")
for coll in colls:
exps = rmt.list_experiments(coll)
for exp in exps:
exp_res = rmt.get_experiment(coll, exp)
coord_frame_res = rmt.get_coordinate_frame(exp_res.coord_frame)
chs = rmt.list_channels(coll, exp)
for ch in chs:
ch_res = rmt.get_channel(ch, coll, exp)
with open("public_datasets.csv", "a") as f:
f.write(
f"{coll},{exp},{ch},{exp_res.description},{exp_res.num_hierarchy_levels},{ch_res.datatype},{coord_frame_res.x_start},{coord_frame_res.x_stop},{coord_frame_res.y_start},{coord_frame_res.y_stop},{coord_frame_res.z_start},{coord_frame_res.z_stop}\n"
)
class ProjectServiceTest_v1(unittest.TestCase):
"""Integration tests of the Boss resource API.
"""
def setUp(self):
self.rmt = BossRemote('test.cfg', API_VER)
# Turn off SSL cert verification. This is necessary for interacting with
# developer instances of the Boss.
self.rmt.project_service.session_send_opts = {'verify': False}
self.rmt.metadata_service.session_send_opts = {'verify': False}
self.rmt.volume_service.session_send_opts = {'verify': False}
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
coll_name = 'collection2309-{}'.format(random.randint(0, 9999))
self.coll = CollectionResource(coll_name, 'bar')
coll_name_upd = '{}-{}'.format(coll_name, random.randint(0, 9999))
self.coll_upd = CollectionResource(coll_name_upd, 'latest')
cf_name = 'ProjTestFrame{}'.format(random.randint(0, 9999))
self.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.',
0, 10, -5, 5, 3, 6, 1, 1, 1,
'nanometers')
self.coord_upd = copy.copy(self.coord)
self.coord_upd.name = 'MouseFrame{}'.format(random.randint(0, 9999))
self.coord_upd.description = 'Mouse coordinate frame.'
self.exp = ExperimentResource('exp2309-2',
self.coll.name,
self.coord.name,
'my experiment',
1,
'isotropic',
time_step=2,
time_step_unit='nanoseconds')
self.exp_upd = ExperimentResource('exp2309-2a', self.coll.name,
self.coord.name,
'my first experiment', 2,
'anisotropic')
self.source_chan = ChannelResource('sourceChan', self.coll.name,
self.exp.name, 'image',
'test source channel', 0, 'uint8',
0)
self.related_chan = ChannelResource('relatedChan', self.coll.name,
self.exp.name, 'image',
'test related channel', 0, 'uint8',
0)
self.chan = ChannelResource('myChan',
self.coll.name,
self.exp.name,
'annotation',
'test annotation channel',
0,
'uint8',
0,
sources=['sourceChan'],
related=['relatedChan'])
self.chan_upd = ChannelResource('yourChan',
self.coll.name,
self.exp.name,
'annotation',
'your test annotation channel',
0,
'uint8',
1,
sources=['sourceChan'],
related=['relatedChan'])
def tearDown(self):
try:
self.rmt.delete_project(self.chan_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.chan)
except HTTPError:
pass
try:
self.rmt.delete_project(self.related_chan)
except HTTPError:
pass
try:
self.rmt.delete_project(self.source_chan)
except HTTPError:
pass
try:
self.rmt.delete_project(self.exp_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.exp)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coord_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coord)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coll_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coll)
except HTTPError:
pass
def test_create_coord_frame(self):
cf = self.rmt.create_project(self.coord)
self.assertEqual(self.coord.name, cf.name)
self.assertEqual(self.coord.description, cf.description)
self.assertEqual(self.coord.x_start, cf.x_start)
self.assertEqual(self.coord.x_stop, cf.x_stop)
self.assertEqual(self.coord.y_start, cf.y_start)
self.assertEqual(self.coord.y_stop, cf.y_stop)
self.assertEqual(self.coord.z_start, cf.z_start)
self.assertEqual(self.coord.z_stop, cf.z_stop)
self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size)
self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size)
self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size)
self.assertEqual(self.coord.voxel_unit, cf.voxel_unit)
def test_create_collection(self):
c = self.rmt.create_project(self.coll)
self.assertEqual(self.coll.name, c.name)
self.assertEqual(self.coll.description, c.description)
def test_create_experiment(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
self.assertEqual(self.exp.name, e.name)
self.assertEqual(self.exp.description, e.description)
self.assertEqual(self.coll.name, e.coll_name)
self.assertEqual(self.exp.coord_frame, e.coord_frame)
self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method)
self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels)
self.assertEqual(self.exp.num_time_samples, e.num_time_samples)
self.assertEqual(self.exp.time_step, e.time_step)
self.assertEqual(self.exp.time_step_unit, e.time_step_unit)
def test_create_channel(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
ch = self.rmt.create_project(self.source_chan)
self.assertEqual(self.source_chan.name, ch.name)
self.assertEqual(self.exp.name, ch.exp_name)
self.assertEqual(self.source_chan.description, ch.description)
self.assertEqual(self.coll.name, ch.coll_name)
self.assertEqual(self.source_chan.datatype, ch.datatype)
self.assertEqual(self.source_chan.default_time_sample,
ch.default_time_sample)
self.assertEqual(self.source_chan.base_resolution, ch.base_resolution)
def test_create_annotation_channel_without_source_fails(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
rel_ch = self.rmt.create_project(self.related_chan)
chan = ChannelResource('myChan',
self.coll.name,
self.exp.name,
'annotation',
'test annotation channel',
0,
'uint8',
0,
related=['relatedChan'])
with self.assertRaises(HTTPError):
self.rmt.create_project(chan)
def test_create_annotation_channel(self):
"""Annotation channels require a source channel."""
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
ch = self.rmt.create_project(self.source_chan)
rel_ch = self.rmt.create_project(self.related_chan)
ann_ch = self.rmt.create_project(self.chan)
self.assertEqual(self.chan.name, ann_ch.name)
self.assertEqual(self.exp.name, ann_ch.exp_name)
self.assertEqual(self.chan.description, ann_ch.description)
self.assertEqual(self.coll.name, ann_ch.coll_name)
self.assertEqual(self.chan.datatype, ann_ch.datatype)
self.assertEqual(self.chan.default_time_sample,
ann_ch.default_time_sample)
self.assertEqual(self.chan.base_resolution, ann_ch.base_resolution)
self.assertEqual(self.chan.sources, ann_ch.sources)
self.assertEqual(self.chan.related, ann_ch.related)
def test_get_collection(self):
coll = self.rmt.create_project(self.coll)
c = self.rmt.get_project(self.coll)
self.assertEqual(self.coll.name, c.name)
self.assertEqual(self.coll.description, c.description)
def test_get_coord_frame(self):
coord = self.rmt.create_project(self.coord)
cf = self.rmt.get_project(self.coord)
self.assertEqual(self.coord.name, cf.name)
self.assertEqual(self.coord.description, cf.description)
self.assertEqual(self.coord.x_start, cf.x_start)
self.assertEqual(self.coord.x_stop, cf.x_stop)
self.assertEqual(self.coord.y_start, cf.y_start)
self.assertEqual(self.coord.y_stop, cf.y_stop)
self.assertEqual(self.coord.z_start, cf.z_start)
self.assertEqual(self.coord.z_stop, cf.z_stop)
self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size)
self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size)
self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size)
self.assertEqual(self.coord.voxel_unit, cf.voxel_unit)
def test_get_experiment(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
exp = self.rmt.create_project(self.exp)
e = self.rmt.get_project(self.exp)
self.assertEqual(self.exp.name, e.name)
self.assertEqual(self.exp.description, e.description)
self.assertEqual(self.coll.name, e.coll_name)
self.assertEqual(self.exp.coord_frame, e.coord_frame)
self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method)
self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels)
self.assertEqual(self.exp.num_time_samples, e.num_time_samples)
self.assertEqual(self.exp.time_step, e.time_step)
self.assertEqual(self.exp.time_step_unit, e.time_step_unit)
def test_get_channel(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
chan = self.rmt.create_project(self.source_chan)
ch = self.rmt.get_project(self.source_chan)
self.assertEqual(self.source_chan.name, ch.name)
self.assertEqual(self.exp.name, ch.exp_name)
self.assertEqual(self.source_chan.description, ch.description)
self.assertEqual(self.coll.name, ch.coll_name)
self.assertEqual(self.source_chan.datatype, ch.datatype)
self.assertEqual(self.source_chan.default_time_sample,
ch.default_time_sample)
self.assertEqual(self.source_chan.base_resolution, ch.base_resolution)
def test_get_channel_helper(self):
"""
Test the helper get_channel() as opposed to getting a channel via get_project().
"""
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
chan = self.rmt.create_project(self.source_chan)
actual = self.rmt.get_channel(chan.name, chan.coll_name, chan.exp_name)
# This is not an exhaustive list of attributes, but they are the
# important ones for correct interaction with the volume service.
self.assertTrue(actual.cutout_ready)
self.assertEqual(chan.datatype, actual.datatype)
self.assertEqual(chan.default_time_sample, actual.default_time_sample)
self.assertEqual(chan.base_resolution, actual.base_resolution)
self.assertEqual(chan.downsample_status, actual.downsample_status)
self.assertEqual(chan.type, actual.type)
self.assertEqual(chan.name, actual.name)
self.assertEqual(chan.coll_name, actual.coll_name)
self.assertEqual(chan.exp_name, actual.exp_name)
def test_update_collection(self):
coll = self.rmt.create_project(self.coll)
c = self.rmt.update_project(self.coll.name, self.coll_upd)
self.assertEqual(self.coll_upd.name, c.name)
self.assertEqual(self.coll_upd.description, c.description)
def test_update_coord_frame(self):
c = self.rmt.create_project(self.coll)
coord = self.rmt.create_project(self.coord)
cf = self.rmt.update_project(self.coord.name, self.coord_upd)
self.assertEqual(self.coord_upd.name, cf.name)
self.assertEqual(self.coord_upd.description, cf.description)
def test_update_experiment(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
eup = self.rmt.update_project(self.exp.name, self.exp_upd)
self.assertEqual(self.exp_upd.name, eup.name)
self.assertEqual(self.exp_upd.description, eup.description)
self.assertEqual(self.coll.name, eup.coll_name)
self.assertEqual(self.exp_upd.coord_frame, eup.coord_frame)
self.assertEqual(self.exp_upd.hierarchy_method, eup.hierarchy_method)
self.assertEqual(self.exp_upd.num_hierarchy_levels,
eup.num_hierarchy_levels)
self.assertEqual(self.exp_upd.num_time_samples, eup.num_time_samples)
self.assertEqual(self.exp.time_step, eup.time_step)
self.assertEqual(self.exp.time_step_unit, eup.time_step_unit)
def test_update_channel(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
source_ch = self.rmt.create_project(self.source_chan)
rel_ch = self.rmt.create_project(self.related_chan)
chan = self.rmt.create_project(self.chan)
ch = self.rmt.update_project(self.chan.name, self.chan_upd)
self.assertEqual(self.chan_upd.name, ch.name)
self.assertEqual(self.exp.name, ch.exp_name)
self.assertEqual(self.chan_upd.description, ch.description)
self.assertEqual(self.coll.name, ch.coll_name)
self.assertEqual(self.chan_upd.datatype, ch.datatype)
self.assertEqual(self.chan_upd.default_time_sample,
ch.default_time_sample)
self.assertEqual(self.chan_upd.base_resolution, ch.base_resolution)
self.assertEqual(self.chan_upd.sources, ch.sources)
self.assertEqual(self.chan_upd.related, ch.related)
def test_list_collections(self):
coll = self.rmt.create_project(self.coll)
coll_list = self.rmt.list_collections()
c = [name for name in coll_list if name == self.coll.name]
self.assertEqual(1, len(c))
self.assertEqual(self.coll.name, c[0])
def test_list_coord_frames(self):
cf = self.rmt.create_project(self.coord)
cf_list = self.rmt.list_coordinate_frames()
c = [name for name in cf_list if name == self.coord.name]
self.assertEqual(1, len(c))
self.assertEqual(self.coord.name, c[0])
def test_list_experiments(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
exp = self.rmt.create_project(self.exp)
exp_list = self.rmt.list_experiments(self.coll.name)
e = [name for name in exp_list if name == self.exp.name]
self.assertEqual(1, len(e))
self.assertEqual(self.exp.name, e[0])
#def test_list_channels(self):
# c = self.rmt.create_project(self.coll)
# cf = self.rmt.create_project(self.coord)
# e = self.rmt.create_project(self.exp)
# chan = self.rmt.create_project(self.chan)
# chan_list = self.rmt.list_channels(self.coll.name, self.exp.name)
# ch = [name for name in chan_list if name == self.chan.name]
# self.assertEqual(1, len(ch))
# self.assertEqual(self.chan.name, ch[0])
def test_delete_all(self):
"""Formally test delete at all levels of the data model.
Delete happens all the time in the tearDown() but specifically test
it here.
"""
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
ch = self.rmt.create_project(self.source_chan)
self.rmt.delete_project(self.source_chan)
self.rmt.delete_project(self.exp)
self.rmt.delete_project(self.coord)
self.rmt.delete_project(self.coll)
class RelayStorageManager(StorageManager):
"""
"""
def __init__(self, **kwargs):
"""
Create a new RelayStorageManager.
Arguments:
block_size: How much data should go in each file
"""
self.block_size = kwargs.get("block_size", (256, 256, 16))
if "next_layer" in kwargs:
self._next = kwargs["next_layer"]
self.is_terminal = False
else:
self.is_terminal = True
if "boss_remote" in kwargs:
self.boss_remote = kwargs["boss_remote"]
elif "upstream_uri" in kwargs:
self.token = kwargs.get("token", "public")
self.boss_remote = BossRemote({
"host":
kwargs["upstream_uri"],
"protocol":
kwargs.get("protocol", "http"),
"token":
kwargs.get("token", "public"),
})
def hasdata(
self,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
):
has_data = self.boss_remote.get_channel(f"bossdb://{col}/{exp}/{chan}")
if has_data:
return True
if not self.is_terminal:
return self._next.hasdata(col, exp, chan, res, xs, ys, zs)
return False
def setdata(
self,
data: np.array,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
):
return self.boss_remote.create_cutout(
self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs,
data)
def getdata(
self,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
) -> np.array:
return self.boss_remote.get_cutout(
self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs)
def __repr__(self):
return f"<RelayStorageManager [BossRemote]>"
def get_stack_names(self):
if self.is_terminal:
return [str(self), f"↳{self.boss_remote}"]
else:
return [
str(self), f"↳{self.boss_remote}",
*self._next.get_stack_names()
]
