class NeuroDataResource:
def __init__(self, host, token, collection, experiment, chanList):
self._collection = collection
self._experiment = experiment
self._bossRemote = BossRemote({'protocol':'https',
'host':host,
'token':token})
self._chanList = {}
for chanDict in chanList:
try:
self._chanList[chanDict['name']] = ChannelResource(chanDict['name'],
collection,
experiment,
'image',
datatype=chanDict['dtype'])
except:
#TODO error handle here
raise
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if not chan in self._chanList.keys():
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print('Error: You must supply zRange, yRange, xRange kwargs in list format')
data = self._bossRemote.get_cutout(self._chanList[chan],
0,
xRange,
yRange,
zRange)
return data
class NeuroDataResource:
def __init__(self, host, token, collection, experiment, chanList):
self._collection = collection
self._experiment = experiment
self._bossRemote = BossRemote({'protocol':'https',
'host':host,
'token':token})
self._chanList = {}
for chanDict in chanList:
try:
self._chanList[chanDict['name']] = ChannelResource(chanDict['name'],
collection,
experiment,
'image',
datatype=chanDict['dtype'])
except:
#TODO error handle here
raise
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if not chan in self._chanList.keys():
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print('Error: You must supply zRange, yRange, xRange kwargs in list format')
data = self._bossRemote.get_cutout(self._chanList[chan],
0,
xRange,
yRange,
zRange)
return data
def save_img(self, fname, cutout, format = 'tif'):
'''
Saves a image as a tif or nii file
Leave the file type suffix out of fname
'''
if (not fname) or (type(fname) != str):
print('Error: Give valid filename')
if (not cutout) or type(cutout) != np.array:
print('Error: Give valid image cutout')
if format == 'tif':
max_grey_val = cutout.max(axis=1).max(axis=1).max()
dim = cutout.shape()
cutout_rgb = np.zeros((dim[0], dim[1], dim[2], 3))
for i in range(dim[0]):
for j in range(dim[1]):
for k in range(dim[2]):
normalized_rgb = int( 255 * (cutout[i][j][k]/max_grey_val) )
for p in range(3):
cutout_rgb[i][j][k][p] = normalized_rgb
imsave(fname+'.tif', cutout_rgb.astype(np.uint8))
else:
nifti_img = nib.Nifti1Image(cutout, np.eye(4))
nib.save(nifti_img, fname+'.nii')
def test_can_retrieve_correct_u64_data(self):
data = array("bossdb://kasthuri2015/em/3cylneuron_v1")
boss = BossRemote(_DEFAULT_BOSS_OPTIONS)
get_cutout_data = boss.get_cutout(
boss.get_channel("3cylneuron_v1", "kasthuri2015", "em"),
0,
[6000, 6200],
[12000, 12500],
[923, 924],
)
self.assertTrue(
(get_cutout_data == data[923:924, 12000:12500, 6000:6200]).all()
)
def get_boss_data(args):
args.x_rng = [args.xmin, args.xmax]
args.y_rng = [args.ymin, args.ymax]
args.z_rng = [args.zmin, args.zmax]
config = {"protocol": "https",
"host": args.host,
"token": args.token}
rmt = BossRemote(config)
segchan = ChannelResource(args.chan_seg,
args.coll,
args.exp,
'annotation',
datatype=args.dtype_seg)
# Get the image data from the BOSS
seg_data = rmt.get_cutout(segchan, args.res,
args.x_rng,
args.y_rng,
args.z_rng)
synchan = ChannelResource(args.chan_syn,
args.coll,
args.exp,
'annotation',
datatype=args.dtype_syn)
syn_data = rmt.get_cutout(synchan, args.res,
args.x_rng,
args.y_rng,
args.z_rng)
#Wany XYZ
seg_data = np.transpose(seg_data,(2,1,0))
syn_data = np.transpose(syn_data,(2,1,0))
return seg_data, syn_data
def get_boss_data(args):
config = {
"protocol": "https",
"host": "api.bossdb.io",
"token": args.token
}
rmt = BossRemote(config)
print('[info] Downloading data from BOSS')
chan = ChannelResource(args.chan_img,
args.coll,
args.exp,
'image',
datatype=args.dtype_img)
# Get the image data from the BOSS
x_train = rmt.get_cutout(chan, args.res, [args.xmin, args.xmax],
[args.ymin, args.ymax], [args.zmin, args.zmax])
lchan = ChannelResource(args.chan_labels,
args.coll,
args.exp,
'annotation',
datatype=args.dtype_lbl)
y_train = rmt.get_cutout(lchan, args.res, [args.xmin, args.xmax],
[args.ymin, args.ymax], [args.zmin, args.zmax])
print('[info] Downloaded BOSS data')
# Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X])
x_train = x_train[:, np.newaxis, :, :].astype(np.float32)
y_train = y_train[:, np.newaxis, :, :].astype(np.float32)
# Pixel values must be in [0,1]
x_train /= 255.
y_train = (y_train > 0).astype('float32')
return x_train, y_train
def get_boss_data(args):
config = {
"protocol": "https",
"host": "api.theBoss.io",
"token": args.token
}
rmt = BossRemote(config)
chan = ChannelResource(args.img_channel,
args.collection,
args.experiment,
'image',
datatype='uint8')
# Get the image data from the BOSS
x_train = rmt.get_cutout(chan, args.resolution, args.x_rng, args.y_rng,
args.z_rng)
lchan = ChannelResource(args.lbl_channel,
args.collection,
args.experiment,
'annotation',
datatype='uint64')
y_train = rmt.get_cutout(lchan, args.resolution, args.x_rng, args.y_rng,
args.z_rng)
# Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X])
x_train = x_train[:, np.newaxis, :, :].astype(np.float32)
y_train = y_train[:, np.newaxis, :, :].astype(np.float32)
# Pixel values must be in [0,1]
x_train /= 255.
y_train = (y_train > 0).astype('float32')
return x_train, y_train
def download(self, bbox, mip, parallel=1, renumber=False):
if parallel != 1:
raise ValueError("Only parallel=1 is supported for boss.")
elif renumber != False:
raise ValueError("Only renumber=False is supported for boss.")
bounds = Bbox.clamp(bbox, self.meta.bounds(mip))
if self.autocrop:
image, bounds = autocropfn(self.meta, image, bounds, mip)
if bounds.subvoxel():
raise exceptions.EmptyRequestException(
'Requested less than one pixel of volume. {}'.format(bounds))
x_rng = [bounds.minpt.x, bounds.maxpt.x]
y_rng = [bounds.minpt.y, bounds.maxpt.y]
z_rng = [bounds.minpt.z, bounds.maxpt.z]
layer_type = 'image' if self.meta.layer_type == 'unknown' else self.meta.layer_type
chan = ChannelResource(
collection_name=self.meta.path.bucket,
experiment_name=self.meta.path.dataset,
name=self.meta.path.layer, # Channel
type=layer_type,
datatype=self.meta.data_type,
)
rmt = BossRemote(boss_credentials)
cutout = rmt.get_cutout(chan, mip, x_rng, y_rng, z_rng, no_cache=True)
cutout = cutout.T
cutout = cutout.astype(self.meta.dtype)
cutout = cutout[::steps.x, ::steps.y, ::steps.z]
if len(cutout.shape) == 3:
cutout = cutout.reshape(tuple(list(cutout.shape) + [1]))
if self.bounded or self.autocrop or bounds == bbox:
return VolumeCutout.from_volume(self.meta, mip, cutout, bounds)
# This section below covers the case where the requested volume is bigger
# than the dataset volume and the bounds guards have been switched
# off. This is useful for Marching Cubes where a 1px excess boundary
# is needed.
shape = list(bbox.size3()) + [cutout.shape[3]]
renderbuffer = np.zeros(shape=shape, dtype=self.meta.dtype, order='F')
shade(renderbuffer, bbox, cutout, bounds)
return VolumeCutout.from_volume(self.meta, mip, renderbuffer, bbox)
def _boss_cutout(self, requested_bbox, steps, channel_slice=slice(None)):
bounds = Bbox.clamp(requested_bbox, self.bounds)
if bounds.subvoxel():
raise exceptions.EmptyRequestException(
'Requested less than one pixel of volume. {}'.format(bounds))
x_rng = [bounds.minpt.x, bounds.maxpt.x]
y_rng = [bounds.minpt.y, bounds.maxpt.y]
z_rng = [bounds.minpt.z, bounds.maxpt.z]
layer_type = 'image' if self.layer_type == 'unknown' else self.layer_type
chan = ChannelResource(
collection_name=self.path.bucket,
experiment_name=self.path.dataset,
name=self.path.layer, # Channel
type=layer_type,
datatype=self.dtype,
)
rmt = BossRemote(boss_credentials)
cutout = rmt.get_cutout(chan,
self.mip,
x_rng,
y_rng,
z_rng,
no_cache=True)
cutout = cutout.T
cutout = cutout.astype(self.dtype)
cutout = cutout[::steps.x, ::steps.y, ::steps.z]
if len(cutout.shape) == 3:
cutout = cutout.reshape(tuple(list(cutout.shape) + [1]))
if self.bounded or self.autocrop or bounds == requested_bbox:
return VolumeCutout.from_volume(self, cutout, bounds)
# This section below covers the case where the requested volume is bigger
# than the dataset volume and the bounds guards have been switched
# off. This is useful for Marching Cubes where a 1px excess boundary
# is needed.
shape = list(requested_bbox.size3()) + [cutout.shape[3]]
renderbuffer = np.zeros(shape=shape, dtype=self.dtype, order='F')
txrx.shade(renderbuffer, requested_bbox, cutout, bounds)
return VolumeCutout.from_volume(self, renderbuffer, requested_bbox)
class VolumeServiceTest_v1(unittest.TestCase):
"""Integration tests of the Boss volume service API.
Because setup and teardown involves many REST calls, tests are only
divided into tests of the different types of data model resources. All
operations are performed within a single test of each resource.
"""
@classmethod
def setUpClass(cls):
"""Do an initial DB clean up in case something went wrong the last time.
If a test failed really badly, the DB might be in a bad state despite
attempts to clean up during tearDown().
"""
cls.rmt = BossRemote('test.cfg', API_VER)
# Turn off SSL cert verification. This is necessary for interacting with
# developer instances of the Boss.
cls.rmt.project_service.session_send_opts = {'verify': False}
cls.rmt.metadata_service.session_send_opts = {'verify': False}
cls.rmt.volume_service.session_send_opts = {'verify': False}
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
coll_name = 'collection2323{}'.format(random.randint(0, 9999))
cls.coll = CollectionResource(coll_name, 'bar')
cf_name = 'BestFrame{}'.format(random.randint(0, 9999))
cls.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.',
0, 2048, 0, 2048, 0, 100, 1, 1, 1,
'nanometers', 0, 'nanoseconds')
# cls.exp.coord_frame must be set with valid id before creating.
cls.exp = ExperimentResource('exp2323x2', cls.coll.name,
cls.coord.name, 'my experiment', 1,
'isotropic', 10)
cls.chan = ChannelResource('myVolChan', cls.coll.name, cls.exp.name,
'image', 'test channel', 0, 'uint8', 0)
cls.chan16 = ChannelResource('myVol16bitChan', cls.coll.name,
cls.exp.name, 'image',
'16 bit test channel', 0, 'uint16', 0)
cls.ann_chan = ChannelResource('annVolChan2',
cls.coll.name,
cls.exp.name,
'annotation',
'annotation test channel',
0,
'uint64',
0,
sources=[cls.chan.name])
# This channel reserved for testing get_ids_in_region(). This is a
# separate channel so we don't have to worry about ids written by
# other tests.
cls.ann_region_chan = ChannelResource(
'annRegionChan2',
cls.coll.name,
cls.exp.name,
'annotation',
'annotation ids in region test channel',
0,
'uint64',
0,
sources=[cls.chan.name])
# This channel reerved for testing tight bounding boxes.
cls.ann_bounding_chan = ChannelResource(
'annRegionChan3',
cls.coll.name,
cls.exp.name,
'annotation',
'annotation ids in bounding box test channel',
0,
'uint64',
0,
sources=[cls.chan.name])
cls.rmt.create_project(cls.coll)
cls.rmt.create_project(cls.coord)
cls.rmt.create_project(cls.exp)
cls.rmt.create_project(cls.chan16)
cls.rmt.create_project(cls.chan)
cls.rmt.create_project(cls.ann_chan)
cls.rmt.create_project(cls.ann_region_chan)
cls.rmt.create_project(cls.ann_bounding_chan)
@classmethod
def tearDownClass(cls):
"""Clean up the data model objects used by this test case.
This method is used by both tearDownClass() and setUpClass().
"""
try:
cls.rmt.delete_project(cls.ann_bounding_chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.ann_region_chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.ann_chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.chan16)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.exp)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.coord)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.coll)
except HTTPError:
pass
def setUp(self):
self.rmt = BossRemote('test.cfg')
def tearDown(self):
pass
def test_reserve_ids(self):
first_id = self.rmt.reserve_ids(self.ann_chan, 20)
self.assertTrue(first_id > 0)
def test_get_bounding_box_id_doesnt_exist(self):
resolution = 0
id = 12345678
with self.assertRaises(HTTPError) as err:
self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose')
expected_msg_prefix = 'Reserve ids failed'
self.assertTrue(err.message.startwswith(expected_msg_prefix))
@unittest.skip('Skipping - currently indexing disabled')
def test_get_bounding_box_spans_cuboids_in_x(self):
x_rng = [511, 515]
y_rng = [0, 8]
z_rng = [0, 5]
t_rng = [0, 1]
id = 77555
data = numpy.zeros((5, 8, 4), dtype='uint64')
data[1][0][0] = id
data[2][1][1] = id
data[3][2][3] = id
resolution = 0
self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng,
data)
# Get cutout to make sure data is done writing and indices updated.
actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng,
z_rng)
numpy.testing.assert_array_equal(data, actual)
expected = {
'x_range': [0, 1024],
'y_range': [0, 512],
'z_range': [0, 16],
't_range': [0, 1]
}
actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id,
'loose')
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_bounding_box_spans_cuboids_in_y(self):
x_rng = [0, 8]
y_rng = [511, 515]
z_rng = [0, 5]
t_rng = [0, 1]
id = 77666
data = numpy.zeros((5, 4, 8), dtype='uint64')
data[1][0][0] = id
data[2][1][0] = id
data[3][2][0] = id
resolution = 0
self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng,
data)
# Get cutout to make sure data is done writing and indices updated.
actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng,
z_rng)
numpy.testing.assert_array_equal(data, actual)
expected = {
'x_range': [0, 512],
'y_range': [0, 1024],
'z_range': [0, 16],
't_range': [0, 1]
}
actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id,
'loose')
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_bounding_box_spans_cuboids_in_z(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [30, 35]
t_rng = [0, 1]
id = 77888
data = numpy.zeros((5, 4, 8), dtype='uint64')
data[1][0][0] = id
data[2][1][0] = id
data[3][2][0] = id
resolution = 0
self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng,
data)
# Get cutout to make sure data is done writing and indices updated.
actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng,
z_rng)
numpy.testing.assert_array_equal(data, actual)
expected = {
'x_range': [0, 512],
'y_range': [0, 512],
'z_range': [16, 48],
't_range': [0, 1]
}
actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id,
'loose')
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_tight_bounding_box_x_axis(self):
"""Test tight bounding box with ids that span three cuboids along the x axis."""
resolution = 0
x_rng = [511, 1025]
y_rng = [512, 1024]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 512, 514), dtype='uint64')
x_id = 123
y_id = 127
z_id = 500000000000000000
# Id in partial region on x axis closest to origin.
data[1][1][0] = x_id
# Id in partial region on x axis furthest from origin.
data[1][1][513] = x_id
# Id in cuboid aligned region.
data[2][2][21] = x_id
data[2][1][22] = y_id
data[4][24][72] = z_id
expected = {
'x_range': [511, 1025],
'y_range': [513, 515],
'z_range': [17, 19]
}
self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng,
y_rng, z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_bounding_box(self.ann_bounding_chan,
resolution,
x_id,
bb_type='tight')
@unittest.skip('Skipping - currently indexing disabled')
def test_tight_bounding_box_y_axis(self):
"""Test tight bounding box with ids that span three cuboids along the x axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [511, 1025]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 514, 512), dtype='uint64')
x_id = 123
y_id = 127
z_id = 500000000000000000
# Id in partial region on y axis closest to origin.
data[1][0][10] = y_id
# Id in partial region on y axis furthest from origin.
data[1][513][13] = y_id
# Id in cuboid aligned region.
data[2][2][21] = y_id
data[2][3][20] = x_id
data[4][25][71] = z_id
expected = {
'x_range': [522, 526],
'y_range': [511, 1025],
'z_range': [17, 19]
}
self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng,
y_rng, z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_bounding_box(self.ann_bounding_chan,
resolution,
y_id,
bb_type='tight')
@unittest.skip('Skipping - currently indexing disabled')
def test_tight_bounding_box_z_axis(self):
"""Test tight bounding box with ids that span three cuboids along the x axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [512, 1024]
z_rng = [15, 33]
t_rng = [0, 1]
data = numpy.zeros((18, 512, 512), dtype='uint64')
x_id = 123
y_id = 127
z_id = 500000000000000000
# Id in partial region on z axis closest to origin.
data[0][22][60] = z_id
# Id in partial region on z axis furthest from origin.
data[17][23][63] = z_id
# Id in cuboid aligned region.
data[5][24][71] = z_id
data[3][2][20] = x_id
data[3][1][21] = y_id
expected = {
'x_range': [572, 583],
'y_range': [534, 537],
'z_range': [15, 33]
}
self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng,
y_rng, z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_bounding_box(self.ann_bounding_chan,
resolution,
z_id,
bb_type='tight')
def test_get_ids_in_region_none(self):
"""Run on region that hasn't been written with ids, yet."""
resolution = 0
x_rng = [1536, 1540]
y_rng = [1536, 1540]
z_rng = [48, 56]
t_rng = [0, 1]
data = numpy.zeros((8, 4, 4), dtype='uint64')
expected = []
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
def test_filtered_cutout(self):
"""Test filtered cutout using same data written for get_ids_in_region_x_axis."""
resolution = 0
x_rng = [511, 1025]
y_rng = [512, 1024]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 512, 514), dtype='uint64')
# Id in partial region on x axis closest to origin.
data[1][1][0] = 123
# Id in partial region on x axis furthest from origin.
data[1][1][513] = 321
# Id in cuboid aligned region.
data[10][20][21] = 55555
expected = [123, 321, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Should get back the exact data given in create_cutout().
filtered_data1 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[123, 321, 55555])
numpy.testing.assert_array_equal(data, filtered_data1)
# Filter on id 123.
expected_data_123 = numpy.zeros((16, 512, 514), dtype='uint64')
expected_data_123[1][1][0] = 123
filtered_data_123 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[123])
numpy.testing.assert_array_equal(expected_data_123, filtered_data_123)
# Filter on id 321.
expected_data_321 = numpy.zeros((16, 512, 514), dtype='uint64')
expected_data_321[1][1][513] = 321
filtered_data_321 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[321])
numpy.testing.assert_array_equal(expected_data_321, filtered_data_321)
# Filter on ids 123 and 55555.
expected_data_123_55555 = numpy.zeros((16, 512, 514), dtype='uint64')
expected_data_123_55555[1][1][0] = 123
expected_data_123_55555[10][20][21] = 55555
filtered_data_123_55555 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[123, 55555])
numpy.testing.assert_array_equal(expected_data_123_55555,
filtered_data_123_55555)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_ids_in_region_x_axis(self):
"""Test using a region that's cuboid aligned except for the x axis."""
resolution = 0
x_rng = [511, 1025]
y_rng = [512, 1024]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 512, 514), dtype='uint64')
# Id in partial region on x axis closest to origin.
data[1][1][0] = 123
# Id in partial region on x axis furthest from origin.
data[1][1][513] = 321
# Id in cuboid aligned region.
data[10][20][21] = 55555
expected = [123, 321, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_ids_in_region_y_axis(self):
"""Test using a region that's cuboid aligned except for the y axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [511, 1025]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 514, 512), dtype='uint64')
# Id in partial region on y axis closest to origin.
data[1][0][1] = 456
# Id in partial region on y axis furthest from origin.
data[1][513][1] = 654
# Id in cuboid aligned region.
data[10][21][20] = 55555
# expected = [123, 321, 456, 654, 789, 987, 55555]
expected = [456, 654, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_ids_in_region_z_axis(self):
"""Test using a region that's cuboid aligned except for the z axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [512, 1024]
z_rng = [15, 33]
t_rng = [0, 1]
data = numpy.zeros((18, 512, 512), dtype='uint64')
# Id in partial region on z axis closest to origin.
data[0][1][1] = 789
# Id in partial region on z axis furthest from origin.
data[17][1][1] = 987
# Id in cuboid aligned region.
data[11][20][20] = 55555
expected = [789, 987, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
def test_upload_and_download_to_channel(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
data = numpy.random.randint(1, 254, (5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_to_channel_with_time(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
t_rng = [3, 6]
data = numpy.random.randint(1, 254, (3, 5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
data,
time_range=t_rng)
actual = self.rmt.get_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_subsection_to_channel(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 19]
sub_x = [12, 14]
sub_y = [7, 10]
sub_z = [12, 17]
data = numpy.random.randint(1, 10, (9, 5, 10))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan, 0, sub_x, sub_y, sub_z)
numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual)
def test_upload_to_x_edge_of_channel(self):
x_rng = [10, 2048]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_y_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 2048]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_z_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 100]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_x_edge_of_channel(self):
x_rng = [10, 2049]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_y_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 2049]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_z_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 101]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_and_download_to_channel_16bit(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan16, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_subsection_to_channel_16bit(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 19]
sub_x = [12, 14]
sub_y = [7, 10]
sub_z = [12, 17]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan16, 0, sub_x, sub_y, sub_z)
numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual)
def test_upload_to_x_edge_of_channel_16bit(self):
x_rng = [2000, 2048]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_y_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [2000, 2048]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_z_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 100]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_x_edge_of_channel_16bit(self):
x_rng = [2000, 2049]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_y_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [2000, 2049]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_z_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 101]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_and_download_to_anno_chan(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_subsection_to_anno_chan(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 19]
sub_x = [12, 14]
sub_y = [7, 10]
sub_z = [12, 17]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.ann_chan, 0, sub_x, sub_y, sub_z)
numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual)
def test_upload_to_x_edge_of_anno_chan(self):
x_rng = [2000, 2048]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_y_edge_of_anno_chan(self):
x_rng = [10, 20]
y_rng = [2000, 2048]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_z_edge_of_anno_chan(self):
x_rng = [10, 100]
y_rng = [5, 10]
z_rng = [10, 100]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_x_edge_of_anno_chan(self):
x_rng = [10, 2049]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_y_edge_of_anno_chan(self):
x_rng = [10, 991]
y_rng = [5, 2049]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_z_edge_of_anno_chan(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 101]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_and_download_to_channel_4D(self):
x_rng = [600, 680]
y_rng = [600, 640]
z_rng = [50, 55]
t_rng = [0, 1]
shape = (t_rng[1] - t_rng[0], z_rng[1] - z_rng[0], y_rng[1] - y_rng[0],
x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
data,
time_range=t_rng)
actual = self.rmt.get_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_cutout_to_black(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
data = numpy.random.randint(1, 254, (5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
self.rmt.create_cutout_to_black(self.chan, 0, x_rng, y_rng, z_rng)
actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(numpy.zeros((5, 4, 8)), actual)
def test_upload_and_cutout_to_black_with_time(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
t_rng = [3, 6]
data = numpy.random.randint(1, 254, (3, 5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
data,
time_range=t_rng)
self.rmt.create_cutout_to_black(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
actual = self.rmt.get_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
numpy.testing.assert_array_equal(numpy.zeros((3, 5, 4, 8)), actual)
def test_upload_and_cutout_to_black_partial(self):
x_rng = [0, 1024]
y_rng = [0, 1024]
z_rng = [0, 5]
x_rng_black = [0, 256]
y_rng_black = [0, 512]
z_rng_black = [2, 3]
data = numpy.random.randint(1, 254, (5, 1024, 1024))
data = data.astype(numpy.uint8)
expected = numpy.copy(data)
expected[2:3, 0:512, 0:256] = 0
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
self.rmt.create_cutout_to_black(self.chan, 0, x_rng_black, y_rng_black,
z_rng_black)
actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(expected, actual)
class bossHandler:
"""
This is a class for interacting with BOSS. It basically is a wrapper for Intern.
"""
def __init__(self, collection_name):
"""
Constructor:
Takes the following argument:
:param collection_name: name of the collection in BOSS
:type collection_name: string
"""
self.collection_name = collection_name
try:
self.rmt = BossRemote()
except:
print('Unexpected Error:', sys.exc_info()[0])
# def get_collection_list(self):
# return self.rmt.list_collections()
def list_experiments(self):
"""
:return: all the experiments available in current collection
"""
exp_list = self.rmt.list_experiments(self.collection_name)
return exp_list
def select_experiment(self, experiment_name):
"""
Select an experiment to be added to this handler
"""
tmp = ExperimentResource(collection_name=self.collection_name,
name=experiment_name)
exp = self.rmt.get_project(tmp)
self.experiment = exp
# return exp
def get_experiment(self):
"""
:return: the currently selected experiment for this handler
"""
if hasattr(self, 'experiment'):
return self.experiment
else:
raise AttributeError(
'No experiment exists. First, select an experiment using select_experiment'
)
def list_channels(self):
"""
:return: all channel in currently selected experiment
"""
return self.rmt.list_channels(self.collection_name,
self.experiment.name)
def select_channel(self, channel_name):
"""
Select a channel to be added to this handler
"""
self.channel = self.rmt.get_channel(chan_name=channel_name,
coll_name=self.collection_name,
exp_name=self.experiment.name)
def get_coordinate_frame(self):
"""
Return: current experiment's coordinate frame
"""
tmp = CoordinateFrameResource(name=self.experiment.coord_frame)
coor = self.rmt.get_project(tmp)
self.coordinate_frame = coor
return coor
def get_all(self):
"""
:return: the entire channel image data at its native resolution
"""
x_rng = [self.coordinate_frame.x_start, self.coordinate_frame.x_stop]
y_rng = [self.coordinate_frame.y_start, self.coordinate_frame.y_stop]
z_rng = [self.coordinate_frame.z_start, self.coordinate_frame.z_stop]
return self.rmt.get_cutout(self.channel, 0, x_rng, y_rng, z_rng)
def get_cutout(self, x_range, y_range, z_range, resolution=0):
"""
:return: a cutout of the image data
"""
return self.rmt.get_cutout(self.channel, resolution, x_range, y_range,
z_range)
from intern.remote.localFile import LocalRemote
from intern.remote.boss import BossRemote
from intern.resource.boss.resource import *
import matplotlib.pyplot as plt
import numpy as np
# BOSS Data fetch which we will upload to the Local Storage:
boss = BossRemote({
"protocol": "https",
"host": "api.theboss.io",
"token": "db1cec2c865fc84e48772f4f4a5f010c0a180b88",
})
volumeB = boss.get_cutout(
boss.get_channel("em", "pinky40", "v7"),
1,
[10000, 10200],
[10000, 10090],
[500, 520],
)
#Local Upload
local = LocalRemote({
"host": "/Users/rodrilm2/InternRel/",
"datastore": "LocalBossDummy"
})
print(local)
chan_setup = local.get_channel('em', 'pinky40')
proj = local.create_project(chan_setup)
volume = local.create_cutout(proj, 'v1', volumeB)
#LocalMetadata updates
Channel1 = local.retrieve('em')
class NeuroDataResource:
def __init__(self,
host,
token,
collection,
experiment,
requested_channels,
x_range=None,
y_range=None,
z_range=None,
resolution=0):
self._bossRemote = BossRemote({
'protocol': 'https',
'host': host,
'token': token
})
self.collection = collection
self.experiment = experiment
self.resolution = resolution
self.channels = self._bossRemote.list_channels(collection, experiment)
if len(requested_channels) == 0:
self.requested_channels = self.channels
else:
self.requested_channels = requested_channels
self._get_coord_frame_details()
# validate range
#if not self.correct_range(z_range, y_range, x_range):
# raise Exception("Error: Inccorect dimension range")
self.x_range = x_range or [0, self.max_dimensions[2]]
self.y_range = y_range or [0, self.max_dimensions[1]]
self.z_range = z_range or [0, self.max_dimensions[0]]
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size,
data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print(
'Error: You must supply zRange, yRange, xRange kwargs in list format'
)
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource, self.resolution,
xRange, yRange, zRange)
#Datatype check. Recast to original datatype if data is float64
if data.dtype == datatype:
return data
else:
return data.astype(datatype)
def correct_range(self, z_range, y_range, x_range):
x_start, x_end = x_range
if x_start < 0 or x_end > self.max_dimensions[2]:
return False
y_start, y_end = y_range
if y_start < 0 or y_end > self.max_dimensions[1]:
return False
z_start, z_end = z_range
if z_start < 0 or z_end > self.max_dimensions[0]:
return False
return True
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME, COLL_NAME, EXP_NAME, 'image', '', datatype='uint16')
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
x_rng = [0, xmax]
y_rng = [0, ymax]
z_rng = [0, zmax]
t_rng = [0, tmax]
print('Data model setup.')
data = np.random.randint(1, 3000, (tmax, zmax, ymax, xmax))
data = data.astype(np.uint16)
# Upload the cutout to the channel.
rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data,
time_range=t_rng)
cutout_data = rmt.get_cutout(
chan_actual, 0, x_rng, y_rng, z_rng, time_range=t_rng)
np.testing.assert_array_equal(data, cutout_data)
print(np.shape(cutout_data))
# (10, 5, 4, 8)
#You are ready!
#If you would like to get multiple images you can make a while loop or for loop, let me know
#if you want me to do it :)
boss = BossRemote({
"protocol": "https",
"host": "api.theboss.io",
#Remember to change your token here. You can get your own at: https://api.theboss.io/token/
"token": "Token"
})
#Here you will specify form where the data is coming from, the resolution, and the size of your image.
volume = boss.get_cutout(
boss.get_channel("cc", "kasthuri2015", "em"),
0,
[5000, 6000],
[8000, 9000],
[1100, 1200],
)
volume2 = boss.get_cutout(
boss.get_channel("mitochondria", "kasthuri2015", "em"),
0,
[5000, 6000],
[8000, 9000],
[1100, 1200],
)
volume3 = boss.get_cutout(
boss.get_channel("3cylsynapse_v1", "kasthuri2015", "em"),
0,
from intern.remote.boss import BossRemote
from intern.resource.boss.resource import ChannelResource
import matplotlib.pyplot as plt
import numpy as np
boss = BossRemote({
"protocol": "https",
"host": "api.theboss.io",
"token": "db1cec2c865fc84e48772f4f4a5f010c0a180b88"
})
#Here you will specify form where the data is coming from, the resolution, and the size of your image.
volume = boss.get_cutout(
boss.get_channel("em", "pinky40", "v7"),
0,
[10000, 10500],
[10000, 10500],
[500, 550],
)
print(volume)
plt.imshow(volume[1, :, :], cmap="gray")
plt.show()
class RelayStorageManager(StorageManager):
"""
"""
def __init__(self, **kwargs):
"""
Create a new RelayStorageManager.
Arguments:
block_size: How much data should go in each file
"""
self.block_size = kwargs.get("block_size", (256, 256, 16))
if "next_layer" in kwargs:
self._next = kwargs["next_layer"]
self.is_terminal = False
else:
self.is_terminal = True
if "boss_remote" in kwargs:
self.boss_remote = kwargs["boss_remote"]
elif "upstream_uri" in kwargs:
self.token = kwargs.get("token", "public")
self.boss_remote = BossRemote({
"host":
kwargs["upstream_uri"],
"protocol":
kwargs.get("protocol", "http"),
"token":
kwargs.get("token", "public"),
})
def hasdata(
self,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
):
has_data = self.boss_remote.get_channel(f"bossdb://{col}/{exp}/{chan}")
if has_data:
return True
if not self.is_terminal:
return self._next.hasdata(col, exp, chan, res, xs, ys, zs)
return False
def setdata(
self,
data: np.array,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
):
return self.boss_remote.create_cutout(
self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs,
data)
def getdata(
self,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
) -> np.array:
return self.boss_remote.get_cutout(
self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs)
def __repr__(self):
return f"<RelayStorageManager [BossRemote]>"
def get_stack_names(self):
if self.is_terminal:
return [str(self), f"↳{self.boss_remote}"]
else:
return [
str(self), f"↳{self.boss_remote}",
*self._next.get_stack_names()
]
# Note that the numpy matrix is in Z, Y, X order.
ann_data = numpy.random.randint(start_id,
start_id + 10,
np_size,
dtype='uint64')
# Make sure start_id is used at least once.
ann_data[0][1][1] = start_id
# 0 is native resolution.
res = 0
# Upload annotation data to the channel.
rmt.create_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng, ann_data)
# Verify that the cutout uploaded correctly.
ann_cutout_data = rmt.get_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(ann_data, ann_cutout_data)
print('Annotation data uploaded and verified.')
# Get ids in the region.
ids = rmt.get_ids_in_region(ann_chan_actual, res, x_rng, y_rng, z_rng)
print('Ids in region are:')
print(ids)
# Get the loose bounding box for id start_id. This should be the bounds of the
# first cuboid (512, 512, 16).
loose = rmt.get_bounding_box(ann_chan_actual, res, start_id, bb_type='loose')
args = parse_args()
config = {
"protocol": "https",
"host": "api.theBoss.io",
"token": args.token
}
rmt = BossRemote(config)
chan = ChannelResource(args.in_channel,
args.collection,
args.experiment,
'image',
datatype='uint8')
# Get the image data from the BOSS
x_test = rmt.get_cutout(chan, args.resolution, args.x_rng, args.y_rng,
args.z_rng)
# Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X])
x_test = x_test[:, np.newaxis, :, :].astype(np.float32)
# Pixel values must be in [0,1]
if x_test.max() > 1.0:
x_test /= 255.
tile_size = (512, 512)
z_step = args.z_step
# ----------------------------------------------------------------------
# load model
model = create_unet((1, tile_size[0], tile_size[1]))
if args.do_synapse:
model.load_weights('/src/weights/synapse_weights.hdf5')
chan = ChannelResource(CHAN_NAME,
COLL_NAME,
EXP_NAME,
"image",
datatype="uint8")
x_rng = [0, 250]
y_rng = [0, 250]
z_rng = [0, 250]
# Use base resolution.
res = 0
# Download the cutout from the channel.
data = boss_rmt.get_cutout(chan, res, x_rng, y_rng, z_rng)
# cloud-volume likes data in X Y Z format
data = np.transpose(data)
# Set up cloud-volume remote
cv_config = {
"protocol": "gcp",
"cloudpath": "neuroproof_examples/training_sample/images",
}
cv_rmt = CloudVolumeRemote(cv_config)
# create a new info (similar to coordinate frame)
info = cv_rmt.create_new_info(
num_channels=1,
layer_type="image",
class TestRemoteGetCutout(unittest.TestCase):
def setUp(self):
config = {"protocol": "https",
"host": "test.theboss.io",
"token": "my_secret"}
self.remote = BossRemote(config)
@patch.object(VolumeService, 'get_cutout', autospec=True)
def test_get_cutout_access_mode_defaults_no_cache(self, fake_volume):
"""
Test that if no_cache not provided, it defaults to True when calling
the volume service's get_cutout().
"""
chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16')
resolution = 0
x_range = [20, 40]
y_range = [50, 70]
z_range = [30, 50]
self.remote.get_cutout(chan, resolution, x_range, y_range, z_range)
fake_volume.assert_called_with(
ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY,
CacheMode.no_cache, parallel=True) # This should be the no_cache argument.
@patch.object(VolumeService, 'get_cutout', autospec=True)
def test_get_cutout_access_mode_raw(self, fake_volume):
"""
Test that if no_cache not provided, it defaults to True when calling
the volume service's get_cutout().
"""
chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16')
resolution = 0
x_range = [20, 40]
y_range = [50, 70]
z_range = [30, 50]
self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, access_mode=CacheMode.raw)
fake_volume.assert_called_with(
ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY,
CacheMode.raw, parallel=True) # This should be the no_cache argument.
@patch.object(VolumeService, 'get_cutout', autospec=True)
def test_get_cutout_access_mode_cache(self, fake_volume):
"""
Test that if no_cache not provided, it defaults to True when calling
the volume service's get_cutout().
"""
chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16')
resolution = 0
x_range = [20, 40]
y_range = [50, 70]
z_range = [30, 50]
self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, access_mode=CacheMode.cache)
fake_volume.assert_called_with(
ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY,
CacheMode.cache, parallel=True) # This should be the no_cache argument.
##REMOVE IN THE FUTURE, TESTS BACKWARDS COMPATABILITY
@patch.object(VolumeService, 'get_cutout', autospec=True)
def test_get_cutout_no_cache_True_backwards_compatability(self, fake_volume):
"""
Test that if no_cache not provided, it defaults to True when calling
the volume service's get_cutout().
"""
chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16')
resolution = 0
x_range = [20, 40]
y_range = [50, 70]
z_range = [30, 50]
self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, no_cache=True)
fake_volume.assert_called_with(
ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY,
CacheMode.no_cache, parallel=True) # This should be the no_cache argument.
@patch.object(VolumeService, 'get_cutout', autospec=True)
def test_get_cutout_no_cache_False_backwards_compatability(self, fake_volume):
"""
Test that if no_cache not provided, it defaults to True when calling
the volume service's get_cutout().
"""
chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16')
resolution = 0
x_range = [20, 40]
y_range = [50, 70]
z_range = [30, 50]
self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, no_cache=False)
fake_volume.assert_called_with(
ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY,
CacheMode.cache, parallel=True) # This should be the no_cache argument.
rmt = BossRemote('/jobs/boss_config.cfg')
img_chan = ChannelResource(params['img_channel'],
params['collection'],
params['experiment'],
type='image',
datatype='uint8')
lbl_chan = ChannelResource(params['lbl_channel'],
params['collection'],
params['experiment'],
type='annotation',
datatype='uint64')
# Get the image data from the BOSS
x_train = rmt.get_cutout(img_chan, params['resolution'], params['x_rng'],
params['y_rng'], params['z_rng'])
y_train = rmt.get_cutout(lbl_chan, params['resolution'], params['x_rng'],
params['y_rng'], params['z_rng'])
# Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X])
x_train = x_train[:, np.newaxis, :, :].astype(np.float32)
y_train = y_train[:, np.newaxis, :, :].astype(np.float32)
# Pixel values must be in [0,1]
x_train /= 255.
y_train = (y_train > 0).astype('float32')
tile_size = tuple(params['tile_size'])
train_pct = params['train_pct']
# -------------------------------------------------------------------------
# Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X])
class NeuroDataResource:
def __init__(self, host, token, collection, experiment):
self._bossRemote = BossRemote({
'protocol': 'https',
'host': host,
'token': token
})
self.collection = collection
self.experiment = experiment
self.channels = self._bossRemote.list_channels(collection, experiment)
self.channels.remove('empty') #Delete "empty" channel
self._get_coord_frame_details()
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size,
data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print(
'Error: You must supply zRange, yRange, xRange kwargs in list format'
)
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange,
zRange)
#Datatype check. Recast to original datatype if data is float64
if data.dtype == datatype:
return data
else:
return data.astype(datatype)
"typename": "uint8blk",
"dataname": "Luis1",
"versioned": "0"
}))
print(dat1.content)
# BOSS Data fetch:
boss = BossRemote({
"protocol": "https",
"host": "api.theboss.io",
"token": "db1cec2c865fc84e48772f4f4a5f010c0a180b88",
})
volumeB = boss.get_cutout(
boss.get_channel("em", "pinky40", "v7"),
2,
[10000, 10100],
[10000, 10100],
[501, 502],
)
volumeB = volumeB.tobytes()
print(len(volumeB))
dif = (32 * 32 * 32) - len(volumeB)
volumeB = volumeB + str("".join((["0"] * dif)))
print(len(volumeB))
# print len(data)
import requests
# Ranges use the Python convention where the number after the : is the stop
# value. Thus, x_rng specifies x values where: 0 <= x < 8.
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
# Note that the numpy matrix is in Z, Y, X order.
data = numpy.random.randint(1, 3000, (5, 4, 8))
data = data.astype(numpy.uint16)
# Upload the cutout to the channel.
rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data)
# Verify that the cutout uploaded correctly.
cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, cutout_data)
print('Cutout uploaded and verified.')
# Get only a small piece of the cutout.
small_cutout_data = rmt.get_cutout(chan_actual, 0, [0, 1], [0, 1], [0, 5])
numpy.testing.assert_array_equal(data[0:5, 0:1, 0:1], small_cutout_data)
# For times series data, the matrix is in t, Z, Y, X order.
time_rng = [0, 3]
time_data = numpy.random.randint(1, 3000, (3, 5, 4, 8), numpy.uint16)
rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, time_data, time_rng)
time_cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng,
class InternTileProcessor(TileProcessor):
"""A Tile processor for a single image file identified by z index"""
def __init__(self):
"""Constructor to add custom class var"""
TileProcessor.__init__(self)
self.remote = None
self.channel = None
def setup(self, parameters):
""" Method to load the file for uploading data. Assumes intern token is set via environment variable or config
default file
Args:
parameters (dict): Parameters for the dataset to be processed
MUST HAVE THE CUSTOM PARAMETERS: "x_offset": offset to apply when querying the Boss
"y_offset": offset to apply when querying the Boss
"z_offset": offset to apply when querying the Boss
"x_tile": size of a tile in x dimension
"y_tile": size of a tile in y dimension
"collection": source collection
"experiment": source experiment
"channel": source channel
"resolution": source resolution
Returns:
None
"""
self.parameters = parameters
self.remote = BossRemote()
self.channel = ChannelResource(self.parameters["channel"],
self.parameters["collection"],
self.parameters["experiment"])
self.channel = self.remote.get_project(self.channel)
def process(self, file_path, x_index, y_index, z_index, t_index=0):
"""
Method to load the image file.
Args:
file_path(str): An absolute file path for the specified tile
x_index(int): The tile index in the X dimension
y_index(int): The tile index in the Y dimension
z_index(int): The tile index in the Z dimension
t_index(int): The time index
Returns:
(io.BufferedReader): A file handle for the specified tile
"""
# Compute cutout args
x_rng = [
self.parameters["x_tile"] * x_index + self.parameters["x_offset"],
self.parameters["x_tile"] * (x_index + 1) +
self.parameters["x_offset"]
]
y_rng = [
self.parameters["y_tile"] * y_index + self.parameters["y_offset"],
self.parameters["y_tile"] * (y_index + 1) +
self.parameters["y_offset"]
]
z_rng = [
z_index + self.parameters["z_offset"],
z_index + 1 + self.parameters["z_offset"]
]
if z_index + self.parameters["z_offset"] < 0:
data = np.zeros(
(self.parameters["x_tile"], self.parameters["y_tile"]),
dtype=np.int32,
order="C")
else:
# Get data
cnt = 0
while cnt < 5:
try:
data = self.remote.get_cutout(
self.channel, self.parameters["resolution"], x_rng,
y_rng, z_rng)
data = np.asarray(data, np.uint32)
break
except Exception as err:
if cnt > 5:
raise err
cnt += 1
time.sleep(10)
# Save sub-img to png and return handle
upload_img = Image.fromarray(np.squeeze(data))
output = six.BytesIO()
upload_img.save(output, format="TIFF")
# Send handle back
return output
def boss_pull_cutout(args):
if args.config:
rmt = BossRemote(args.config)
else:
cfg = _generate_config(args.token, args)
with open("intern.cfg", "w") as f:
cfg.write(f)
rmt = BossRemote("intern.cfg")
COLL_NAME = args.coll
EXP_NAME = args.exp
CHAN_NAME = args.chan
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype
)
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
# get coordinate frame to determine padding bounds
cfr = CoordinateFrameResource(args.coord)
cfr_actual = rmt.get_project(cfr)
x_min_bound = cfr_actual.x_start
x_max_bound = cfr_actual.x_stop
y_min_bound = cfr_actual.y_start
y_max_bound = cfr_actual.y_stop
z_min_bound = cfr_actual.z_start
z_max_bound = cfr_actual.z_stop
print("Data model setup.")
xmin = np.max([x_min_bound, args.xmin - args.padding])
xmax = np.min([x_max_bound, args.xmax + args.padding])
x_rng = [xmin, xmax]
ymin = np.max([y_min_bound, args.ymin - args.padding])
ymax = np.min([y_max_bound, args.ymax + args.padding])
y_rng = [ymin, ymax]
zmin = np.max([z_min_bound, args.zmin - args.padding])
zmax = np.min([z_max_bound, args.zmax + args.padding])
z_rng = [zmin, zmax]
# Verify that the cutout uploaded correctly.
attempts = 0
while attempts < 3:
try:
cutout_data = rmt.get_cutout(chan_actual, args.res, x_rng, y_rng, z_rng)
break
except HTTPError as e:
if attempts < 3:
attempts += 1
print("Obtained HTTP error from server. Trial {}".format(attempts))
else:
print("Failed 3 times: {}".format(e))
# Data will be in Z,Y,X format
# Change to X,Y,Z for pipeline
cutout_data = np.transpose(cutout_data, (2, 1, 0))
def _upload(f):
print("Uploading to s3:/{}/{}".format(args.bucket, args.output))
s3 = boto3.resource("s3")
f.seek(0, 0)
s3.Object(args.bucket, args.output).put(Body=f)
# Clean up.
if args.bucket and args.s3_only:
with tempfile.TemporaryFile() as f:
np.save(f, cutout_data)
_upload(f)
else:
with open(args.output, "w+b") as f:
np.save(f, cutout_data)
if args.bucket:
_upload(f)
# Write the mesh obj
with open("mesh_22_test.obj", "wb") as fh:
fh.write(mesh_obj)
# Write the Neuroglancer ready mesh
with open("mesh_ng_test", "wb") as fh:
fh.write(mesh_ng)
###### Use MeshService from direct import ######
## No need for remote import.
# Define channel resource
ann_chan = rmt.get_channel(CHAN, COLL, EXP)
# Grab cutout volume
volume = rmt.get_cutout(ann_chan, res, x_rng, y_rng, z_rng)
# Initialize MeshService
mesh_serv = MeshService()
# Create mesh
mesh = mesh_serv.create(volume, x_rng, y_rng, z_rng)
# Convert mesh data to obj
mesh_obj = mesh.obj_mesh()
# Convert mesh data to precompute format for neuroglancer
mesh_ng = mesh.ng_mesh()
# Write the mesh obj
with open("mesh_22_test_serv.obj", "wb") as fh:
class NeuroDataResource:
def __init__(self, host, token, collection, experiment):
self._bossRemote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
self.collection = collection
self.experiment = experiment
self.channels = self._bossRemote.list_channels(collection, experiment)
self.channels.remove('empty') #Delete "empty" channel
self._get_coord_frame_details()
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print('Error: You must supply zRange, yRange, xRange kwargs in list format')
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource,
0,
xRange,
yRange,
zRange)
return data
@staticmethod
def ingest_volume(host, token, channel_name, collection, experiment, volume):
"""
Assumes the collection and experiment exists in BOSS.
"""
remote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
if volume.dtype == 'uint64':
dtype = 'uint64'
img_type = 'annotation'
sources = ['empty']
else:
dtype = volume.dtype.name
img_type = 'image'
sources = []
try:
channel_resource = ChannelResource(channel_name, collection, experiment)
channel = remote.get_project(channel_resource)
except:
channel_resource = ChannelResource(channel_name, collection, experiment,
type=img_type,
sources=sources,
datatype=dtype)
channel = remote.create_project(channel_resource)
#Get max size of experiment
exp_resource = ExperimentResource(experiment, collection)
coord_frame = remote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = remote.get_project(coord_frame_resource)
y_stop, x_stop = data.y_stop, data.x_stop
for z in range(volume.shape[0]):
print('Uploading {} slice'.format(z))
remote.create_cutout(channel,
0,
(0, x_stop),
(0, y_stop),
(z, z + 1),
volume[z, :, :].reshape((-1, y_stop, x_stop)))
