class NeuroDataResource: def __init__(self, host, token, collection, experiment, chanList): self._collection = collection self._experiment = experiment self._bossRemote = BossRemote({'protocol':'https', 'host':host, 'token':token}) self._chanList = {} for chanDict in chanList: try: self._chanList[chanDict['name']] = ChannelResource(chanDict['name'], collection, experiment, 'image', datatype=chanDict['dtype']) except: #TODO error handle here raise def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if not chan in self._chanList.keys(): print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print('Error: You must supply zRange, yRange, xRange kwargs in list format') data = self._bossRemote.get_cutout(self._chanList[chan], 0, xRange, yRange, zRange) return data
class NeuroDataResource: def __init__(self, host, token, collection, experiment, chanList): self._collection = collection self._experiment = experiment self._bossRemote = BossRemote({'protocol':'https', 'host':host, 'token':token}) self._chanList = {} for chanDict in chanList: try: self._chanList[chanDict['name']] = ChannelResource(chanDict['name'], collection, experiment, 'image', datatype=chanDict['dtype']) except: #TODO error handle here raise def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if not chan in self._chanList.keys(): print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print('Error: You must supply zRange, yRange, xRange kwargs in list format') data = self._bossRemote.get_cutout(self._chanList[chan], 0, xRange, yRange, zRange) return data def save_img(self, fname, cutout, format = 'tif'): ''' Saves a image as a tif or nii file Leave the file type suffix out of fname ''' if (not fname) or (type(fname) != str): print('Error: Give valid filename') if (not cutout) or type(cutout) != np.array: print('Error: Give valid image cutout') if format == 'tif': max_grey_val = cutout.max(axis=1).max(axis=1).max() dim = cutout.shape() cutout_rgb = np.zeros((dim[0], dim[1], dim[2], 3)) for i in range(dim[0]): for j in range(dim[1]): for k in range(dim[2]): normalized_rgb = int( 255 * (cutout[i][j][k]/max_grey_val) ) for p in range(3): cutout_rgb[i][j][k][p] = normalized_rgb imsave(fname+'.tif', cutout_rgb.astype(np.uint8)) else: nifti_img = nib.Nifti1Image(cutout, np.eye(4)) nib.save(nifti_img, fname+'.nii')
def test_can_retrieve_correct_u64_data(self): data = array("bossdb://kasthuri2015/em/3cylneuron_v1") boss = BossRemote(_DEFAULT_BOSS_OPTIONS) get_cutout_data = boss.get_cutout( boss.get_channel("3cylneuron_v1", "kasthuri2015", "em"), 0, [6000, 6200], [12000, 12500], [923, 924], ) self.assertTrue( (get_cutout_data == data[923:924, 12000:12500, 6000:6200]).all() )
def get_boss_data(args): args.x_rng = [args.xmin, args.xmax] args.y_rng = [args.ymin, args.ymax] args.z_rng = [args.zmin, args.zmax] config = {"protocol": "https", "host": args.host, "token": args.token} rmt = BossRemote(config) segchan = ChannelResource(args.chan_seg, args.coll, args.exp, 'annotation', datatype=args.dtype_seg) # Get the image data from the BOSS seg_data = rmt.get_cutout(segchan, args.res, args.x_rng, args.y_rng, args.z_rng) synchan = ChannelResource(args.chan_syn, args.coll, args.exp, 'annotation', datatype=args.dtype_syn) syn_data = rmt.get_cutout(synchan, args.res, args.x_rng, args.y_rng, args.z_rng) #Wany XYZ seg_data = np.transpose(seg_data,(2,1,0)) syn_data = np.transpose(syn_data,(2,1,0)) return seg_data, syn_data
def get_boss_data(args): config = { "protocol": "https", "host": "api.bossdb.io", "token": args.token } rmt = BossRemote(config) print('[info] Downloading data from BOSS') chan = ChannelResource(args.chan_img, args.coll, args.exp, 'image', datatype=args.dtype_img) # Get the image data from the BOSS x_train = rmt.get_cutout(chan, args.res, [args.xmin, args.xmax], [args.ymin, args.ymax], [args.zmin, args.zmax]) lchan = ChannelResource(args.chan_labels, args.coll, args.exp, 'annotation', datatype=args.dtype_lbl) y_train = rmt.get_cutout(lchan, args.res, [args.xmin, args.xmax], [args.ymin, args.ymax], [args.zmin, args.zmax]) print('[info] Downloaded BOSS data') # Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X]) x_train = x_train[:, np.newaxis, :, :].astype(np.float32) y_train = y_train[:, np.newaxis, :, :].astype(np.float32) # Pixel values must be in [0,1] x_train /= 255. y_train = (y_train > 0).astype('float32') return x_train, y_train
def get_boss_data(args): config = { "protocol": "https", "host": "api.theBoss.io", "token": args.token } rmt = BossRemote(config) chan = ChannelResource(args.img_channel, args.collection, args.experiment, 'image', datatype='uint8') # Get the image data from the BOSS x_train = rmt.get_cutout(chan, args.resolution, args.x_rng, args.y_rng, args.z_rng) lchan = ChannelResource(args.lbl_channel, args.collection, args.experiment, 'annotation', datatype='uint64') y_train = rmt.get_cutout(lchan, args.resolution, args.x_rng, args.y_rng, args.z_rng) # Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X]) x_train = x_train[:, np.newaxis, :, :].astype(np.float32) y_train = y_train[:, np.newaxis, :, :].astype(np.float32) # Pixel values must be in [0,1] x_train /= 255. y_train = (y_train > 0).astype('float32') return x_train, y_train
def download(self, bbox, mip, parallel=1, renumber=False): if parallel != 1: raise ValueError("Only parallel=1 is supported for boss.") elif renumber != False: raise ValueError("Only renumber=False is supported for boss.") bounds = Bbox.clamp(bbox, self.meta.bounds(mip)) if self.autocrop: image, bounds = autocropfn(self.meta, image, bounds, mip) if bounds.subvoxel(): raise exceptions.EmptyRequestException( 'Requested less than one pixel of volume. {}'.format(bounds)) x_rng = [bounds.minpt.x, bounds.maxpt.x] y_rng = [bounds.minpt.y, bounds.maxpt.y] z_rng = [bounds.minpt.z, bounds.maxpt.z] layer_type = 'image' if self.meta.layer_type == 'unknown' else self.meta.layer_type chan = ChannelResource( collection_name=self.meta.path.bucket, experiment_name=self.meta.path.dataset, name=self.meta.path.layer, # Channel type=layer_type, datatype=self.meta.data_type, ) rmt = BossRemote(boss_credentials) cutout = rmt.get_cutout(chan, mip, x_rng, y_rng, z_rng, no_cache=True) cutout = cutout.T cutout = cutout.astype(self.meta.dtype) cutout = cutout[::steps.x, ::steps.y, ::steps.z] if len(cutout.shape) == 3: cutout = cutout.reshape(tuple(list(cutout.shape) + [1])) if self.bounded or self.autocrop or bounds == bbox: return VolumeCutout.from_volume(self.meta, mip, cutout, bounds) # This section below covers the case where the requested volume is bigger # than the dataset volume and the bounds guards have been switched # off. This is useful for Marching Cubes where a 1px excess boundary # is needed. shape = list(bbox.size3()) + [cutout.shape[3]] renderbuffer = np.zeros(shape=shape, dtype=self.meta.dtype, order='F') shade(renderbuffer, bbox, cutout, bounds) return VolumeCutout.from_volume(self.meta, mip, renderbuffer, bbox)
def _boss_cutout(self, requested_bbox, steps, channel_slice=slice(None)): bounds = Bbox.clamp(requested_bbox, self.bounds) if bounds.subvoxel(): raise exceptions.EmptyRequestException( 'Requested less than one pixel of volume. {}'.format(bounds)) x_rng = [bounds.minpt.x, bounds.maxpt.x] y_rng = [bounds.minpt.y, bounds.maxpt.y] z_rng = [bounds.minpt.z, bounds.maxpt.z] layer_type = 'image' if self.layer_type == 'unknown' else self.layer_type chan = ChannelResource( collection_name=self.path.bucket, experiment_name=self.path.dataset, name=self.path.layer, # Channel type=layer_type, datatype=self.dtype, ) rmt = BossRemote(boss_credentials) cutout = rmt.get_cutout(chan, self.mip, x_rng, y_rng, z_rng, no_cache=True) cutout = cutout.T cutout = cutout.astype(self.dtype) cutout = cutout[::steps.x, ::steps.y, ::steps.z] if len(cutout.shape) == 3: cutout = cutout.reshape(tuple(list(cutout.shape) + [1])) if self.bounded or self.autocrop or bounds == requested_bbox: return VolumeCutout.from_volume(self, cutout, bounds) # This section below covers the case where the requested volume is bigger # than the dataset volume and the bounds guards have been switched # off. This is useful for Marching Cubes where a 1px excess boundary # is needed. shape = list(requested_bbox.size3()) + [cutout.shape[3]] renderbuffer = np.zeros(shape=shape, dtype=self.dtype, order='F') txrx.shade(renderbuffer, requested_bbox, cutout, bounds) return VolumeCutout.from_volume(self, renderbuffer, requested_bbox)
class VolumeServiceTest_v1(unittest.TestCase): """Integration tests of the Boss volume service API. Because setup and teardown involves many REST calls, tests are only divided into tests of the different types of data model resources. All operations are performed within a single test of each resource. """ @classmethod def setUpClass(cls): """Do an initial DB clean up in case something went wrong the last time. If a test failed really badly, the DB might be in a bad state despite attempts to clean up during tearDown(). """ cls.rmt = BossRemote('test.cfg', API_VER) # Turn off SSL cert verification. This is necessary for interacting with # developer instances of the Boss. cls.rmt.project_service.session_send_opts = {'verify': False} cls.rmt.metadata_service.session_send_opts = {'verify': False} cls.rmt.volume_service.session_send_opts = {'verify': False} requests.packages.urllib3.disable_warnings(InsecureRequestWarning) coll_name = 'collection2323{}'.format(random.randint(0, 9999)) cls.coll = CollectionResource(coll_name, 'bar') cf_name = 'BestFrame{}'.format(random.randint(0, 9999)) cls.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.', 0, 2048, 0, 2048, 0, 100, 1, 1, 1, 'nanometers', 0, 'nanoseconds') # cls.exp.coord_frame must be set with valid id before creating. cls.exp = ExperimentResource('exp2323x2', cls.coll.name, cls.coord.name, 'my experiment', 1, 'isotropic', 10) cls.chan = ChannelResource('myVolChan', cls.coll.name, cls.exp.name, 'image', 'test channel', 0, 'uint8', 0) cls.chan16 = ChannelResource('myVol16bitChan', cls.coll.name, cls.exp.name, 'image', '16 bit test channel', 0, 'uint16', 0) cls.ann_chan = ChannelResource('annVolChan2', cls.coll.name, cls.exp.name, 'annotation', 'annotation test channel', 0, 'uint64', 0, sources=[cls.chan.name]) # This channel reserved for testing get_ids_in_region(). This is a # separate channel so we don't have to worry about ids written by # other tests. cls.ann_region_chan = ChannelResource( 'annRegionChan2', cls.coll.name, cls.exp.name, 'annotation', 'annotation ids in region test channel', 0, 'uint64', 0, sources=[cls.chan.name]) # This channel reerved for testing tight bounding boxes. cls.ann_bounding_chan = ChannelResource( 'annRegionChan3', cls.coll.name, cls.exp.name, 'annotation', 'annotation ids in bounding box test channel', 0, 'uint64', 0, sources=[cls.chan.name]) cls.rmt.create_project(cls.coll) cls.rmt.create_project(cls.coord) cls.rmt.create_project(cls.exp) cls.rmt.create_project(cls.chan16) cls.rmt.create_project(cls.chan) cls.rmt.create_project(cls.ann_chan) cls.rmt.create_project(cls.ann_region_chan) cls.rmt.create_project(cls.ann_bounding_chan) @classmethod def tearDownClass(cls): """Clean up the data model objects used by this test case. This method is used by both tearDownClass() and setUpClass(). """ try: cls.rmt.delete_project(cls.ann_bounding_chan) except HTTPError: pass try: cls.rmt.delete_project(cls.ann_region_chan) except HTTPError: pass try: cls.rmt.delete_project(cls.ann_chan) except HTTPError: pass try: cls.rmt.delete_project(cls.chan16) except HTTPError: pass try: cls.rmt.delete_project(cls.chan) except HTTPError: pass try: cls.rmt.delete_project(cls.exp) except HTTPError: pass try: cls.rmt.delete_project(cls.coord) except HTTPError: pass try: cls.rmt.delete_project(cls.coll) except HTTPError: pass def setUp(self): self.rmt = BossRemote('test.cfg') def tearDown(self): pass def test_reserve_ids(self): first_id = self.rmt.reserve_ids(self.ann_chan, 20) self.assertTrue(first_id > 0) def test_get_bounding_box_id_doesnt_exist(self): resolution = 0 id = 12345678 with self.assertRaises(HTTPError) as err: self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') expected_msg_prefix = 'Reserve ids failed' self.assertTrue(err.message.startwswith(expected_msg_prefix)) @unittest.skip('Skipping - currently indexing disabled') def test_get_bounding_box_spans_cuboids_in_x(self): x_rng = [511, 515] y_rng = [0, 8] z_rng = [0, 5] t_rng = [0, 1] id = 77555 data = numpy.zeros((5, 8, 4), dtype='uint64') data[1][0][0] = id data[2][1][1] = id data[3][2][3] = id resolution = 0 self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) expected = { 'x_range': [0, 1024], 'y_range': [0, 512], 'z_range': [0, 16], 't_range': [0, 1] } actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_bounding_box_spans_cuboids_in_y(self): x_rng = [0, 8] y_rng = [511, 515] z_rng = [0, 5] t_rng = [0, 1] id = 77666 data = numpy.zeros((5, 4, 8), dtype='uint64') data[1][0][0] = id data[2][1][0] = id data[3][2][0] = id resolution = 0 self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) expected = { 'x_range': [0, 512], 'y_range': [0, 1024], 'z_range': [0, 16], 't_range': [0, 1] } actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_bounding_box_spans_cuboids_in_z(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [30, 35] t_rng = [0, 1] id = 77888 data = numpy.zeros((5, 4, 8), dtype='uint64') data[1][0][0] = id data[2][1][0] = id data[3][2][0] = id resolution = 0 self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) expected = { 'x_range': [0, 512], 'y_range': [0, 512], 'z_range': [16, 48], 't_range': [0, 1] } actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_tight_bounding_box_x_axis(self): """Test tight bounding box with ids that span three cuboids along the x axis.""" resolution = 0 x_rng = [511, 1025] y_rng = [512, 1024] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 512, 514), dtype='uint64') x_id = 123 y_id = 127 z_id = 500000000000000000 # Id in partial region on x axis closest to origin. data[1][1][0] = x_id # Id in partial region on x axis furthest from origin. data[1][1][513] = x_id # Id in cuboid aligned region. data[2][2][21] = x_id data[2][1][22] = y_id data[4][24][72] = z_id expected = { 'x_range': [511, 1025], 'y_range': [513, 515], 'z_range': [17, 19] } self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_bounding_box(self.ann_bounding_chan, resolution, x_id, bb_type='tight') @unittest.skip('Skipping - currently indexing disabled') def test_tight_bounding_box_y_axis(self): """Test tight bounding box with ids that span three cuboids along the x axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [511, 1025] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 514, 512), dtype='uint64') x_id = 123 y_id = 127 z_id = 500000000000000000 # Id in partial region on y axis closest to origin. data[1][0][10] = y_id # Id in partial region on y axis furthest from origin. data[1][513][13] = y_id # Id in cuboid aligned region. data[2][2][21] = y_id data[2][3][20] = x_id data[4][25][71] = z_id expected = { 'x_range': [522, 526], 'y_range': [511, 1025], 'z_range': [17, 19] } self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_bounding_box(self.ann_bounding_chan, resolution, y_id, bb_type='tight') @unittest.skip('Skipping - currently indexing disabled') def test_tight_bounding_box_z_axis(self): """Test tight bounding box with ids that span three cuboids along the x axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [512, 1024] z_rng = [15, 33] t_rng = [0, 1] data = numpy.zeros((18, 512, 512), dtype='uint64') x_id = 123 y_id = 127 z_id = 500000000000000000 # Id in partial region on z axis closest to origin. data[0][22][60] = z_id # Id in partial region on z axis furthest from origin. data[17][23][63] = z_id # Id in cuboid aligned region. data[5][24][71] = z_id data[3][2][20] = x_id data[3][1][21] = y_id expected = { 'x_range': [572, 583], 'y_range': [534, 537], 'z_range': [15, 33] } self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_bounding_box(self.ann_bounding_chan, resolution, z_id, bb_type='tight') def test_get_ids_in_region_none(self): """Run on region that hasn't been written with ids, yet.""" resolution = 0 x_rng = [1536, 1540] y_rng = [1536, 1540] z_rng = [48, 56] t_rng = [0, 1] data = numpy.zeros((8, 4, 4), dtype='uint64') expected = [] # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) def test_filtered_cutout(self): """Test filtered cutout using same data written for get_ids_in_region_x_axis.""" resolution = 0 x_rng = [511, 1025] y_rng = [512, 1024] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 512, 514), dtype='uint64') # Id in partial region on x axis closest to origin. data[1][1][0] = 123 # Id in partial region on x axis furthest from origin. data[1][1][513] = 321 # Id in cuboid aligned region. data[10][20][21] = 55555 expected = [123, 321, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Should get back the exact data given in create_cutout(). filtered_data1 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[123, 321, 55555]) numpy.testing.assert_array_equal(data, filtered_data1) # Filter on id 123. expected_data_123 = numpy.zeros((16, 512, 514), dtype='uint64') expected_data_123[1][1][0] = 123 filtered_data_123 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[123]) numpy.testing.assert_array_equal(expected_data_123, filtered_data_123) # Filter on id 321. expected_data_321 = numpy.zeros((16, 512, 514), dtype='uint64') expected_data_321[1][1][513] = 321 filtered_data_321 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[321]) numpy.testing.assert_array_equal(expected_data_321, filtered_data_321) # Filter on ids 123 and 55555. expected_data_123_55555 = numpy.zeros((16, 512, 514), dtype='uint64') expected_data_123_55555[1][1][0] = 123 expected_data_123_55555[10][20][21] = 55555 filtered_data_123_55555 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[123, 55555]) numpy.testing.assert_array_equal(expected_data_123_55555, filtered_data_123_55555) @unittest.skip('Skipping - currently indexing disabled') def test_get_ids_in_region_x_axis(self): """Test using a region that's cuboid aligned except for the x axis.""" resolution = 0 x_rng = [511, 1025] y_rng = [512, 1024] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 512, 514), dtype='uint64') # Id in partial region on x axis closest to origin. data[1][1][0] = 123 # Id in partial region on x axis furthest from origin. data[1][1][513] = 321 # Id in cuboid aligned region. data[10][20][21] = 55555 expected = [123, 321, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_ids_in_region_y_axis(self): """Test using a region that's cuboid aligned except for the y axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [511, 1025] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 514, 512), dtype='uint64') # Id in partial region on y axis closest to origin. data[1][0][1] = 456 # Id in partial region on y axis furthest from origin. data[1][513][1] = 654 # Id in cuboid aligned region. data[10][21][20] = 55555 # expected = [123, 321, 456, 654, 789, 987, 55555] expected = [456, 654, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_ids_in_region_z_axis(self): """Test using a region that's cuboid aligned except for the z axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [512, 1024] z_rng = [15, 33] t_rng = [0, 1] data = numpy.zeros((18, 512, 512), dtype='uint64') # Id in partial region on z axis closest to origin. data[0][1][1] = 789 # Id in partial region on z axis furthest from origin. data[17][1][1] = 987 # Id in cuboid aligned region. data[11][20][20] = 55555 expected = [789, 987, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) def test_upload_and_download_to_channel(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] data = numpy.random.randint(1, 254, (5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_to_channel_with_time(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] t_rng = [3, 6] data = numpy.random.randint(1, 254, (3, 5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_subsection_to_channel(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 19] sub_x = [12, 14] sub_y = [7, 10] sub_z = [12, 17] data = numpy.random.randint(1, 10, (9, 5, 10)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan, 0, sub_x, sub_y, sub_z) numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual) def test_upload_to_x_edge_of_channel(self): x_rng = [10, 2048] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_y_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 2048] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_z_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 100] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_x_edge_of_channel(self): x_rng = [10, 2049] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_y_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 2049] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_z_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 101] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_and_download_to_channel_16bit(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan16, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_subsection_to_channel_16bit(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 19] sub_x = [12, 14] sub_y = [7, 10] sub_z = [12, 17] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan16, 0, sub_x, sub_y, sub_z) numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual) def test_upload_to_x_edge_of_channel_16bit(self): x_rng = [2000, 2048] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_to_y_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [2000, 2048] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_to_z_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 100] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_past_x_edge_of_channel_16bit(self): x_rng = [2000, 2049] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_past_y_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [2000, 2049] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_past_z_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 101] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_and_download_to_anno_chan(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_subsection_to_anno_chan(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 19] sub_x = [12, 14] sub_y = [7, 10] sub_z = [12, 17] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.ann_chan, 0, sub_x, sub_y, sub_z) numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual) def test_upload_to_x_edge_of_anno_chan(self): x_rng = [2000, 2048] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_y_edge_of_anno_chan(self): x_rng = [10, 20] y_rng = [2000, 2048] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_z_edge_of_anno_chan(self): x_rng = [10, 100] y_rng = [5, 10] z_rng = [10, 100] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_x_edge_of_anno_chan(self): x_rng = [10, 2049] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_y_edge_of_anno_chan(self): x_rng = [10, 991] y_rng = [5, 2049] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_z_edge_of_anno_chan(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 101] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_and_download_to_channel_4D(self): x_rng = [600, 680] y_rng = [600, 640] z_rng = [50, 55] t_rng = [0, 1] shape = (t_rng[1] - t_rng[0], z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_cutout_to_black(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] data = numpy.random.randint(1, 254, (5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) self.rmt.create_cutout_to_black(self.chan, 0, x_rng, y_rng, z_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(numpy.zeros((5, 4, 8)), actual) def test_upload_and_cutout_to_black_with_time(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] t_rng = [3, 6] data = numpy.random.randint(1, 254, (3, 5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) self.rmt.create_cutout_to_black(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) numpy.testing.assert_array_equal(numpy.zeros((3, 5, 4, 8)), actual) def test_upload_and_cutout_to_black_partial(self): x_rng = [0, 1024] y_rng = [0, 1024] z_rng = [0, 5] x_rng_black = [0, 256] y_rng_black = [0, 512] z_rng_black = [2, 3] data = numpy.random.randint(1, 254, (5, 1024, 1024)) data = data.astype(numpy.uint8) expected = numpy.copy(data) expected[2:3, 0:512, 0:256] = 0 self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) self.rmt.create_cutout_to_black(self.chan, 0, x_rng_black, y_rng_black, z_rng_black) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(expected, actual)
class bossHandler: """ This is a class for interacting with BOSS. It basically is a wrapper for Intern. """ def __init__(self, collection_name): """ Constructor: Takes the following argument: :param collection_name: name of the collection in BOSS :type collection_name: string """ self.collection_name = collection_name try: self.rmt = BossRemote() except: print('Unexpected Error:', sys.exc_info()[0]) # def get_collection_list(self): # return self.rmt.list_collections() def list_experiments(self): """ :return: all the experiments available in current collection """ exp_list = self.rmt.list_experiments(self.collection_name) return exp_list def select_experiment(self, experiment_name): """ Select an experiment to be added to this handler """ tmp = ExperimentResource(collection_name=self.collection_name, name=experiment_name) exp = self.rmt.get_project(tmp) self.experiment = exp # return exp def get_experiment(self): """ :return: the currently selected experiment for this handler """ if hasattr(self, 'experiment'): return self.experiment else: raise AttributeError( 'No experiment exists. First, select an experiment using select_experiment' ) def list_channels(self): """ :return: all channel in currently selected experiment """ return self.rmt.list_channels(self.collection_name, self.experiment.name) def select_channel(self, channel_name): """ Select a channel to be added to this handler """ self.channel = self.rmt.get_channel(chan_name=channel_name, coll_name=self.collection_name, exp_name=self.experiment.name) def get_coordinate_frame(self): """ Return: current experiment's coordinate frame """ tmp = CoordinateFrameResource(name=self.experiment.coord_frame) coor = self.rmt.get_project(tmp) self.coordinate_frame = coor return coor def get_all(self): """ :return: the entire channel image data at its native resolution """ x_rng = [self.coordinate_frame.x_start, self.coordinate_frame.x_stop] y_rng = [self.coordinate_frame.y_start, self.coordinate_frame.y_stop] z_rng = [self.coordinate_frame.z_start, self.coordinate_frame.z_stop] return self.rmt.get_cutout(self.channel, 0, x_rng, y_rng, z_rng) def get_cutout(self, x_range, y_range, z_range, resolution=0): """ :return: a cutout of the image data """ return self.rmt.get_cutout(self.channel, resolution, x_range, y_range, z_range)
from intern.remote.localFile import LocalRemote from intern.remote.boss import BossRemote from intern.resource.boss.resource import * import matplotlib.pyplot as plt import numpy as np # BOSS Data fetch which we will upload to the Local Storage: boss = BossRemote({ "protocol": "https", "host": "api.theboss.io", "token": "db1cec2c865fc84e48772f4f4a5f010c0a180b88", }) volumeB = boss.get_cutout( boss.get_channel("em", "pinky40", "v7"), 1, [10000, 10200], [10000, 10090], [500, 520], ) #Local Upload local = LocalRemote({ "host": "/Users/rodrilm2/InternRel/", "datastore": "LocalBossDummy" }) print(local) chan_setup = local.get_channel('em', 'pinky40') proj = local.create_project(chan_setup) volume = local.create_cutout(proj, 'v1', volumeB) #LocalMetadata updates Channel1 = local.retrieve('em')
class NeuroDataResource: def __init__(self, host, token, collection, experiment, requested_channels, x_range=None, y_range=None, z_range=None, resolution=0): self._bossRemote = BossRemote({ 'protocol': 'https', 'host': host, 'token': token }) self.collection = collection self.experiment = experiment self.resolution = resolution self.channels = self._bossRemote.list_channels(collection, experiment) if len(requested_channels) == 0: self.requested_channels = self.channels else: self.requested_channels = requested_channels self._get_coord_frame_details() # validate range #if not self.correct_range(z_range, y_range, x_range): # raise Exception("Error: Inccorect dimension range") self.x_range = x_range or [0, self.max_dimensions[2]] self.y_range = y_range or [0, self.max_dimensions[1]] self.z_range = z_range or [0, self.max_dimensions[0]] def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print( 'Error: You must supply zRange, yRange, xRange kwargs in list format' ) return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, self.resolution, xRange, yRange, zRange) #Datatype check. Recast to original datatype if data is float64 if data.dtype == datatype: return data else: return data.astype(datatype) def correct_range(self, z_range, y_range, x_range): x_start, x_end = x_range if x_start < 0 or x_end > self.max_dimensions[2]: return False y_start, y_end = y_range if y_start < 0 or y_end > self.max_dimensions[1]: return False z_start, z_end = z_range if z_start < 0 or z_end > self.max_dimensions[0]: return False return True
# Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, 'image', '', datatype='uint16') try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) x_rng = [0, xmax] y_rng = [0, ymax] z_rng = [0, zmax] t_rng = [0, tmax] print('Data model setup.') data = np.random.randint(1, 3000, (tmax, zmax, ymax, xmax)) data = data.astype(np.uint16) # Upload the cutout to the channel. rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) cutout_data = rmt.get_cutout( chan_actual, 0, x_rng, y_rng, z_rng, time_range=t_rng) np.testing.assert_array_equal(data, cutout_data) print(np.shape(cutout_data)) # (10, 5, 4, 8)
#You are ready! #If you would like to get multiple images you can make a while loop or for loop, let me know #if you want me to do it :) boss = BossRemote({ "protocol": "https", "host": "api.theboss.io", #Remember to change your token here. You can get your own at: https://api.theboss.io/token/ "token": "Token" }) #Here you will specify form where the data is coming from, the resolution, and the size of your image. volume = boss.get_cutout( boss.get_channel("cc", "kasthuri2015", "em"), 0, [5000, 6000], [8000, 9000], [1100, 1200], ) volume2 = boss.get_cutout( boss.get_channel("mitochondria", "kasthuri2015", "em"), 0, [5000, 6000], [8000, 9000], [1100, 1200], ) volume3 = boss.get_cutout( boss.get_channel("3cylsynapse_v1", "kasthuri2015", "em"), 0,
from intern.remote.boss import BossRemote from intern.resource.boss.resource import ChannelResource import matplotlib.pyplot as plt import numpy as np boss = BossRemote({ "protocol": "https", "host": "api.theboss.io", "token": "db1cec2c865fc84e48772f4f4a5f010c0a180b88" }) #Here you will specify form where the data is coming from, the resolution, and the size of your image. volume = boss.get_cutout( boss.get_channel("em", "pinky40", "v7"), 0, [10000, 10500], [10000, 10500], [500, 550], ) print(volume) plt.imshow(volume[1, :, :], cmap="gray") plt.show()
class RelayStorageManager(StorageManager): """ """ def __init__(self, **kwargs): """ Create a new RelayStorageManager. Arguments: block_size: How much data should go in each file """ self.block_size = kwargs.get("block_size", (256, 256, 16)) if "next_layer" in kwargs: self._next = kwargs["next_layer"] self.is_terminal = False else: self.is_terminal = True if "boss_remote" in kwargs: self.boss_remote = kwargs["boss_remote"] elif "upstream_uri" in kwargs: self.token = kwargs.get("token", "public") self.boss_remote = BossRemote({ "host": kwargs["upstream_uri"], "protocol": kwargs.get("protocol", "http"), "token": kwargs.get("token", "public"), }) def hasdata( self, col: str, exp: str, chan: str, res: int, xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int], ): has_data = self.boss_remote.get_channel(f"bossdb://{col}/{exp}/{chan}") if has_data: return True if not self.is_terminal: return self._next.hasdata(col, exp, chan, res, xs, ys, zs) return False def setdata( self, data: np.array, col: str, exp: str, chan: str, res: int, xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int], ): return self.boss_remote.create_cutout( self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs, data) def getdata( self, col: str, exp: str, chan: str, res: int, xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int], ) -> np.array: return self.boss_remote.get_cutout( self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs) def __repr__(self): return f"<RelayStorageManager [BossRemote]>" def get_stack_names(self): if self.is_terminal: return [str(self), f"↳{self.boss_remote}"] else: return [ str(self), f"↳{self.boss_remote}", *self._next.get_stack_names() ]
# Note that the numpy matrix is in Z, Y, X order. ann_data = numpy.random.randint(start_id, start_id + 10, np_size, dtype='uint64') # Make sure start_id is used at least once. ann_data[0][1][1] = start_id # 0 is native resolution. res = 0 # Upload annotation data to the channel. rmt.create_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng, ann_data) # Verify that the cutout uploaded correctly. ann_cutout_data = rmt.get_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(ann_data, ann_cutout_data) print('Annotation data uploaded and verified.') # Get ids in the region. ids = rmt.get_ids_in_region(ann_chan_actual, res, x_rng, y_rng, z_rng) print('Ids in region are:') print(ids) # Get the loose bounding box for id start_id. This should be the bounds of the # first cuboid (512, 512, 16). loose = rmt.get_bounding_box(ann_chan_actual, res, start_id, bb_type='loose')
args = parse_args() config = { "protocol": "https", "host": "api.theBoss.io", "token": args.token } rmt = BossRemote(config) chan = ChannelResource(args.in_channel, args.collection, args.experiment, 'image', datatype='uint8') # Get the image data from the BOSS x_test = rmt.get_cutout(chan, args.resolution, args.x_rng, args.y_rng, args.z_rng) # Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X]) x_test = x_test[:, np.newaxis, :, :].astype(np.float32) # Pixel values must be in [0,1] if x_test.max() > 1.0: x_test /= 255. tile_size = (512, 512) z_step = args.z_step # ---------------------------------------------------------------------- # load model model = create_unet((1, tile_size[0], tile_size[1])) if args.do_synapse: model.load_weights('/src/weights/synapse_weights.hdf5')
chan = ChannelResource(CHAN_NAME, COLL_NAME, EXP_NAME, "image", datatype="uint8") x_rng = [0, 250] y_rng = [0, 250] z_rng = [0, 250] # Use base resolution. res = 0 # Download the cutout from the channel. data = boss_rmt.get_cutout(chan, res, x_rng, y_rng, z_rng) # cloud-volume likes data in X Y Z format data = np.transpose(data) # Set up cloud-volume remote cv_config = { "protocol": "gcp", "cloudpath": "neuroproof_examples/training_sample/images", } cv_rmt = CloudVolumeRemote(cv_config) # create a new info (similar to coordinate frame) info = cv_rmt.create_new_info( num_channels=1, layer_type="image",
class TestRemoteGetCutout(unittest.TestCase): def setUp(self): config = {"protocol": "https", "host": "test.theboss.io", "token": "my_secret"} self.remote = BossRemote(config) @patch.object(VolumeService, 'get_cutout', autospec=True) def test_get_cutout_access_mode_defaults_no_cache(self, fake_volume): """ Test that if no_cache not provided, it defaults to True when calling the volume service's get_cutout(). """ chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16') resolution = 0 x_range = [20, 40] y_range = [50, 70] z_range = [30, 50] self.remote.get_cutout(chan, resolution, x_range, y_range, z_range) fake_volume.assert_called_with( ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY, CacheMode.no_cache, parallel=True) # This should be the no_cache argument. @patch.object(VolumeService, 'get_cutout', autospec=True) def test_get_cutout_access_mode_raw(self, fake_volume): """ Test that if no_cache not provided, it defaults to True when calling the volume service's get_cutout(). """ chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16') resolution = 0 x_range = [20, 40] y_range = [50, 70] z_range = [30, 50] self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, access_mode=CacheMode.raw) fake_volume.assert_called_with( ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY, CacheMode.raw, parallel=True) # This should be the no_cache argument. @patch.object(VolumeService, 'get_cutout', autospec=True) def test_get_cutout_access_mode_cache(self, fake_volume): """ Test that if no_cache not provided, it defaults to True when calling the volume service's get_cutout(). """ chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16') resolution = 0 x_range = [20, 40] y_range = [50, 70] z_range = [30, 50] self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, access_mode=CacheMode.cache) fake_volume.assert_called_with( ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY, CacheMode.cache, parallel=True) # This should be the no_cache argument. ##REMOVE IN THE FUTURE, TESTS BACKWARDS COMPATABILITY @patch.object(VolumeService, 'get_cutout', autospec=True) def test_get_cutout_no_cache_True_backwards_compatability(self, fake_volume): """ Test that if no_cache not provided, it defaults to True when calling the volume service's get_cutout(). """ chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16') resolution = 0 x_range = [20, 40] y_range = [50, 70] z_range = [30, 50] self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, no_cache=True) fake_volume.assert_called_with( ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY, CacheMode.no_cache, parallel=True) # This should be the no_cache argument. @patch.object(VolumeService, 'get_cutout', autospec=True) def test_get_cutout_no_cache_False_backwards_compatability(self, fake_volume): """ Test that if no_cache not provided, it defaults to True when calling the volume service's get_cutout(). """ chan = ChannelResource('chan', 'foo', 'bar', 'image', datatype='uint16') resolution = 0 x_range = [20, 40] y_range = [50, 70] z_range = [30, 50] self.remote.get_cutout(chan, resolution, x_range, y_range, z_range, no_cache=False) fake_volume.assert_called_with( ANY, chan, resolution, x_range, y_range, z_range, ANY, ANY, CacheMode.cache, parallel=True) # This should be the no_cache argument.
rmt = BossRemote('/jobs/boss_config.cfg') img_chan = ChannelResource(params['img_channel'], params['collection'], params['experiment'], type='image', datatype='uint8') lbl_chan = ChannelResource(params['lbl_channel'], params['collection'], params['experiment'], type='annotation', datatype='uint64') # Get the image data from the BOSS x_train = rmt.get_cutout(img_chan, params['resolution'], params['x_rng'], params['y_rng'], params['z_rng']) y_train = rmt.get_cutout(lbl_chan, params['resolution'], params['x_rng'], params['y_rng'], params['z_rng']) # Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X]) x_train = x_train[:, np.newaxis, :, :].astype(np.float32) y_train = y_train[:, np.newaxis, :, :].astype(np.float32) # Pixel values must be in [0,1] x_train /= 255. y_train = (y_train > 0).astype('float32') tile_size = tuple(params['tile_size']) train_pct = params['train_pct'] # ------------------------------------------------------------------------- # Data must be [slices, chan, row, col] (i.e., [Z, chan, Y, X])
class NeuroDataResource: def __init__(self, host, token, collection, experiment): self._bossRemote = BossRemote({ 'protocol': 'https', 'host': host, 'token': token }) self.collection = collection self.experiment = experiment self.channels = self._bossRemote.list_channels(collection, experiment) self.channels.remove('empty') #Delete "empty" channel self._get_coord_frame_details() def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print( 'Error: You must supply zRange, yRange, xRange kwargs in list format' ) return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange, zRange) #Datatype check. Recast to original datatype if data is float64 if data.dtype == datatype: return data else: return data.astype(datatype)
"typename": "uint8blk", "dataname": "Luis1", "versioned": "0" })) print(dat1.content) # BOSS Data fetch: boss = BossRemote({ "protocol": "https", "host": "api.theboss.io", "token": "db1cec2c865fc84e48772f4f4a5f010c0a180b88", }) volumeB = boss.get_cutout( boss.get_channel("em", "pinky40", "v7"), 2, [10000, 10100], [10000, 10100], [501, 502], ) volumeB = volumeB.tobytes() print(len(volumeB)) dif = (32 * 32 * 32) - len(volumeB) volumeB = volumeB + str("".join((["0"] * dif))) print(len(volumeB)) # print len(data) import requests
# Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] # Note that the numpy matrix is in Z, Y, X order. data = numpy.random.randint(1, 3000, (5, 4, 8)) data = data.astype(numpy.uint16) # Upload the cutout to the channel. rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data) # Verify that the cutout uploaded correctly. cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, cutout_data) print('Cutout uploaded and verified.') # Get only a small piece of the cutout. small_cutout_data = rmt.get_cutout(chan_actual, 0, [0, 1], [0, 1], [0, 5]) numpy.testing.assert_array_equal(data[0:5, 0:1, 0:1], small_cutout_data) # For times series data, the matrix is in t, Z, Y, X order. time_rng = [0, 3] time_data = numpy.random.randint(1, 3000, (3, 5, 4, 8), numpy.uint16) rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, time_data, time_rng) time_cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng,
class InternTileProcessor(TileProcessor): """A Tile processor for a single image file identified by z index""" def __init__(self): """Constructor to add custom class var""" TileProcessor.__init__(self) self.remote = None self.channel = None def setup(self, parameters): """ Method to load the file for uploading data. Assumes intern token is set via environment variable or config default file Args: parameters (dict): Parameters for the dataset to be processed MUST HAVE THE CUSTOM PARAMETERS: "x_offset": offset to apply when querying the Boss "y_offset": offset to apply when querying the Boss "z_offset": offset to apply when querying the Boss "x_tile": size of a tile in x dimension "y_tile": size of a tile in y dimension "collection": source collection "experiment": source experiment "channel": source channel "resolution": source resolution Returns: None """ self.parameters = parameters self.remote = BossRemote() self.channel = ChannelResource(self.parameters["channel"], self.parameters["collection"], self.parameters["experiment"]) self.channel = self.remote.get_project(self.channel) def process(self, file_path, x_index, y_index, z_index, t_index=0): """ Method to load the image file. Args: file_path(str): An absolute file path for the specified tile x_index(int): The tile index in the X dimension y_index(int): The tile index in the Y dimension z_index(int): The tile index in the Z dimension t_index(int): The time index Returns: (io.BufferedReader): A file handle for the specified tile """ # Compute cutout args x_rng = [ self.parameters["x_tile"] * x_index + self.parameters["x_offset"], self.parameters["x_tile"] * (x_index + 1) + self.parameters["x_offset"] ] y_rng = [ self.parameters["y_tile"] * y_index + self.parameters["y_offset"], self.parameters["y_tile"] * (y_index + 1) + self.parameters["y_offset"] ] z_rng = [ z_index + self.parameters["z_offset"], z_index + 1 + self.parameters["z_offset"] ] if z_index + self.parameters["z_offset"] < 0: data = np.zeros( (self.parameters["x_tile"], self.parameters["y_tile"]), dtype=np.int32, order="C") else: # Get data cnt = 0 while cnt < 5: try: data = self.remote.get_cutout( self.channel, self.parameters["resolution"], x_rng, y_rng, z_rng) data = np.asarray(data, np.uint32) break except Exception as err: if cnt > 5: raise err cnt += 1 time.sleep(10) # Save sub-img to png and return handle upload_img = Image.fromarray(np.squeeze(data)) output = six.BytesIO() upload_img.save(output, format="TIFF") # Send handle back return output
def boss_pull_cutout(args): if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop print("Data model setup.") xmin = np.max([x_min_bound, args.xmin - args.padding]) xmax = np.min([x_max_bound, args.xmax + args.padding]) x_rng = [xmin, xmax] ymin = np.max([y_min_bound, args.ymin - args.padding]) ymax = np.min([y_max_bound, args.ymax + args.padding]) y_rng = [ymin, ymax] zmin = np.max([z_min_bound, args.zmin - args.padding]) zmax = np.min([z_max_bound, args.zmax + args.padding]) z_rng = [zmin, zmax] # Verify that the cutout uploaded correctly. attempts = 0 while attempts < 3: try: cutout_data = rmt.get_cutout(chan_actual, args.res, x_rng, y_rng, z_rng) break except HTTPError as e: if attempts < 3: attempts += 1 print("Obtained HTTP error from server. Trial {}".format(attempts)) else: print("Failed 3 times: {}".format(e)) # Data will be in Z,Y,X format # Change to X,Y,Z for pipeline cutout_data = np.transpose(cutout_data, (2, 1, 0)) def _upload(f): print("Uploading to s3:/{}/{}".format(args.bucket, args.output)) s3 = boto3.resource("s3") f.seek(0, 0) s3.Object(args.bucket, args.output).put(Body=f) # Clean up. if args.bucket and args.s3_only: with tempfile.TemporaryFile() as f: np.save(f, cutout_data) _upload(f) else: with open(args.output, "w+b") as f: np.save(f, cutout_data) if args.bucket: _upload(f)
# Write the mesh obj with open("mesh_22_test.obj", "wb") as fh: fh.write(mesh_obj) # Write the Neuroglancer ready mesh with open("mesh_ng_test", "wb") as fh: fh.write(mesh_ng) ###### Use MeshService from direct import ###### ## No need for remote import. # Define channel resource ann_chan = rmt.get_channel(CHAN, COLL, EXP) # Grab cutout volume volume = rmt.get_cutout(ann_chan, res, x_rng, y_rng, z_rng) # Initialize MeshService mesh_serv = MeshService() # Create mesh mesh = mesh_serv.create(volume, x_rng, y_rng, z_rng) # Convert mesh data to obj mesh_obj = mesh.obj_mesh() # Convert mesh data to precompute format for neuroglancer mesh_ng = mesh.ng_mesh() # Write the mesh obj with open("mesh_22_test_serv.obj", "wb") as fh:
class NeuroDataResource: def __init__(self, host, token, collection, experiment): self._bossRemote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) self.collection = collection self.experiment = experiment self.channels = self._bossRemote.list_channels(collection, experiment) self.channels.remove('empty') #Delete "empty" channel self._get_coord_frame_details() def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print('Error: You must supply zRange, yRange, xRange kwargs in list format') return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange, zRange) return data @staticmethod def ingest_volume(host, token, channel_name, collection, experiment, volume): """ Assumes the collection and experiment exists in BOSS. """ remote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) if volume.dtype == 'uint64': dtype = 'uint64' img_type = 'annotation' sources = ['empty'] else: dtype = volume.dtype.name img_type = 'image' sources = [] try: channel_resource = ChannelResource(channel_name, collection, experiment) channel = remote.get_project(channel_resource) except: channel_resource = ChannelResource(channel_name, collection, experiment, type=img_type, sources=sources, datatype=dtype) channel = remote.create_project(channel_resource) #Get max size of experiment exp_resource = ExperimentResource(experiment, collection) coord_frame = remote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = remote.get_project(coord_frame_resource) y_stop, x_stop = data.y_stop, data.x_stop for z in range(volume.shape[0]): print('Uploading {} slice'.format(z)) remote.create_cutout(channel, 0, (0, x_stop), (0, y_stop), (z, z + 1), volume[z, :, :].reshape((-1, y_stop, x_stop)))