def push_tif(): # message1 = request.form['channel'] #if message1: # print(message1) col = request.form["collection"] exp = request.form["experiment"] channel = request.form["channel"] dtype = request.form["datatype"] host,token = get_host_token() x_range = [int(request.form["xstart"]),int(request.form["xstop"])] y_range = [int(request.form["ystart"]),int(request.form["ystop"])] z_range = [int(request.form["zstart"]),int(request.form["zstop"])] remote = BossRemote('./neurodata.cfg') voxel_size = '1 1 1' voxel_unit = 'micrometers' channel_resource = ChannelResource(channel, col, exp, 'image', '', 0, dtype, 0) project = remote.get_project(channel_resource) my_file = request.form["file"] my_array = np.array(io.imread('./DATA/'+my_file)).astype(np.uint8) for z in range(z_range[0],z_range[1]): remote.create_cutout(channel_resource, 0, (x_range[0],x_range[1]), (y_range[0],y_range[1]), (z,z+1), my_array[z].reshape(-1,my_array.shape[1],my_array.shape[2])) return 'Successfully pushed'
def upload_boss_image(self, img, offset): shape = Vec(*img.shape[:3]) offset = Vec(*offset) bounds = Bbox(offset, shape + offset) if bounds.subvoxel(): raise exceptions.EmptyRequestException( 'Requested less than one pixel of volume. {}'.format(bounds)) x_rng = [bounds.minpt.x, bounds.maxpt.x] y_rng = [bounds.minpt.y, bounds.maxpt.y] z_rng = [bounds.minpt.z, bounds.maxpt.z] layer_type = 'image' if self.layer_type == 'unknown' else self.layer_type chan = ChannelResource( collection_name=self.path.bucket, experiment_name=self.path.dataset, name=self.path.layer, # Channel type=layer_type, datatype=self.dtype, ) if img.shape[3] == 1: img = img.reshape(img.shape[:3]) rmt = BossRemote(boss_credentials) img = img.T img = np.asfortranarray(img.astype(self.dtype)) rmt.create_cutout(chan, self.mip, x_rng, y_rng, z_rng, img)
def ingest_volume(host, token, channel_name, collection, experiment, volume): """ Assumes the collection and experiment exists in BOSS. """ remote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) if volume.dtype == 'uint64': dtype = 'uint64' img_type = 'annotation' sources = ['empty'] else: dtype = volume.dtype.name img_type = 'image' sources = [] try: channel_resource = ChannelResource(channel_name, collection, experiment) channel = remote.get_project(channel_resource) except: channel_resource = ChannelResource(channel_name, collection, experiment, type=img_type, sources=sources, datatype=dtype) channel = remote.create_project(channel_resource) #Get max size of experiment exp_resource = ExperimentResource(experiment, collection) coord_frame = remote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = remote.get_project(coord_frame_resource) y_stop, x_stop = data.y_stop, data.x_stop for z in range(volume.shape[0]): print('Uploading {} slice'.format(z)) remote.create_cutout(channel, 0, (0, x_stop), (0, y_stop), (z, z + 1), volume[z, :, :].reshape((-1, y_stop, x_stop)))
def upload(self, image, offset, mip): shape = Vec(*image.shape[:3]) offset = Vec(*offset) bounds = Bbox(offset, shape + offset) if bounds.subvoxel(): raise exceptions.EmptyRequestException( 'Requested less than one pixel of volume. {}'.format(bounds)) if self.autocrop: image, bounds = autocropfn(self.meta, image, bounds, mip) offset = bounds.minpt check_grid_aligned(self.meta, image, bounds, mip, throw_error=(self.non_aligned_writes == False)) x_rng = [bounds.minpt.x, bounds.maxpt.x] y_rng = [bounds.minpt.y, bounds.maxpt.y] z_rng = [bounds.minpt.z, bounds.maxpt.z] layer_type = 'image' if self.layer_type == 'unknown' else self.meta.layer_type chan = ChannelResource( collection_name=self.meta.path.bucket, experiment_name=self.meta.path.dataset, name=self.meta.path.layer, # Channel type=layer_type, datatype=self.meta.data_type, ) if image.shape[3] == 1: image = image.reshape(image.shape[:3]) rmt = BossRemote(boss_credentials) image = image.T image = np.asfortranarray(image.astype(self.meta.dtype)) rmt.create_cutout(chan, mip, x_rng, y_rng, z_rng, image)
class VolumeServiceTest_v1(unittest.TestCase): """Integration tests of the Boss volume service API. Because setup and teardown involves many REST calls, tests are only divided into tests of the different types of data model resources. All operations are performed within a single test of each resource. """ @classmethod def setUpClass(cls): """Do an initial DB clean up in case something went wrong the last time. If a test failed really badly, the DB might be in a bad state despite attempts to clean up during tearDown(). """ cls.rmt = BossRemote('test.cfg', API_VER) # Turn off SSL cert verification. This is necessary for interacting with # developer instances of the Boss. cls.rmt.project_service.session_send_opts = {'verify': False} cls.rmt.metadata_service.session_send_opts = {'verify': False} cls.rmt.volume_service.session_send_opts = {'verify': False} requests.packages.urllib3.disable_warnings(InsecureRequestWarning) coll_name = 'collection2323{}'.format(random.randint(0, 9999)) cls.coll = CollectionResource(coll_name, 'bar') cf_name = 'BestFrame{}'.format(random.randint(0, 9999)) cls.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.', 0, 2048, 0, 2048, 0, 100, 1, 1, 1, 'nanometers', 0, 'nanoseconds') # cls.exp.coord_frame must be set with valid id before creating. cls.exp = ExperimentResource('exp2323x2', cls.coll.name, cls.coord.name, 'my experiment', 1, 'isotropic', 10) cls.chan = ChannelResource('myVolChan', cls.coll.name, cls.exp.name, 'image', 'test channel', 0, 'uint8', 0) cls.chan16 = ChannelResource('myVol16bitChan', cls.coll.name, cls.exp.name, 'image', '16 bit test channel', 0, 'uint16', 0) cls.ann_chan = ChannelResource('annVolChan2', cls.coll.name, cls.exp.name, 'annotation', 'annotation test channel', 0, 'uint64', 0, sources=[cls.chan.name]) # This channel reserved for testing get_ids_in_region(). This is a # separate channel so we don't have to worry about ids written by # other tests. cls.ann_region_chan = ChannelResource( 'annRegionChan2', cls.coll.name, cls.exp.name, 'annotation', 'annotation ids in region test channel', 0, 'uint64', 0, sources=[cls.chan.name]) # This channel reerved for testing tight bounding boxes. cls.ann_bounding_chan = ChannelResource( 'annRegionChan3', cls.coll.name, cls.exp.name, 'annotation', 'annotation ids in bounding box test channel', 0, 'uint64', 0, sources=[cls.chan.name]) cls.rmt.create_project(cls.coll) cls.rmt.create_project(cls.coord) cls.rmt.create_project(cls.exp) cls.rmt.create_project(cls.chan16) cls.rmt.create_project(cls.chan) cls.rmt.create_project(cls.ann_chan) cls.rmt.create_project(cls.ann_region_chan) cls.rmt.create_project(cls.ann_bounding_chan) @classmethod def tearDownClass(cls): """Clean up the data model objects used by this test case. This method is used by both tearDownClass() and setUpClass(). """ try: cls.rmt.delete_project(cls.ann_bounding_chan) except HTTPError: pass try: cls.rmt.delete_project(cls.ann_region_chan) except HTTPError: pass try: cls.rmt.delete_project(cls.ann_chan) except HTTPError: pass try: cls.rmt.delete_project(cls.chan16) except HTTPError: pass try: cls.rmt.delete_project(cls.chan) except HTTPError: pass try: cls.rmt.delete_project(cls.exp) except HTTPError: pass try: cls.rmt.delete_project(cls.coord) except HTTPError: pass try: cls.rmt.delete_project(cls.coll) except HTTPError: pass def setUp(self): self.rmt = BossRemote('test.cfg') def tearDown(self): pass def test_reserve_ids(self): first_id = self.rmt.reserve_ids(self.ann_chan, 20) self.assertTrue(first_id > 0) def test_get_bounding_box_id_doesnt_exist(self): resolution = 0 id = 12345678 with self.assertRaises(HTTPError) as err: self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') expected_msg_prefix = 'Reserve ids failed' self.assertTrue(err.message.startwswith(expected_msg_prefix)) @unittest.skip('Skipping - currently indexing disabled') def test_get_bounding_box_spans_cuboids_in_x(self): x_rng = [511, 515] y_rng = [0, 8] z_rng = [0, 5] t_rng = [0, 1] id = 77555 data = numpy.zeros((5, 8, 4), dtype='uint64') data[1][0][0] = id data[2][1][1] = id data[3][2][3] = id resolution = 0 self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) expected = { 'x_range': [0, 1024], 'y_range': [0, 512], 'z_range': [0, 16], 't_range': [0, 1] } actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_bounding_box_spans_cuboids_in_y(self): x_rng = [0, 8] y_rng = [511, 515] z_rng = [0, 5] t_rng = [0, 1] id = 77666 data = numpy.zeros((5, 4, 8), dtype='uint64') data[1][0][0] = id data[2][1][0] = id data[3][2][0] = id resolution = 0 self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) expected = { 'x_range': [0, 512], 'y_range': [0, 1024], 'z_range': [0, 16], 't_range': [0, 1] } actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_bounding_box_spans_cuboids_in_z(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [30, 35] t_rng = [0, 1] id = 77888 data = numpy.zeros((5, 4, 8), dtype='uint64') data[1][0][0] = id data[2][1][0] = id data[3][2][0] = id resolution = 0 self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) expected = { 'x_range': [0, 512], 'y_range': [0, 512], 'z_range': [16, 48], 't_range': [0, 1] } actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose') self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_tight_bounding_box_x_axis(self): """Test tight bounding box with ids that span three cuboids along the x axis.""" resolution = 0 x_rng = [511, 1025] y_rng = [512, 1024] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 512, 514), dtype='uint64') x_id = 123 y_id = 127 z_id = 500000000000000000 # Id in partial region on x axis closest to origin. data[1][1][0] = x_id # Id in partial region on x axis furthest from origin. data[1][1][513] = x_id # Id in cuboid aligned region. data[2][2][21] = x_id data[2][1][22] = y_id data[4][24][72] = z_id expected = { 'x_range': [511, 1025], 'y_range': [513, 515], 'z_range': [17, 19] } self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_bounding_box(self.ann_bounding_chan, resolution, x_id, bb_type='tight') @unittest.skip('Skipping - currently indexing disabled') def test_tight_bounding_box_y_axis(self): """Test tight bounding box with ids that span three cuboids along the x axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [511, 1025] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 514, 512), dtype='uint64') x_id = 123 y_id = 127 z_id = 500000000000000000 # Id in partial region on y axis closest to origin. data[1][0][10] = y_id # Id in partial region on y axis furthest from origin. data[1][513][13] = y_id # Id in cuboid aligned region. data[2][2][21] = y_id data[2][3][20] = x_id data[4][25][71] = z_id expected = { 'x_range': [522, 526], 'y_range': [511, 1025], 'z_range': [17, 19] } self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_bounding_box(self.ann_bounding_chan, resolution, y_id, bb_type='tight') @unittest.skip('Skipping - currently indexing disabled') def test_tight_bounding_box_z_axis(self): """Test tight bounding box with ids that span three cuboids along the x axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [512, 1024] z_rng = [15, 33] t_rng = [0, 1] data = numpy.zeros((18, 512, 512), dtype='uint64') x_id = 123 y_id = 127 z_id = 500000000000000000 # Id in partial region on z axis closest to origin. data[0][22][60] = z_id # Id in partial region on z axis furthest from origin. data[17][23][63] = z_id # Id in cuboid aligned region. data[5][24][71] = z_id data[3][2][20] = x_id data[3][1][21] = y_id expected = { 'x_range': [572, 583], 'y_range': [534, 537], 'z_range': [15, 33] } self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_bounding_box(self.ann_bounding_chan, resolution, z_id, bb_type='tight') def test_get_ids_in_region_none(self): """Run on region that hasn't been written with ids, yet.""" resolution = 0 x_rng = [1536, 1540] y_rng = [1536, 1540] z_rng = [48, 56] t_rng = [0, 1] data = numpy.zeros((8, 4, 4), dtype='uint64') expected = [] # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) def test_filtered_cutout(self): """Test filtered cutout using same data written for get_ids_in_region_x_axis.""" resolution = 0 x_rng = [511, 1025] y_rng = [512, 1024] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 512, 514), dtype='uint64') # Id in partial region on x axis closest to origin. data[1][1][0] = 123 # Id in partial region on x axis furthest from origin. data[1][1][513] = 321 # Id in cuboid aligned region. data[10][20][21] = 55555 expected = [123, 321, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Should get back the exact data given in create_cutout(). filtered_data1 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[123, 321, 55555]) numpy.testing.assert_array_equal(data, filtered_data1) # Filter on id 123. expected_data_123 = numpy.zeros((16, 512, 514), dtype='uint64') expected_data_123[1][1][0] = 123 filtered_data_123 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[123]) numpy.testing.assert_array_equal(expected_data_123, filtered_data_123) # Filter on id 321. expected_data_321 = numpy.zeros((16, 512, 514), dtype='uint64') expected_data_321[1][1][513] = 321 filtered_data_321 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[321]) numpy.testing.assert_array_equal(expected_data_321, filtered_data_321) # Filter on ids 123 and 55555. expected_data_123_55555 = numpy.zeros((16, 512, 514), dtype='uint64') expected_data_123_55555[1][1][0] = 123 expected_data_123_55555[10][20][21] = 55555 filtered_data_123_55555 = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, id_list=[123, 55555]) numpy.testing.assert_array_equal(expected_data_123_55555, filtered_data_123_55555) @unittest.skip('Skipping - currently indexing disabled') def test_get_ids_in_region_x_axis(self): """Test using a region that's cuboid aligned except for the x axis.""" resolution = 0 x_rng = [511, 1025] y_rng = [512, 1024] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 512, 514), dtype='uint64') # Id in partial region on x axis closest to origin. data[1][1][0] = 123 # Id in partial region on x axis furthest from origin. data[1][1][513] = 321 # Id in cuboid aligned region. data[10][20][21] = 55555 expected = [123, 321, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_ids_in_region_y_axis(self): """Test using a region that's cuboid aligned except for the y axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [511, 1025] z_rng = [16, 32] t_rng = [0, 1] data = numpy.zeros((16, 514, 512), dtype='uint64') # Id in partial region on y axis closest to origin. data[1][0][1] = 456 # Id in partial region on y axis furthest from origin. data[1][513][1] = 654 # Id in cuboid aligned region. data[10][21][20] = 55555 # expected = [123, 321, 456, 654, 789, 987, 55555] expected = [456, 654, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) @unittest.skip('Skipping - currently indexing disabled') def test_get_ids_in_region_z_axis(self): """Test using a region that's cuboid aligned except for the z axis.""" resolution = 0 x_rng = [512, 1024] y_rng = [512, 1024] z_rng = [15, 33] t_rng = [0, 1] data = numpy.zeros((18, 512, 512), dtype='uint64') # Id in partial region on z axis closest to origin. data[0][1][1] = 789 # Id in partial region on z axis furthest from origin. data[17][1][1] = 987 # Id in cuboid aligned region. data[11][20][20] = 55555 expected = [789, 987, 55555] self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng, data) # Get cutout to make sure data is done writing and indices updated. actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual_data) # Method under test. actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution, x_rng, y_rng, z_rng) self.assertEqual(expected, actual) def test_upload_and_download_to_channel(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] data = numpy.random.randint(1, 254, (5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_to_channel_with_time(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] t_rng = [3, 6] data = numpy.random.randint(1, 254, (3, 5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_subsection_to_channel(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 19] sub_x = [12, 14] sub_y = [7, 10] sub_z = [12, 17] data = numpy.random.randint(1, 10, (9, 5, 10)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan, 0, sub_x, sub_y, sub_z) numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual) def test_upload_to_x_edge_of_channel(self): x_rng = [10, 2048] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_y_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 2048] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_z_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 100] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_x_edge_of_channel(self): x_rng = [10, 2049] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_y_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 2049] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_z_edge_of_channel(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 101] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) def test_upload_and_download_to_channel_16bit(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan16, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_subsection_to_channel_16bit(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 19] sub_x = [12, 14] sub_y = [7, 10] sub_z = [12, 17] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.chan16, 0, sub_x, sub_y, sub_z) numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual) def test_upload_to_x_edge_of_channel_16bit(self): x_rng = [2000, 2048] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_to_y_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [2000, 2048] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_to_z_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 100] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_past_x_edge_of_channel_16bit(self): x_rng = [2000, 2049] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_past_y_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [2000, 2049] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_past_z_edge_of_channel_16bit(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 101] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint16) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data) def test_upload_and_download_to_anno_chan(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_download_subsection_to_anno_chan(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 19] sub_x = [12, 14] sub_y = [7, 10] sub_z = [12, 17] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) actual = self.rmt.get_cutout(self.ann_chan, 0, sub_x, sub_y, sub_z) numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual) def test_upload_to_x_edge_of_anno_chan(self): x_rng = [2000, 2048] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_y_edge_of_anno_chan(self): x_rng = [10, 20] y_rng = [2000, 2048] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_to_z_edge_of_anno_chan(self): x_rng = [10, 100] y_rng = [5, 10] z_rng = [10, 100] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_x_edge_of_anno_chan(self): x_rng = [10, 2049] y_rng = [5, 10] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_y_edge_of_anno_chan(self): x_rng = [10, 991] y_rng = [5, 2049] z_rng = [10, 19] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_past_z_edge_of_anno_chan(self): x_rng = [10, 20] y_rng = [5, 10] z_rng = [10, 101] shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint64) with self.assertRaises(HTTPError): self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data) def test_upload_and_download_to_channel_4D(self): x_rng = [600, 680] y_rng = [600, 640] z_rng = [50, 55] t_rng = [0, 1] shape = (t_rng[1] - t_rng[0], z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0]) data = numpy.random.randint(1, 10, shape) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) numpy.testing.assert_array_equal(data, actual) def test_upload_and_cutout_to_black(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] data = numpy.random.randint(1, 254, (5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) self.rmt.create_cutout_to_black(self.chan, 0, x_rng, y_rng, z_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(numpy.zeros((5, 4, 8)), actual) def test_upload_and_cutout_to_black_with_time(self): x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] t_rng = [3, 6] data = numpy.random.randint(1, 254, (3, 5, 4, 8)) data = data.astype(numpy.uint8) self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) self.rmt.create_cutout_to_black(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng, time_range=t_rng) numpy.testing.assert_array_equal(numpy.zeros((3, 5, 4, 8)), actual) def test_upload_and_cutout_to_black_partial(self): x_rng = [0, 1024] y_rng = [0, 1024] z_rng = [0, 5] x_rng_black = [0, 256] y_rng_black = [0, 512] z_rng_black = [2, 3] data = numpy.random.randint(1, 254, (5, 1024, 1024)) data = data.astype(numpy.uint8) expected = numpy.copy(data) expected[2:3, 0:512, 0:256] = 0 self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data) self.rmt.create_cutout_to_black(self.chan, 0, x_rng_black, y_rng_black, z_rng_black) actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(expected, actual)
coll_data = {'poc': 'Jane Doe'} local.create_metadata(Channel1, coll_data) exp_data = {'weight': '20g', 'diet': 'C2', 'date': '23-May-2016'} local.create_metadata(Collection1, exp_data) chan_new_data = {'weight': '45g', 'date': '23-May-2017'} local.update_metadata(Collection1, chan_new_data) local.delete_metadata(Channel1, ['poc']) CollectionMeta = local.list_metadata(Collection1) print(CollectionMeta) #Data processing can also be done here before re-upload #Local to Boss upload of annotated data chan_setup = get_channel('CHAN_NAME', 'COLL_NAME', 'EXP_NAME', 'image', datatype='uint16') chan = BossRemote.create_project(chan_setup) xspan = [0, 200] yspan = [0, 90] zspan = [0, 20] VolumeLmeta.astype(numpy.uint16) BossRemote.create_cutout(chan, 0, xspan, yspan, zspan, VolumeLMeta)
def boss_merge_xbrain(args): # Verify that the cutout uploaded correctly. def pull_margin_cutout(chan_actual, x_rng, y_rng, z_rng): attempts = 0 while attempts < 3: try: cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng) break except HTTPError as e: if attempts < 3: attempts += 1 print("Obtained HTTP error from server. Trial {}".format(attempts)) else: print("Failed 3 times: {}".format(e)) # Data will be in Z,Y,X format # Change to X,Y,Z for pipeline cutout_data = np.transpose(cutout_data, (2, 1, 0)) return cutout_data templatesize = args.templatesize if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") # data is desired range if args.bucket: s3 = boto3.resource("s3") with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.input, f) f.seek(0, 0) data = np.load(f) with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.centroids, f) f.seek(0, 0) centroids = np.load(f) else: data = np.load(args.input) centroids = np.load(args.centroids) COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop # coordinates of data block in original coordinate frame, before padding x_block = [args.xmin, args.xmax] y_block = [args.ymin, args.ymax] z_block = [args.zmin, args.zmax] # Coordinates of data block with padding in original coordinate frame x_block_pad = [ np.amax([args.xmin - args.padding, x_min_bound]), np.amin([args.xmax + args.padding, x_max_bound]), ] y_block_pad = [ np.amax([args.ymin - args.padding, y_min_bound]), np.amin([args.ymax + args.padding, y_max_bound]), ] z_block_pad = [ np.amax([args.zmin - args.padding, z_min_bound]), np.amin([args.zmax + args.padding, z_max_bound]), ] # Coordinates of core data block in local coordinate frame xstart = np.amin([args.padding, args.xmin - x_min_bound]) xend = np.amax( [data.shape[0] - args.padding, data.shape[0] - (x_max_bound - args.xmax)] ) ystart = np.amin([args.padding, args.ymin - y_min_bound]) yend = np.amax( [data.shape[1] - args.padding, data.shape[1] - (y_max_bound - args.ymax)] ) zstart = np.amin([args.padding, args.zmin - z_min_bound]) zend = np.amax( [data.shape[2] - args.padding, data.shape[2] - (z_max_bound - args.zmax)] ) print("Data model setup.") # Template size to decide which centroids to eliminate # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. if args.onesided: # Only merge on the max side, to prevent duplication of detection # Binarize Map data[np.where(data > 0)] = 1 # Search through centroids # On side of max values, keep anything where centroid is in padded region # On side of min values, remove anything that is partially in padded region (covered by another block) n_centers, _ = centroids.shape # n by 4 bad_inds = [] for i in range(0, n_centers): if centroids[i, 0] < xstart or centroids[i, 0] - templatesize / 2 > xend: bad_inds.append(i) elif centroids[i, 1] < ystart or centroids[i, 1] - templatesize / 2 > yend: bad_inds.append(i) elif centroids[i, 2] < zstart or centroids[i, 2] - templatesize / 2 > zend: bad_inds.append(i) centroids_out = np.delete(centroids, bad_inds, axis=0) # translate into global coordinates from local data block centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin # Eliminate any cells form data which overlap with the padding edge for ind in bad_inds: xi = np.array( [ centroids[ind, 0] - np.ceil(templatesize / 2), centroids[ind, 0] + np.ceil(templatesize / 2), ] ).astype(int) yi = np.array( [ centroids[ind, 1] - np.ceil(templatesize / 2), centroids[ind, 1] + np.ceil(templatesize / 2), ] ).astype(int) zi = np.array( [ centroids[ind, 2] - np.ceil(templatesize / 2), centroids[ind, 2] + np.ceil(templatesize / 2), ] ).astype(int) data[xi, yi, zi] = 0 # Keep any interior cells, any which overlap original boundary and not padding # Pull down existing boundary area, if area is valid # Test side 4 if ( xend < data.shape[0] ): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[1], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 5 if ( yend < data.shape[1] ): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[1], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 6 if ( zend < data.shape[2] ): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[1], z_block_pad[1]], ) data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ], margin, ) # push results over entire padded area # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = data[ xstart : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] data = np.transpose(data, (2, 1, 0)) data = data.copy(order="C").astype(eval("np.{}".format(args.dtype))) # Verify that the cutout uploaded correctly. rmt.create_cutout( chan_actual, 0, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], data, ) # Clean up. else: # Binarize Map data[np.where(data > 0)] = 1 # Search through centroids n_centers, _ = centroids.shape # n by 4 bad_inds = [] for i in range(0, n_centers): if ( centroids[i, 0] + templatesize / 2 < xstart or centroids[i, 0] - templatesize / 2 > xend ): bad_inds.append(i) elif ( centroids[i, 1] + templatesize / 2 < ystart or centroids[i, 1] - templatesize / 2 > yend ): bad_inds.append(i) elif ( centroids[i, 2] + templatesize / 2 < zstart or centroids[i, 2] - templatesize / 2 > zend ): bad_inds.append(i) centroids_out = np.delete(centroids, bad_inds, axis=0) # translate into global coordinates from local data block centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin # Eliminate any cells form data which overlap with the padding edge for ind in bad_inds: xi = np.array( [ centroids[ind, 0] - np.ceil(templatesize / 2), centroids[ind, 0] + np.ceil(templatesize / 2), ] ).astype(int) yi = np.array( [ centroids[ind, 1] - np.ceil(templatesize / 2), centroids[ind, 1] + np.ceil(templatesize / 2), ] ).astype(int) zi = np.array( [ centroids[ind, 2] - np.ceil(templatesize / 2), centroids[ind, 2] + np.ceil(templatesize / 2), ] ).astype(int) data[xi, yi, zi] = 0 # Keep any interior cells, any which overlap original boundary and not padding # Pull down existing boundary area, if area is valid # Test side 1 if ( xstart > 0 ): # There is padding on side 1 of cube [0:pad,0:ymax+2*pad,0:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block_pad[0], x_block[0]], y_block_pad, z_block_pad ) data[0:xstart, :, :] = np.maximum(data[0:xstart, :, :], margin) # Test side 2 if ( ystart > 0 ): # There is padding on side 2 of cube [pad:xmax+2*pad,0:pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block_pad[0], y_block[0]], [z_block[0], z_block_pad[1]], ) data[xstart : data.shape[0], 0:ystart, zstart : data.shape[2]] = np.maximum( data[xstart : data.shape[0], 0:ystart, zstart : data.shape[1]], margin ) # Test side 3 if ( zstart > 0 ): # There is padding on side 3 of cube [pad:xmax+2*pad,pad:ymax+2*pad,0:pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block_pad[0], z_block[0]], ) data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart] = np.maximum( data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart], margin ) # Test side 4 if ( xend < data.shape[0] ): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[1], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 5 if ( yend < data.shape[1] ): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[1], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 6 if ( zend < data.shape[2] ): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[1], z_block_pad[1]], ) data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ], margin, ) # push results over entire padded area # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = np.transpose(data, (2, 1, 0)) data = data.copy(order="C").astype(eval("np.{}".format(args.dtype))) # Verify that the cutout uploaded correctly. rmt.create_cutout(chan_actual, 0, x_block_pad, y_block_pad, z_block_pad, data) # Clean up. def _upload(f): print("Uploading to s3:/{}/{}".format(args.bucket, args.output)) s3 = boto3.resource("s3") f.seek(0, 0) s3.Object(args.bucket, args.output).put(Body=f) # Clean up. if args.bucket and args.s3_only: with tempfile.TemporaryFile() as f: np.save(f, centroids_out) _upload(f) else: print("Saving output") with open(args.output, "w+b") as f: np.save(f, centroids_out) if args.bucket: _upload(f) return
def boss_push_cutout(args): if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") # data is desired range if args.bucket: s3 = boto3.resource("s3") with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.input, f) f.seek(0, 0) data = np.load(f) else: data = np.load(args.input) numpyType = np.uint8 if args.dtype == "uint32": numpyType = np.uint32 elif args.dtype == "uint64": numpyType = np.uint64 if data.dtype != args.dtype: data = data.astype(numpyType) sources = [] if args.source: sources.append(args.source) COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype, sources=sources, ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop print("Data model setup.") # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. data_shape = data.shape # with padding, will be bigger than needed # find data cutoffs to get rid of padding # if nmin = 0, this means that the data wasn't padded on there to begin with xstart = args.padding if args.xmin != 0 else 0 ystart = args.padding if args.ymin != 0 else 0 zstart = args.padding if args.zmin != 0 else 0 xend = data_shape[0] - args.padding yend = data_shape[1] - args.padding zend = data_shape[2] - args.padding # xstart = np.min([args.padding,args.xmin-x_min_bound]) # xend = np.max([data.shape[0]-args.padding,data.shape[0]-(x_max_bound-args.xmax)]) # ystart = np.min([args.padding,args.ymin-y_min_bound]) # yend = np.max([data.shape[1]-args.padding,data.shape[1]-(y_max_bound-args.ymax)]) # zstart = np.min([args.padding,args.zmin-z_min_bound]) # zend = np.max([data.shape[2]-args.padding,data.shape[2]-(z_max_bound-args.zmax)]) # get range which will be uploaded x_rng = [args.xmin, args.xmax] y_rng = [args.ymin, args.ymax] z_rng = [args.zmin, args.zmax] # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = np.transpose(data, (2, 1, 0)) data = data[zstart:zend, ystart:yend, xstart:xend] data = data.copy(order="C") # Verify that the cutout uploaded correctly. attempts = 0 while attempts < 3: try: rmt.create_cutout(chan_actual, args.res, x_rng, y_rng, z_rng, data) break except HTTPError as e: if attempts < 3: attempts += 1 print("These are the dimensions: ") print(data.shape) print("This is the data type:") print(data.dtype) print("Specified data type was:") print(args.dtype) print("Specified image type") print(args.itype) print("Obtained HTTP error from server. Trial {}".format(attempts)) print("The error: {}".format(e)) else: raise Exception("Failed 3 times: {}".format(e))
chan_actual = rmt.create_project(chan_setup) print('Data model setup.') # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] # Note that the numpy matrix is in Z, Y, X order. data = numpy.random.randint(1, 3000, (5, 4, 8)) data = data.astype(numpy.uint16) # Upload the cutout to the channel. rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data) # Verify that the cutout uploaded correctly. cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(data, cutout_data) print('Cutout uploaded and verified.') # Get only a small piece of the cutout. small_cutout_data = rmt.get_cutout(chan_actual, 0, [0, 1], [0, 1], [0, 5]) numpy.testing.assert_array_equal(data[0:5, 0:1, 0:1], small_cutout_data) # For times series data, the matrix is in t, Z, Y, X order. time_rng = [0, 3] time_data = numpy.random.randint(1, 3000, (3, 5, 4, 8), numpy.uint16)
# Reserve a block of 10 ids. start_id = rmt.reserve_ids(ann_chan_actual, 10) # Note that the numpy matrix is in Z, Y, X order. ann_data = numpy.random.randint(start_id, start_id + 10, np_size, dtype='uint64') # Make sure start_id is used at least once. ann_data[0][1][1] = start_id # 0 is native resolution. res = 0 # Upload annotation data to the channel. rmt.create_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng, ann_data) # Verify that the cutout uploaded correctly. ann_cutout_data = rmt.get_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng) numpy.testing.assert_array_equal(ann_data, ann_cutout_data) print('Annotation data uploaded and verified.') # Get ids in the region. ids = rmt.get_ids_in_region(ann_chan_actual, res, x_rng, y_rng, z_rng) print('Ids in region are:') print(ids) # Get the loose bounding box for id start_id. This should be the bounds of the # first cuboid (512, 512, 16).
# Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, 'image', '', datatype='uint16') try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) x_rng = [0, xmax] y_rng = [0, ymax] z_rng = [0, zmax] t_rng = [0, tmax] print('Data model setup.') data = np.random.randint(1, 3000, (tmax, zmax, ymax, xmax)) data = data.astype(np.uint16) # Upload the cutout to the channel. rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data, time_range=t_rng) cutout_data = rmt.get_cutout( chan_actual, 0, x_rng, y_rng, z_rng, time_range=t_rng) np.testing.assert_array_equal(data, cutout_data) print(np.shape(cutout_data)) # (10, 5, 4, 8)
if args.do_synapse: model.load_weights('/src/weights/synapse_weights.hdf5') else: model.load_weights('/src/weights/membrane_weights.hdf5') tic0 = time.time() tic = time.time() y_hat = np.zeros(x_test.shape) for i in range(0, x_test.shape[0], z_step): y_hat[i:i + z_step, ...] = deploy_model(x_test[i:i + z_step, ...], model) print('[info]: total time to deploy: %0.2f sec' % (time.time() - tic)) y_hat = (255. * np.squeeze(y_hat)).astype(np.uint8) print('Total time to process entire volume: {}'.format(time.time() - tic0)) # ---------------------------------------------------------------------- # print('Saving npz file...') # np.savez(args.output_file, y_hat=y_hat, # x_test=(255 * x_test).astype(np.uint8)) # print('Done.') out_chan = ChannelResource(args.out_channel, args.collection, args.experiment, 'annotation', datatype='uint8') rmt.create_cutout(out_chan, args.resolution, args.x_rng, args.y_rng, args.z_rng, y_hat) # ----------------------------------------------------------------------
class RelayStorageManager(StorageManager): """ """ def __init__(self, **kwargs): """ Create a new RelayStorageManager. Arguments: block_size: How much data should go in each file """ self.block_size = kwargs.get("block_size", (256, 256, 16)) if "next_layer" in kwargs: self._next = kwargs["next_layer"] self.is_terminal = False else: self.is_terminal = True if "boss_remote" in kwargs: self.boss_remote = kwargs["boss_remote"] elif "upstream_uri" in kwargs: self.token = kwargs.get("token", "public") self.boss_remote = BossRemote({ "host": kwargs["upstream_uri"], "protocol": kwargs.get("protocol", "http"), "token": kwargs.get("token", "public"), }) def hasdata( self, col: str, exp: str, chan: str, res: int, xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int], ): has_data = self.boss_remote.get_channel(f"bossdb://{col}/{exp}/{chan}") if has_data: return True if not self.is_terminal: return self._next.hasdata(col, exp, chan, res, xs, ys, zs) return False def setdata( self, data: np.array, col: str, exp: str, chan: str, res: int, xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int], ): return self.boss_remote.create_cutout( self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs, data) def getdata( self, col: str, exp: str, chan: str, res: int, xs: Tuple[int, int], ys: Tuple[int, int], zs: Tuple[int, int], ) -> np.array: return self.boss_remote.get_cutout( self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs) def __repr__(self): return f"<RelayStorageManager [BossRemote]>" def get_stack_names(self): if self.is_terminal: return [str(self), f"↳{self.boss_remote}"] else: return [ str(self), f"↳{self.boss_remote}", *self._next.get_stack_names() ]