def push_tif():
# message1 = request.form['channel']
#if message1:
# print(message1)
col = request.form["collection"]
exp = request.form["experiment"]
channel = request.form["channel"]
dtype = request.form["datatype"]
host,token = get_host_token()
x_range = [int(request.form["xstart"]),int(request.form["xstop"])]
y_range = [int(request.form["ystart"]),int(request.form["ystop"])]
z_range = [int(request.form["zstart"]),int(request.form["zstop"])]
remote = BossRemote('./neurodata.cfg')
voxel_size = '1 1 1'
voxel_unit = 'micrometers'
channel_resource = ChannelResource(channel, col, exp, 'image', '', 0, dtype, 0)
project = remote.get_project(channel_resource)
my_file = request.form["file"]
my_array = np.array(io.imread('./DATA/'+my_file)).astype(np.uint8)
for z in range(z_range[0],z_range[1]):
remote.create_cutout(channel_resource, 0, (x_range[0],x_range[1]), (y_range[0],y_range[1]), (z,z+1), my_array[z].reshape(-1,my_array.shape[1],my_array.shape[2]))
return 'Successfully pushed'
def upload_boss_image(self, img, offset):
shape = Vec(*img.shape[:3])
offset = Vec(*offset)
bounds = Bbox(offset, shape + offset)
if bounds.subvoxel():
raise exceptions.EmptyRequestException(
'Requested less than one pixel of volume. {}'.format(bounds))
x_rng = [bounds.minpt.x, bounds.maxpt.x]
y_rng = [bounds.minpt.y, bounds.maxpt.y]
z_rng = [bounds.minpt.z, bounds.maxpt.z]
layer_type = 'image' if self.layer_type == 'unknown' else self.layer_type
chan = ChannelResource(
collection_name=self.path.bucket,
experiment_name=self.path.dataset,
name=self.path.layer, # Channel
type=layer_type,
datatype=self.dtype,
)
if img.shape[3] == 1:
img = img.reshape(img.shape[:3])
rmt = BossRemote(boss_credentials)
img = img.T
img = np.asfortranarray(img.astype(self.dtype))
rmt.create_cutout(chan, self.mip, x_rng, y_rng, z_rng, img)
def ingest_volume(host, token, channel_name, collection, experiment, volume):
"""
Assumes the collection and experiment exists in BOSS.
"""
remote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
if volume.dtype == 'uint64':
dtype = 'uint64'
img_type = 'annotation'
sources = ['empty']
else:
dtype = volume.dtype.name
img_type = 'image'
sources = []
try:
channel_resource = ChannelResource(channel_name, collection, experiment)
channel = remote.get_project(channel_resource)
except:
channel_resource = ChannelResource(channel_name, collection, experiment,
type=img_type,
sources=sources,
datatype=dtype)
channel = remote.create_project(channel_resource)
#Get max size of experiment
exp_resource = ExperimentResource(experiment, collection)
coord_frame = remote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = remote.get_project(coord_frame_resource)
y_stop, x_stop = data.y_stop, data.x_stop
for z in range(volume.shape[0]):
print('Uploading {} slice'.format(z))
remote.create_cutout(channel,
0,
(0, x_stop),
(0, y_stop),
(z, z + 1),
volume[z, :, :].reshape((-1, y_stop, x_stop)))
def upload(self, image, offset, mip):
shape = Vec(*image.shape[:3])
offset = Vec(*offset)
bounds = Bbox(offset, shape + offset)
if bounds.subvoxel():
raise exceptions.EmptyRequestException(
'Requested less than one pixel of volume. {}'.format(bounds))
if self.autocrop:
image, bounds = autocropfn(self.meta, image, bounds, mip)
offset = bounds.minpt
check_grid_aligned(self.meta,
image,
bounds,
mip,
throw_error=(self.non_aligned_writes == False))
x_rng = [bounds.minpt.x, bounds.maxpt.x]
y_rng = [bounds.minpt.y, bounds.maxpt.y]
z_rng = [bounds.minpt.z, bounds.maxpt.z]
layer_type = 'image' if self.layer_type == 'unknown' else self.meta.layer_type
chan = ChannelResource(
collection_name=self.meta.path.bucket,
experiment_name=self.meta.path.dataset,
name=self.meta.path.layer, # Channel
type=layer_type,
datatype=self.meta.data_type,
)
if image.shape[3] == 1:
image = image.reshape(image.shape[:3])
rmt = BossRemote(boss_credentials)
image = image.T
image = np.asfortranarray(image.astype(self.meta.dtype))
rmt.create_cutout(chan, mip, x_rng, y_rng, z_rng, image)
class VolumeServiceTest_v1(unittest.TestCase):
"""Integration tests of the Boss volume service API.
Because setup and teardown involves many REST calls, tests are only
divided into tests of the different types of data model resources. All
operations are performed within a single test of each resource.
"""
@classmethod
def setUpClass(cls):
"""Do an initial DB clean up in case something went wrong the last time.
If a test failed really badly, the DB might be in a bad state despite
attempts to clean up during tearDown().
"""
cls.rmt = BossRemote('test.cfg', API_VER)
# Turn off SSL cert verification. This is necessary for interacting with
# developer instances of the Boss.
cls.rmt.project_service.session_send_opts = {'verify': False}
cls.rmt.metadata_service.session_send_opts = {'verify': False}
cls.rmt.volume_service.session_send_opts = {'verify': False}
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
coll_name = 'collection2323{}'.format(random.randint(0, 9999))
cls.coll = CollectionResource(coll_name, 'bar')
cf_name = 'BestFrame{}'.format(random.randint(0, 9999))
cls.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.',
0, 2048, 0, 2048, 0, 100, 1, 1, 1,
'nanometers', 0, 'nanoseconds')
# cls.exp.coord_frame must be set with valid id before creating.
cls.exp = ExperimentResource('exp2323x2', cls.coll.name,
cls.coord.name, 'my experiment', 1,
'isotropic', 10)
cls.chan = ChannelResource('myVolChan', cls.coll.name, cls.exp.name,
'image', 'test channel', 0, 'uint8', 0)
cls.chan16 = ChannelResource('myVol16bitChan', cls.coll.name,
cls.exp.name, 'image',
'16 bit test channel', 0, 'uint16', 0)
cls.ann_chan = ChannelResource('annVolChan2',
cls.coll.name,
cls.exp.name,
'annotation',
'annotation test channel',
0,
'uint64',
0,
sources=[cls.chan.name])
# This channel reserved for testing get_ids_in_region(). This is a
# separate channel so we don't have to worry about ids written by
# other tests.
cls.ann_region_chan = ChannelResource(
'annRegionChan2',
cls.coll.name,
cls.exp.name,
'annotation',
'annotation ids in region test channel',
0,
'uint64',
0,
sources=[cls.chan.name])
# This channel reerved for testing tight bounding boxes.
cls.ann_bounding_chan = ChannelResource(
'annRegionChan3',
cls.coll.name,
cls.exp.name,
'annotation',
'annotation ids in bounding box test channel',
0,
'uint64',
0,
sources=[cls.chan.name])
cls.rmt.create_project(cls.coll)
cls.rmt.create_project(cls.coord)
cls.rmt.create_project(cls.exp)
cls.rmt.create_project(cls.chan16)
cls.rmt.create_project(cls.chan)
cls.rmt.create_project(cls.ann_chan)
cls.rmt.create_project(cls.ann_region_chan)
cls.rmt.create_project(cls.ann_bounding_chan)
@classmethod
def tearDownClass(cls):
"""Clean up the data model objects used by this test case.
This method is used by both tearDownClass() and setUpClass().
"""
try:
cls.rmt.delete_project(cls.ann_bounding_chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.ann_region_chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.ann_chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.chan16)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.chan)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.exp)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.coord)
except HTTPError:
pass
try:
cls.rmt.delete_project(cls.coll)
except HTTPError:
pass
def setUp(self):
self.rmt = BossRemote('test.cfg')
def tearDown(self):
pass
def test_reserve_ids(self):
first_id = self.rmt.reserve_ids(self.ann_chan, 20)
self.assertTrue(first_id > 0)
def test_get_bounding_box_id_doesnt_exist(self):
resolution = 0
id = 12345678
with self.assertRaises(HTTPError) as err:
self.rmt.get_bounding_box(self.ann_chan, resolution, id, 'loose')
expected_msg_prefix = 'Reserve ids failed'
self.assertTrue(err.message.startwswith(expected_msg_prefix))
@unittest.skip('Skipping - currently indexing disabled')
def test_get_bounding_box_spans_cuboids_in_x(self):
x_rng = [511, 515]
y_rng = [0, 8]
z_rng = [0, 5]
t_rng = [0, 1]
id = 77555
data = numpy.zeros((5, 8, 4), dtype='uint64')
data[1][0][0] = id
data[2][1][1] = id
data[3][2][3] = id
resolution = 0
self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng,
data)
# Get cutout to make sure data is done writing and indices updated.
actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng,
z_rng)
numpy.testing.assert_array_equal(data, actual)
expected = {
'x_range': [0, 1024],
'y_range': [0, 512],
'z_range': [0, 16],
't_range': [0, 1]
}
actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id,
'loose')
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_bounding_box_spans_cuboids_in_y(self):
x_rng = [0, 8]
y_rng = [511, 515]
z_rng = [0, 5]
t_rng = [0, 1]
id = 77666
data = numpy.zeros((5, 4, 8), dtype='uint64')
data[1][0][0] = id
data[2][1][0] = id
data[3][2][0] = id
resolution = 0
self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng,
data)
# Get cutout to make sure data is done writing and indices updated.
actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng,
z_rng)
numpy.testing.assert_array_equal(data, actual)
expected = {
'x_range': [0, 512],
'y_range': [0, 1024],
'z_range': [0, 16],
't_range': [0, 1]
}
actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id,
'loose')
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_bounding_box_spans_cuboids_in_z(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [30, 35]
t_rng = [0, 1]
id = 77888
data = numpy.zeros((5, 4, 8), dtype='uint64')
data[1][0][0] = id
data[2][1][0] = id
data[3][2][0] = id
resolution = 0
self.rmt.create_cutout(self.ann_chan, resolution, x_rng, y_rng, z_rng,
data)
# Get cutout to make sure data is done writing and indices updated.
actual = self.rmt.get_cutout(self.ann_chan, resolution, x_rng, y_rng,
z_rng)
numpy.testing.assert_array_equal(data, actual)
expected = {
'x_range': [0, 512],
'y_range': [0, 512],
'z_range': [16, 48],
't_range': [0, 1]
}
actual = self.rmt.get_bounding_box(self.ann_chan, resolution, id,
'loose')
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_tight_bounding_box_x_axis(self):
"""Test tight bounding box with ids that span three cuboids along the x axis."""
resolution = 0
x_rng = [511, 1025]
y_rng = [512, 1024]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 512, 514), dtype='uint64')
x_id = 123
y_id = 127
z_id = 500000000000000000
# Id in partial region on x axis closest to origin.
data[1][1][0] = x_id
# Id in partial region on x axis furthest from origin.
data[1][1][513] = x_id
# Id in cuboid aligned region.
data[2][2][21] = x_id
data[2][1][22] = y_id
data[4][24][72] = z_id
expected = {
'x_range': [511, 1025],
'y_range': [513, 515],
'z_range': [17, 19]
}
self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng,
y_rng, z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_bounding_box(self.ann_bounding_chan,
resolution,
x_id,
bb_type='tight')
@unittest.skip('Skipping - currently indexing disabled')
def test_tight_bounding_box_y_axis(self):
"""Test tight bounding box with ids that span three cuboids along the x axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [511, 1025]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 514, 512), dtype='uint64')
x_id = 123
y_id = 127
z_id = 500000000000000000
# Id in partial region on y axis closest to origin.
data[1][0][10] = y_id
# Id in partial region on y axis furthest from origin.
data[1][513][13] = y_id
# Id in cuboid aligned region.
data[2][2][21] = y_id
data[2][3][20] = x_id
data[4][25][71] = z_id
expected = {
'x_range': [522, 526],
'y_range': [511, 1025],
'z_range': [17, 19]
}
self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng,
y_rng, z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_bounding_box(self.ann_bounding_chan,
resolution,
y_id,
bb_type='tight')
@unittest.skip('Skipping - currently indexing disabled')
def test_tight_bounding_box_z_axis(self):
"""Test tight bounding box with ids that span three cuboids along the x axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [512, 1024]
z_rng = [15, 33]
t_rng = [0, 1]
data = numpy.zeros((18, 512, 512), dtype='uint64')
x_id = 123
y_id = 127
z_id = 500000000000000000
# Id in partial region on z axis closest to origin.
data[0][22][60] = z_id
# Id in partial region on z axis furthest from origin.
data[17][23][63] = z_id
# Id in cuboid aligned region.
data[5][24][71] = z_id
data[3][2][20] = x_id
data[3][1][21] = y_id
expected = {
'x_range': [572, 583],
'y_range': [534, 537],
'z_range': [15, 33]
}
self.rmt.create_cutout(self.ann_bounding_chan, resolution, x_rng,
y_rng, z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_bounding_box(self.ann_bounding_chan,
resolution,
z_id,
bb_type='tight')
def test_get_ids_in_region_none(self):
"""Run on region that hasn't been written with ids, yet."""
resolution = 0
x_rng = [1536, 1540]
y_rng = [1536, 1540]
z_rng = [48, 56]
t_rng = [0, 1]
data = numpy.zeros((8, 4, 4), dtype='uint64')
expected = []
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_bounding_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
def test_filtered_cutout(self):
"""Test filtered cutout using same data written for get_ids_in_region_x_axis."""
resolution = 0
x_rng = [511, 1025]
y_rng = [512, 1024]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 512, 514), dtype='uint64')
# Id in partial region on x axis closest to origin.
data[1][1][0] = 123
# Id in partial region on x axis furthest from origin.
data[1][1][513] = 321
# Id in cuboid aligned region.
data[10][20][21] = 55555
expected = [123, 321, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Should get back the exact data given in create_cutout().
filtered_data1 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[123, 321, 55555])
numpy.testing.assert_array_equal(data, filtered_data1)
# Filter on id 123.
expected_data_123 = numpy.zeros((16, 512, 514), dtype='uint64')
expected_data_123[1][1][0] = 123
filtered_data_123 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[123])
numpy.testing.assert_array_equal(expected_data_123, filtered_data_123)
# Filter on id 321.
expected_data_321 = numpy.zeros((16, 512, 514), dtype='uint64')
expected_data_321[1][1][513] = 321
filtered_data_321 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[321])
numpy.testing.assert_array_equal(expected_data_321, filtered_data_321)
# Filter on ids 123 and 55555.
expected_data_123_55555 = numpy.zeros((16, 512, 514), dtype='uint64')
expected_data_123_55555[1][1][0] = 123
expected_data_123_55555[10][20][21] = 55555
filtered_data_123_55555 = self.rmt.get_cutout(self.ann_region_chan,
resolution,
x_rng,
y_rng,
z_rng,
id_list=[123, 55555])
numpy.testing.assert_array_equal(expected_data_123_55555,
filtered_data_123_55555)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_ids_in_region_x_axis(self):
"""Test using a region that's cuboid aligned except for the x axis."""
resolution = 0
x_rng = [511, 1025]
y_rng = [512, 1024]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 512, 514), dtype='uint64')
# Id in partial region on x axis closest to origin.
data[1][1][0] = 123
# Id in partial region on x axis furthest from origin.
data[1][1][513] = 321
# Id in cuboid aligned region.
data[10][20][21] = 55555
expected = [123, 321, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_ids_in_region_y_axis(self):
"""Test using a region that's cuboid aligned except for the y axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [511, 1025]
z_rng = [16, 32]
t_rng = [0, 1]
data = numpy.zeros((16, 514, 512), dtype='uint64')
# Id in partial region on y axis closest to origin.
data[1][0][1] = 456
# Id in partial region on y axis furthest from origin.
data[1][513][1] = 654
# Id in cuboid aligned region.
data[10][21][20] = 55555
# expected = [123, 321, 456, 654, 789, 987, 55555]
expected = [456, 654, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
@unittest.skip('Skipping - currently indexing disabled')
def test_get_ids_in_region_z_axis(self):
"""Test using a region that's cuboid aligned except for the z axis."""
resolution = 0
x_rng = [512, 1024]
y_rng = [512, 1024]
z_rng = [15, 33]
t_rng = [0, 1]
data = numpy.zeros((18, 512, 512), dtype='uint64')
# Id in partial region on z axis closest to origin.
data[0][1][1] = 789
# Id in partial region on z axis furthest from origin.
data[17][1][1] = 987
# Id in cuboid aligned region.
data[11][20][20] = 55555
expected = [789, 987, 55555]
self.rmt.create_cutout(self.ann_region_chan, resolution, x_rng, y_rng,
z_rng, data)
# Get cutout to make sure data is done writing and indices updated.
actual_data = self.rmt.get_cutout(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual_data)
# Method under test.
actual = self.rmt.get_ids_in_region(self.ann_region_chan, resolution,
x_rng, y_rng, z_rng)
self.assertEqual(expected, actual)
def test_upload_and_download_to_channel(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
data = numpy.random.randint(1, 254, (5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_to_channel_with_time(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
t_rng = [3, 6]
data = numpy.random.randint(1, 254, (3, 5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
data,
time_range=t_rng)
actual = self.rmt.get_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_subsection_to_channel(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 19]
sub_x = [12, 14]
sub_y = [7, 10]
sub_z = [12, 17]
data = numpy.random.randint(1, 10, (9, 5, 10))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan, 0, sub_x, sub_y, sub_z)
numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual)
def test_upload_to_x_edge_of_channel(self):
x_rng = [10, 2048]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_y_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 2048]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_z_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 100]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_x_edge_of_channel(self):
x_rng = [10, 2049]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_y_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 2049]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_z_edge_of_channel(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 101]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_and_download_to_channel_16bit(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan16, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_subsection_to_channel_16bit(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 19]
sub_x = [12, 14]
sub_y = [7, 10]
sub_z = [12, 17]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.chan16, 0, sub_x, sub_y, sub_z)
numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual)
def test_upload_to_x_edge_of_channel_16bit(self):
x_rng = [2000, 2048]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_y_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [2000, 2048]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_z_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 100]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_x_edge_of_channel_16bit(self):
x_rng = [2000, 2049]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_y_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [2000, 2049]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_z_edge_of_channel_16bit(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 101]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint16)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.chan16, 0, x_rng, y_rng, z_rng, data)
def test_upload_and_download_to_anno_chan(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_download_subsection_to_anno_chan(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 19]
sub_x = [12, 14]
sub_y = [7, 10]
sub_z = [12, 17]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
actual = self.rmt.get_cutout(self.ann_chan, 0, sub_x, sub_y, sub_z)
numpy.testing.assert_array_equal(data[2:7, 2:5, 2:4], actual)
def test_upload_to_x_edge_of_anno_chan(self):
x_rng = [2000, 2048]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_y_edge_of_anno_chan(self):
x_rng = [10, 20]
y_rng = [2000, 2048]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_to_z_edge_of_anno_chan(self):
x_rng = [10, 100]
y_rng = [5, 10]
z_rng = [10, 100]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_x_edge_of_anno_chan(self):
x_rng = [10, 2049]
y_rng = [5, 10]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_y_edge_of_anno_chan(self):
x_rng = [10, 991]
y_rng = [5, 2049]
z_rng = [10, 19]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_past_z_edge_of_anno_chan(self):
x_rng = [10, 20]
y_rng = [5, 10]
z_rng = [10, 101]
shape = (z_rng[1] - z_rng[0], y_rng[1] - y_rng[0], x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint64)
with self.assertRaises(HTTPError):
self.rmt.create_cutout(self.ann_chan, 0, x_rng, y_rng, z_rng, data)
def test_upload_and_download_to_channel_4D(self):
x_rng = [600, 680]
y_rng = [600, 640]
z_rng = [50, 55]
t_rng = [0, 1]
shape = (t_rng[1] - t_rng[0], z_rng[1] - z_rng[0], y_rng[1] - y_rng[0],
x_rng[1] - x_rng[0])
data = numpy.random.randint(1, 10, shape)
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
data,
time_range=t_rng)
actual = self.rmt.get_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
numpy.testing.assert_array_equal(data, actual)
def test_upload_and_cutout_to_black(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
data = numpy.random.randint(1, 254, (5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
self.rmt.create_cutout_to_black(self.chan, 0, x_rng, y_rng, z_rng)
actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(numpy.zeros((5, 4, 8)), actual)
def test_upload_and_cutout_to_black_with_time(self):
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
t_rng = [3, 6]
data = numpy.random.randint(1, 254, (3, 5, 4, 8))
data = data.astype(numpy.uint8)
self.rmt.create_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
data,
time_range=t_rng)
self.rmt.create_cutout_to_black(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
actual = self.rmt.get_cutout(self.chan,
0,
x_rng,
y_rng,
z_rng,
time_range=t_rng)
numpy.testing.assert_array_equal(numpy.zeros((3, 5, 4, 8)), actual)
def test_upload_and_cutout_to_black_partial(self):
x_rng = [0, 1024]
y_rng = [0, 1024]
z_rng = [0, 5]
x_rng_black = [0, 256]
y_rng_black = [0, 512]
z_rng_black = [2, 3]
data = numpy.random.randint(1, 254, (5, 1024, 1024))
data = data.astype(numpy.uint8)
expected = numpy.copy(data)
expected[2:3, 0:512, 0:256] = 0
self.rmt.create_cutout(self.chan, 0, x_rng, y_rng, z_rng, data)
self.rmt.create_cutout_to_black(self.chan, 0, x_rng_black, y_rng_black,
z_rng_black)
actual = self.rmt.get_cutout(self.chan, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(expected, actual)
coll_data = {'poc': 'Jane Doe'}
local.create_metadata(Channel1, coll_data)
exp_data = {'weight': '20g', 'diet': 'C2', 'date': '23-May-2016'}
local.create_metadata(Collection1, exp_data)
chan_new_data = {'weight': '45g', 'date': '23-May-2017'}
local.update_metadata(Collection1, chan_new_data)
local.delete_metadata(Channel1, ['poc'])
CollectionMeta = local.list_metadata(Collection1)
print(CollectionMeta)
#Data processing can also be done here before re-upload
#Local to Boss upload of annotated data
chan_setup = get_channel('CHAN_NAME',
'COLL_NAME',
'EXP_NAME',
'image',
datatype='uint16')
chan = BossRemote.create_project(chan_setup)
xspan = [0, 200]
yspan = [0, 90]
zspan = [0, 20]
VolumeLmeta.astype(numpy.uint16)
BossRemote.create_cutout(chan, 0, xspan, yspan, zspan, VolumeLMeta)
def boss_merge_xbrain(args):
# Verify that the cutout uploaded correctly.
def pull_margin_cutout(chan_actual, x_rng, y_rng, z_rng):
attempts = 0
while attempts < 3:
try:
cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng)
break
except HTTPError as e:
if attempts < 3:
attempts += 1
print("Obtained HTTP error from server. Trial {}".format(attempts))
else:
print("Failed 3 times: {}".format(e))
# Data will be in Z,Y,X format
# Change to X,Y,Z for pipeline
cutout_data = np.transpose(cutout_data, (2, 1, 0))
return cutout_data
templatesize = args.templatesize
if args.config:
rmt = BossRemote(args.config)
else:
cfg = _generate_config(args.token, args)
with open("intern.cfg", "w") as f:
cfg.write(f)
rmt = BossRemote("intern.cfg")
# data is desired range
if args.bucket:
s3 = boto3.resource("s3")
with tempfile.TemporaryFile() as f:
s3.Bucket(args.bucket).download_fileobj(args.input, f)
f.seek(0, 0)
data = np.load(f)
with tempfile.TemporaryFile() as f:
s3.Bucket(args.bucket).download_fileobj(args.centroids, f)
f.seek(0, 0)
centroids = np.load(f)
else:
data = np.load(args.input)
centroids = np.load(args.centroids)
COLL_NAME = args.coll
EXP_NAME = args.exp
CHAN_NAME = args.chan
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype
)
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
# get coordinate frame to determine padding bounds
cfr = CoordinateFrameResource(args.coord)
cfr_actual = rmt.get_project(cfr)
x_min_bound = cfr_actual.x_start
x_max_bound = cfr_actual.x_stop
y_min_bound = cfr_actual.y_start
y_max_bound = cfr_actual.y_stop
z_min_bound = cfr_actual.z_start
z_max_bound = cfr_actual.z_stop
# coordinates of data block in original coordinate frame, before padding
x_block = [args.xmin, args.xmax]
y_block = [args.ymin, args.ymax]
z_block = [args.zmin, args.zmax]
# Coordinates of data block with padding in original coordinate frame
x_block_pad = [
np.amax([args.xmin - args.padding, x_min_bound]),
np.amin([args.xmax + args.padding, x_max_bound]),
]
y_block_pad = [
np.amax([args.ymin - args.padding, y_min_bound]),
np.amin([args.ymax + args.padding, y_max_bound]),
]
z_block_pad = [
np.amax([args.zmin - args.padding, z_min_bound]),
np.amin([args.zmax + args.padding, z_max_bound]),
]
# Coordinates of core data block in local coordinate frame
xstart = np.amin([args.padding, args.xmin - x_min_bound])
xend = np.amax(
[data.shape[0] - args.padding, data.shape[0] - (x_max_bound - args.xmax)]
)
ystart = np.amin([args.padding, args.ymin - y_min_bound])
yend = np.amax(
[data.shape[1] - args.padding, data.shape[1] - (y_max_bound - args.ymax)]
)
zstart = np.amin([args.padding, args.zmin - z_min_bound])
zend = np.amax(
[data.shape[2] - args.padding, data.shape[2] - (z_max_bound - args.zmax)]
)
print("Data model setup.")
# Template size to decide which centroids to eliminate
# Ranges use the Python convention where the number after the : is the stop
# value. Thus, x_rng specifies x values where: 0 <= x < 8.
if args.onesided:
# Only merge on the max side, to prevent duplication of detection
# Binarize Map
data[np.where(data > 0)] = 1
# Search through centroids
# On side of max values, keep anything where centroid is in padded region
# On side of min values, remove anything that is partially in padded region (covered by another block)
n_centers, _ = centroids.shape # n by 4
bad_inds = []
for i in range(0, n_centers):
if centroids[i, 0] < xstart or centroids[i, 0] - templatesize / 2 > xend:
bad_inds.append(i)
elif centroids[i, 1] < ystart or centroids[i, 1] - templatesize / 2 > yend:
bad_inds.append(i)
elif centroids[i, 2] < zstart or centroids[i, 2] - templatesize / 2 > zend:
bad_inds.append(i)
centroids_out = np.delete(centroids, bad_inds, axis=0)
# translate into global coordinates from local data block
centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin
centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin
centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin
# Eliminate any cells form data which overlap with the padding edge
for ind in bad_inds:
xi = np.array(
[
centroids[ind, 0] - np.ceil(templatesize / 2),
centroids[ind, 0] + np.ceil(templatesize / 2),
]
).astype(int)
yi = np.array(
[
centroids[ind, 1] - np.ceil(templatesize / 2),
centroids[ind, 1] + np.ceil(templatesize / 2),
]
).astype(int)
zi = np.array(
[
centroids[ind, 2] - np.ceil(templatesize / 2),
centroids[ind, 2] + np.ceil(templatesize / 2),
]
).astype(int)
data[xi, yi, zi] = 0
# Keep any interior cells, any which overlap original boundary and not padding
# Pull down existing boundary area, if area is valid
# Test side 4
if (
xend < data.shape[0]
): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[1], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 5
if (
yend < data.shape[1]
): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[1], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 6
if (
zend < data.shape[2]
): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[1], z_block_pad[1]],
)
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
],
margin,
)
# push results over entire padded area
# Pipeline Data will be in X,Y,Z format
# Change to Z,Y,X for upload
data = data[
xstart : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
]
data = np.transpose(data, (2, 1, 0))
data = data.copy(order="C").astype(eval("np.{}".format(args.dtype)))
# Verify that the cutout uploaded correctly.
rmt.create_cutout(
chan_actual,
0,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
data,
)
# Clean up.
else:
# Binarize Map
data[np.where(data > 0)] = 1
# Search through centroids
n_centers, _ = centroids.shape # n by 4
bad_inds = []
for i in range(0, n_centers):
if (
centroids[i, 0] + templatesize / 2 < xstart
or centroids[i, 0] - templatesize / 2 > xend
):
bad_inds.append(i)
elif (
centroids[i, 1] + templatesize / 2 < ystart
or centroids[i, 1] - templatesize / 2 > yend
):
bad_inds.append(i)
elif (
centroids[i, 2] + templatesize / 2 < zstart
or centroids[i, 2] - templatesize / 2 > zend
):
bad_inds.append(i)
centroids_out = np.delete(centroids, bad_inds, axis=0)
# translate into global coordinates from local data block
centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin
centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin
centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin
# Eliminate any cells form data which overlap with the padding edge
for ind in bad_inds:
xi = np.array(
[
centroids[ind, 0] - np.ceil(templatesize / 2),
centroids[ind, 0] + np.ceil(templatesize / 2),
]
).astype(int)
yi = np.array(
[
centroids[ind, 1] - np.ceil(templatesize / 2),
centroids[ind, 1] + np.ceil(templatesize / 2),
]
).astype(int)
zi = np.array(
[
centroids[ind, 2] - np.ceil(templatesize / 2),
centroids[ind, 2] + np.ceil(templatesize / 2),
]
).astype(int)
data[xi, yi, zi] = 0
# Keep any interior cells, any which overlap original boundary and not padding
# Pull down existing boundary area, if area is valid
# Test side 1
if (
xstart > 0
): # There is padding on side 1 of cube [0:pad,0:ymax+2*pad,0:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual, [x_block_pad[0], x_block[0]], y_block_pad, z_block_pad
)
data[0:xstart, :, :] = np.maximum(data[0:xstart, :, :], margin)
# Test side 2
if (
ystart > 0
): # There is padding on side 2 of cube [pad:xmax+2*pad,0:pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block_pad[0], y_block[0]],
[z_block[0], z_block_pad[1]],
)
data[xstart : data.shape[0], 0:ystart, zstart : data.shape[2]] = np.maximum(
data[xstart : data.shape[0], 0:ystart, zstart : data.shape[1]], margin
)
# Test side 3
if (
zstart > 0
): # There is padding on side 3 of cube [pad:xmax+2*pad,pad:ymax+2*pad,0:pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block_pad[0], z_block[0]],
)
data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart] = np.maximum(
data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart], margin
)
# Test side 4
if (
xend < data.shape[0]
): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[1], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 5
if (
yend < data.shape[1]
): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[1], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 6
if (
zend < data.shape[2]
): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[1], z_block_pad[1]],
)
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
],
margin,
)
# push results over entire padded area
# Pipeline Data will be in X,Y,Z format
# Change to Z,Y,X for upload
data = np.transpose(data, (2, 1, 0))
data = data.copy(order="C").astype(eval("np.{}".format(args.dtype)))
# Verify that the cutout uploaded correctly.
rmt.create_cutout(chan_actual, 0, x_block_pad, y_block_pad, z_block_pad, data)
# Clean up.
def _upload(f):
print("Uploading to s3:/{}/{}".format(args.bucket, args.output))
s3 = boto3.resource("s3")
f.seek(0, 0)
s3.Object(args.bucket, args.output).put(Body=f)
# Clean up.
if args.bucket and args.s3_only:
with tempfile.TemporaryFile() as f:
np.save(f, centroids_out)
_upload(f)
else:
print("Saving output")
with open(args.output, "w+b") as f:
np.save(f, centroids_out)
if args.bucket:
_upload(f)
return
def boss_push_cutout(args):
if args.config:
rmt = BossRemote(args.config)
else:
cfg = _generate_config(args.token, args)
with open("intern.cfg", "w") as f:
cfg.write(f)
rmt = BossRemote("intern.cfg")
# data is desired range
if args.bucket:
s3 = boto3.resource("s3")
with tempfile.TemporaryFile() as f:
s3.Bucket(args.bucket).download_fileobj(args.input, f)
f.seek(0, 0)
data = np.load(f)
else:
data = np.load(args.input)
numpyType = np.uint8
if args.dtype == "uint32":
numpyType = np.uint32
elif args.dtype == "uint64":
numpyType = np.uint64
if data.dtype != args.dtype:
data = data.astype(numpyType)
sources = []
if args.source:
sources.append(args.source)
COLL_NAME = args.coll
EXP_NAME = args.exp
CHAN_NAME = args.chan
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME,
COLL_NAME,
EXP_NAME,
type=args.itype,
datatype=args.dtype,
sources=sources,
)
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
# get coordinate frame to determine padding bounds
cfr = CoordinateFrameResource(args.coord)
cfr_actual = rmt.get_project(cfr)
x_min_bound = cfr_actual.x_start
x_max_bound = cfr_actual.x_stop
y_min_bound = cfr_actual.y_start
y_max_bound = cfr_actual.y_stop
z_min_bound = cfr_actual.z_start
z_max_bound = cfr_actual.z_stop
print("Data model setup.")
# Ranges use the Python convention where the number after the : is the stop
# value. Thus, x_rng specifies x values where: 0 <= x < 8.
data_shape = data.shape # with padding, will be bigger than needed
# find data cutoffs to get rid of padding
# if nmin = 0, this means that the data wasn't padded on there to begin with
xstart = args.padding if args.xmin != 0 else 0
ystart = args.padding if args.ymin != 0 else 0
zstart = args.padding if args.zmin != 0 else 0
xend = data_shape[0] - args.padding
yend = data_shape[1] - args.padding
zend = data_shape[2] - args.padding
# xstart = np.min([args.padding,args.xmin-x_min_bound])
# xend = np.max([data.shape[0]-args.padding,data.shape[0]-(x_max_bound-args.xmax)])
# ystart = np.min([args.padding,args.ymin-y_min_bound])
# yend = np.max([data.shape[1]-args.padding,data.shape[1]-(y_max_bound-args.ymax)])
# zstart = np.min([args.padding,args.zmin-z_min_bound])
# zend = np.max([data.shape[2]-args.padding,data.shape[2]-(z_max_bound-args.zmax)])
# get range which will be uploaded
x_rng = [args.xmin, args.xmax]
y_rng = [args.ymin, args.ymax]
z_rng = [args.zmin, args.zmax]
# Pipeline Data will be in X,Y,Z format
# Change to Z,Y,X for upload
data = np.transpose(data, (2, 1, 0))
data = data[zstart:zend, ystart:yend, xstart:xend]
data = data.copy(order="C")
# Verify that the cutout uploaded correctly.
attempts = 0
while attempts < 3:
try:
rmt.create_cutout(chan_actual, args.res, x_rng, y_rng, z_rng, data)
break
except HTTPError as e:
if attempts < 3:
attempts += 1
print("These are the dimensions: ")
print(data.shape)
print("This is the data type:")
print(data.dtype)
print("Specified data type was:")
print(args.dtype)
print("Specified image type")
print(args.itype)
print("Obtained HTTP error from server. Trial {}".format(attempts))
print("The error: {}".format(e))
else:
raise Exception("Failed 3 times: {}".format(e))
chan_actual = rmt.create_project(chan_setup)
print('Data model setup.')
# Ranges use the Python convention where the number after the : is the stop
# value. Thus, x_rng specifies x values where: 0 <= x < 8.
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
# Note that the numpy matrix is in Z, Y, X order.
data = numpy.random.randint(1, 3000, (5, 4, 8))
data = data.astype(numpy.uint16)
# Upload the cutout to the channel.
rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data)
# Verify that the cutout uploaded correctly.
cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(data, cutout_data)
print('Cutout uploaded and verified.')
# Get only a small piece of the cutout.
small_cutout_data = rmt.get_cutout(chan_actual, 0, [0, 1], [0, 1], [0, 5])
numpy.testing.assert_array_equal(data[0:5, 0:1, 0:1], small_cutout_data)
# For times series data, the matrix is in t, Z, Y, X order.
time_rng = [0, 3]
time_data = numpy.random.randint(1, 3000, (3, 5, 4, 8), numpy.uint16)
# Reserve a block of 10 ids.
start_id = rmt.reserve_ids(ann_chan_actual, 10)
# Note that the numpy matrix is in Z, Y, X order.
ann_data = numpy.random.randint(start_id,
start_id + 10,
np_size,
dtype='uint64')
# Make sure start_id is used at least once.
ann_data[0][1][1] = start_id
# 0 is native resolution.
res = 0
# Upload annotation data to the channel.
rmt.create_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng, ann_data)
# Verify that the cutout uploaded correctly.
ann_cutout_data = rmt.get_cutout(ann_chan_actual, res, x_rng, y_rng, z_rng)
numpy.testing.assert_array_equal(ann_data, ann_cutout_data)
print('Annotation data uploaded and verified.')
# Get ids in the region.
ids = rmt.get_ids_in_region(ann_chan_actual, res, x_rng, y_rng, z_rng)
print('Ids in region are:')
print(ids)
# Get the loose bounding box for id start_id. This should be the bounds of the
# first cuboid (512, 512, 16).
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME, COLL_NAME, EXP_NAME, 'image', '', datatype='uint16')
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
x_rng = [0, xmax]
y_rng = [0, ymax]
z_rng = [0, zmax]
t_rng = [0, tmax]
print('Data model setup.')
data = np.random.randint(1, 3000, (tmax, zmax, ymax, xmax))
data = data.astype(np.uint16)
# Upload the cutout to the channel.
rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data,
time_range=t_rng)
cutout_data = rmt.get_cutout(
chan_actual, 0, x_rng, y_rng, z_rng, time_range=t_rng)
np.testing.assert_array_equal(data, cutout_data)
print(np.shape(cutout_data))
# (10, 5, 4, 8)
if args.do_synapse:
model.load_weights('/src/weights/synapse_weights.hdf5')
else:
model.load_weights('/src/weights/membrane_weights.hdf5')
tic0 = time.time()
tic = time.time()
y_hat = np.zeros(x_test.shape)
for i in range(0, x_test.shape[0], z_step):
y_hat[i:i + z_step, ...] = deploy_model(x_test[i:i + z_step, ...],
model)
print('[info]: total time to deploy: %0.2f sec' % (time.time() - tic))
y_hat = (255. * np.squeeze(y_hat)).astype(np.uint8)
print('Total time to process entire volume: {}'.format(time.time() - tic0))
# ----------------------------------------------------------------------
# print('Saving npz file...')
# np.savez(args.output_file, y_hat=y_hat,
# x_test=(255 * x_test).astype(np.uint8))
# print('Done.')
out_chan = ChannelResource(args.out_channel,
args.collection,
args.experiment,
'annotation',
datatype='uint8')
rmt.create_cutout(out_chan, args.resolution, args.x_rng, args.y_rng,
args.z_rng, y_hat)
# ----------------------------------------------------------------------
class RelayStorageManager(StorageManager):
"""
"""
def __init__(self, **kwargs):
"""
Create a new RelayStorageManager.
Arguments:
block_size: How much data should go in each file
"""
self.block_size = kwargs.get("block_size", (256, 256, 16))
if "next_layer" in kwargs:
self._next = kwargs["next_layer"]
self.is_terminal = False
else:
self.is_terminal = True
if "boss_remote" in kwargs:
self.boss_remote = kwargs["boss_remote"]
elif "upstream_uri" in kwargs:
self.token = kwargs.get("token", "public")
self.boss_remote = BossRemote({
"host":
kwargs["upstream_uri"],
"protocol":
kwargs.get("protocol", "http"),
"token":
kwargs.get("token", "public"),
})
def hasdata(
self,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
):
has_data = self.boss_remote.get_channel(f"bossdb://{col}/{exp}/{chan}")
if has_data:
return True
if not self.is_terminal:
return self._next.hasdata(col, exp, chan, res, xs, ys, zs)
return False
def setdata(
self,
data: np.array,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
):
return self.boss_remote.create_cutout(
self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs,
data)
def getdata(
self,
col: str,
exp: str,
chan: str,
res: int,
xs: Tuple[int, int],
ys: Tuple[int, int],
zs: Tuple[int, int],
) -> np.array:
return self.boss_remote.get_cutout(
self.boss_remote.get_channel(chan, col, exp), res, xs, ys, zs)
def __repr__(self):
return f"<RelayStorageManager [BossRemote]>"
def get_stack_names(self):
if self.is_terminal:
return [str(self), f"↳{self.boss_remote}"]
else:
return [
str(self), f"↳{self.boss_remote}",
*self._next.get_stack_names()
]
