def fetch_info(self):
experiment = ExperimentResource(name=self.path.dataset,
collection_name=self.path.bucket)
rmt = BossRemote(boss_credentials)
experiment = rmt.get_project(experiment)
coord_frame = CoordinateFrameResource(name=experiment.coord_frame)
coord_frame = rmt.get_project(coord_frame)
channel = ChannelResource(self.path.layer, self.path.bucket,
self.path.dataset)
channel = rmt.get_project(channel)
unit_factors = {
'nanometers': 1,
'micrometers': 1e3,
'millimeters': 1e6,
'centimeters': 1e7,
}
unit_factor = unit_factors[coord_frame.voxel_unit]
cf = coord_frame
resolution = [cf.x_voxel_size, cf.y_voxel_size, cf.z_voxel_size]
resolution = [int(round(_)) * unit_factor for _ in resolution]
bbox = Bbox((cf.x_start, cf.y_start, cf.z_start),
(cf.x_stop, cf.y_stop, cf.z_stop))
bbox.maxpt = bbox.maxpt
layer_type = 'unknown'
if 'type' in channel.raw:
layer_type = channel.raw['type']
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=layer_type,
data_type=channel.datatype,
encoding='raw',
resolution=resolution,
voxel_offset=bbox.minpt,
volume_size=bbox.size3(),
chunk_size=(512, 512, 16), # fixed in s3 implementation
)
each_factor = Vec(2, 2, 1)
if experiment.hierarchy_method == 'isotropic':
each_factor = Vec(2, 2, 2)
factor = each_factor.clone()
for _ in range(1, experiment.num_hierarchy_levels):
self.add_scale(factor, info=info)
factor *= each_factor
return info
def push_tif():
# message1 = request.form['channel']
#if message1:
# print(message1)
col = request.form["collection"]
exp = request.form["experiment"]
channel = request.form["channel"]
dtype = request.form["datatype"]
host,token = get_host_token()
x_range = [int(request.form["xstart"]),int(request.form["xstop"])]
y_range = [int(request.form["ystart"]),int(request.form["ystop"])]
z_range = [int(request.form["zstart"]),int(request.form["zstop"])]
remote = BossRemote('./neurodata.cfg')
voxel_size = '1 1 1'
voxel_unit = 'micrometers'
channel_resource = ChannelResource(channel, col, exp, 'image', '', 0, dtype, 0)
project = remote.get_project(channel_resource)
my_file = request.form["file"]
my_array = np.array(io.imread('./DATA/'+my_file)).astype(np.uint8)
for z in range(z_range[0],z_range[1]):
remote.create_cutout(channel_resource, 0, (x_range[0],x_range[1]), (y_range[0],y_range[1]), (z,z+1), my_array[z].reshape(-1,my_array.shape[1],my_array.shape[2]))
return 'Successfully pushed'
def delete_channel(remote: BossRemote, channel, coll_name=None, exp_name=None):
chan_obj = None
if isinstance(channel, ChannelResource):
chan_obj = channel
elif isinstance(channel, str) and coll_name is not None and exp_name is not None:
chan_name = channel
chan_obj = remote.get_project(ChannelResource(name=chan_name, experiment_name=exp_name))
if chan_obj is not None:
print('Deleting channel "{0}"...'.format(chan_obj.name))
remote.delete_project(chan_obj)
def ingest_volume(host, token, channel_name, collection, experiment, volume):
"""
Assumes the collection and experiment exists in BOSS.
"""
remote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
if volume.dtype == 'uint64':
dtype = 'uint64'
img_type = 'annotation'
sources = ['empty']
else:
dtype = volume.dtype.name
img_type = 'image'
sources = []
try:
channel_resource = ChannelResource(channel_name, collection, experiment)
channel = remote.get_project(channel_resource)
except:
channel_resource = ChannelResource(channel_name, collection, experiment,
type=img_type,
sources=sources,
datatype=dtype)
channel = remote.create_project(channel_resource)
#Get max size of experiment
exp_resource = ExperimentResource(experiment, collection)
coord_frame = remote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = remote.get_project(coord_frame_resource)
y_stop, x_stop = data.y_stop, data.x_stop
for z in range(volume.shape[0]):
print('Uploading {} slice'.format(z))
remote.create_cutout(channel,
0,
(0, x_stop),
(0, y_stop),
(z, z + 1),
volume[z, :, :].reshape((-1, y_stop, x_stop)))
def delete_coord_frame(remote: BossRemote, coord_frame):
frame_obj = None
# frame_name = None
if isinstance(coord_frame, CoordinateFrameResource):
frame_obj = coord_frame
# frame_name = coord_frame.name
elif isinstance(coord_frame, str):
frame_name = coord_frame
frame_obj = remote.get_project(CoordinateFrameResource(name=frame_name))
if frame_obj is not None:
print('Deleting coordinate frame "{0}"...'.format(frame_obj.name))
remote.delete_project(frame_obj)
def delete_collection(remote: BossRemote, collection):
coll_obj = None
coll_name = None
if isinstance(collection, CollectionResource):
coll_obj = collection
coll_name = collection.name
elif isinstance(collection, str):
coll_name = collection
coll_obj = remote.get_project(CollectionResource(name=coll_name))
if coll_name is not None:
print('Deleting experiments of collection "{0}"...'.format(coll_name))
exp_names = remote.list_experiments(coll_name)
for n in exp_names:
delete_experiment(remote, n, coll_name)
if coll_obj is not None:
print('Deleting collection "{0}"...'.format(coll_name))
remote.delete_project(coll_obj)
def delete_experiment(remote: BossRemote, experiment, coll_name=None):
exp_obj = None
exp_name = None
if isinstance(experiment, ExperimentResource):
exp_obj = experiment
exp_name = experiment.name
coll_name = experiment.coll_name
elif isinstance(experiment, str) and coll_name is not None:
exp_name = experiment
exp_obj = remote.get_project(ExperimentResource(name=exp_name, collection_name=coll_name))
if exp_name is not None:
print('Deleting channels of experiment "{0}"...'.format(exp_name))
chan_names = remote.list_channels(coll_name, exp_name)
for n in chan_names:
delete_channel(remote, n, exp_name, coll_name)
if exp_obj is not None:
print('Deleting experiment "{0}"...'.format(exp_obj.name))
remote.delete_project(exp_obj)
def boss_merge_xbrain(args):
# Verify that the cutout uploaded correctly.
def pull_margin_cutout(chan_actual, x_rng, y_rng, z_rng):
attempts = 0
while attempts < 3:
try:
cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng)
break
except HTTPError as e:
if attempts < 3:
attempts += 1
print("Obtained HTTP error from server. Trial {}".format(attempts))
else:
print("Failed 3 times: {}".format(e))
# Data will be in Z,Y,X format
# Change to X,Y,Z for pipeline
cutout_data = np.transpose(cutout_data, (2, 1, 0))
return cutout_data
templatesize = args.templatesize
if args.config:
rmt = BossRemote(args.config)
else:
cfg = _generate_config(args.token, args)
with open("intern.cfg", "w") as f:
cfg.write(f)
rmt = BossRemote("intern.cfg")
# data is desired range
if args.bucket:
s3 = boto3.resource("s3")
with tempfile.TemporaryFile() as f:
s3.Bucket(args.bucket).download_fileobj(args.input, f)
f.seek(0, 0)
data = np.load(f)
with tempfile.TemporaryFile() as f:
s3.Bucket(args.bucket).download_fileobj(args.centroids, f)
f.seek(0, 0)
centroids = np.load(f)
else:
data = np.load(args.input)
centroids = np.load(args.centroids)
COLL_NAME = args.coll
EXP_NAME = args.exp
CHAN_NAME = args.chan
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype
)
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
# get coordinate frame to determine padding bounds
cfr = CoordinateFrameResource(args.coord)
cfr_actual = rmt.get_project(cfr)
x_min_bound = cfr_actual.x_start
x_max_bound = cfr_actual.x_stop
y_min_bound = cfr_actual.y_start
y_max_bound = cfr_actual.y_stop
z_min_bound = cfr_actual.z_start
z_max_bound = cfr_actual.z_stop
# coordinates of data block in original coordinate frame, before padding
x_block = [args.xmin, args.xmax]
y_block = [args.ymin, args.ymax]
z_block = [args.zmin, args.zmax]
# Coordinates of data block with padding in original coordinate frame
x_block_pad = [
np.amax([args.xmin - args.padding, x_min_bound]),
np.amin([args.xmax + args.padding, x_max_bound]),
]
y_block_pad = [
np.amax([args.ymin - args.padding, y_min_bound]),
np.amin([args.ymax + args.padding, y_max_bound]),
]
z_block_pad = [
np.amax([args.zmin - args.padding, z_min_bound]),
np.amin([args.zmax + args.padding, z_max_bound]),
]
# Coordinates of core data block in local coordinate frame
xstart = np.amin([args.padding, args.xmin - x_min_bound])
xend = np.amax(
[data.shape[0] - args.padding, data.shape[0] - (x_max_bound - args.xmax)]
)
ystart = np.amin([args.padding, args.ymin - y_min_bound])
yend = np.amax(
[data.shape[1] - args.padding, data.shape[1] - (y_max_bound - args.ymax)]
)
zstart = np.amin([args.padding, args.zmin - z_min_bound])
zend = np.amax(
[data.shape[2] - args.padding, data.shape[2] - (z_max_bound - args.zmax)]
)
print("Data model setup.")
# Template size to decide which centroids to eliminate
# Ranges use the Python convention where the number after the : is the stop
# value. Thus, x_rng specifies x values where: 0 <= x < 8.
if args.onesided:
# Only merge on the max side, to prevent duplication of detection
# Binarize Map
data[np.where(data > 0)] = 1
# Search through centroids
# On side of max values, keep anything where centroid is in padded region
# On side of min values, remove anything that is partially in padded region (covered by another block)
n_centers, _ = centroids.shape # n by 4
bad_inds = []
for i in range(0, n_centers):
if centroids[i, 0] < xstart or centroids[i, 0] - templatesize / 2 > xend:
bad_inds.append(i)
elif centroids[i, 1] < ystart or centroids[i, 1] - templatesize / 2 > yend:
bad_inds.append(i)
elif centroids[i, 2] < zstart or centroids[i, 2] - templatesize / 2 > zend:
bad_inds.append(i)
centroids_out = np.delete(centroids, bad_inds, axis=0)
# translate into global coordinates from local data block
centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin
centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin
centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin
# Eliminate any cells form data which overlap with the padding edge
for ind in bad_inds:
xi = np.array(
[
centroids[ind, 0] - np.ceil(templatesize / 2),
centroids[ind, 0] + np.ceil(templatesize / 2),
]
).astype(int)
yi = np.array(
[
centroids[ind, 1] - np.ceil(templatesize / 2),
centroids[ind, 1] + np.ceil(templatesize / 2),
]
).astype(int)
zi = np.array(
[
centroids[ind, 2] - np.ceil(templatesize / 2),
centroids[ind, 2] + np.ceil(templatesize / 2),
]
).astype(int)
data[xi, yi, zi] = 0
# Keep any interior cells, any which overlap original boundary and not padding
# Pull down existing boundary area, if area is valid
# Test side 4
if (
xend < data.shape[0]
): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[1], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 5
if (
yend < data.shape[1]
): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[1], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 6
if (
zend < data.shape[2]
): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[1], z_block_pad[1]],
)
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
],
margin,
)
# push results over entire padded area
# Pipeline Data will be in X,Y,Z format
# Change to Z,Y,X for upload
data = data[
xstart : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
]
data = np.transpose(data, (2, 1, 0))
data = data.copy(order="C").astype(eval("np.{}".format(args.dtype)))
# Verify that the cutout uploaded correctly.
rmt.create_cutout(
chan_actual,
0,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
data,
)
# Clean up.
else:
# Binarize Map
data[np.where(data > 0)] = 1
# Search through centroids
n_centers, _ = centroids.shape # n by 4
bad_inds = []
for i in range(0, n_centers):
if (
centroids[i, 0] + templatesize / 2 < xstart
or centroids[i, 0] - templatesize / 2 > xend
):
bad_inds.append(i)
elif (
centroids[i, 1] + templatesize / 2 < ystart
or centroids[i, 1] - templatesize / 2 > yend
):
bad_inds.append(i)
elif (
centroids[i, 2] + templatesize / 2 < zstart
or centroids[i, 2] - templatesize / 2 > zend
):
bad_inds.append(i)
centroids_out = np.delete(centroids, bad_inds, axis=0)
# translate into global coordinates from local data block
centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin
centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin
centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin
# Eliminate any cells form data which overlap with the padding edge
for ind in bad_inds:
xi = np.array(
[
centroids[ind, 0] - np.ceil(templatesize / 2),
centroids[ind, 0] + np.ceil(templatesize / 2),
]
).astype(int)
yi = np.array(
[
centroids[ind, 1] - np.ceil(templatesize / 2),
centroids[ind, 1] + np.ceil(templatesize / 2),
]
).astype(int)
zi = np.array(
[
centroids[ind, 2] - np.ceil(templatesize / 2),
centroids[ind, 2] + np.ceil(templatesize / 2),
]
).astype(int)
data[xi, yi, zi] = 0
# Keep any interior cells, any which overlap original boundary and not padding
# Pull down existing boundary area, if area is valid
# Test side 1
if (
xstart > 0
): # There is padding on side 1 of cube [0:pad,0:ymax+2*pad,0:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual, [x_block_pad[0], x_block[0]], y_block_pad, z_block_pad
)
data[0:xstart, :, :] = np.maximum(data[0:xstart, :, :], margin)
# Test side 2
if (
ystart > 0
): # There is padding on side 2 of cube [pad:xmax+2*pad,0:pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block_pad[0], y_block[0]],
[z_block[0], z_block_pad[1]],
)
data[xstart : data.shape[0], 0:ystart, zstart : data.shape[2]] = np.maximum(
data[xstart : data.shape[0], 0:ystart, zstart : data.shape[1]], margin
)
# Test side 3
if (
zstart > 0
): # There is padding on side 3 of cube [pad:xmax+2*pad,pad:ymax+2*pad,0:pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block_pad[0], z_block[0]],
)
data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart] = np.maximum(
data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart], margin
)
# Test side 4
if (
xend < data.shape[0]
): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[1], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 5
if (
yend < data.shape[1]
): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[1], y_block_pad[1]],
[z_block[0], z_block_pad[1]],
)
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2]
],
margin,
)
# Test side 6
if (
zend < data.shape[2]
): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad]
margin = pull_margin_cutout(
chan_actual,
[x_block[0], x_block_pad[1]],
[y_block[0], y_block_pad[1]],
[z_block[1], z_block_pad[1]],
)
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
] = np.maximum(
data[
xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2]
],
margin,
)
# push results over entire padded area
# Pipeline Data will be in X,Y,Z format
# Change to Z,Y,X for upload
data = np.transpose(data, (2, 1, 0))
data = data.copy(order="C").astype(eval("np.{}".format(args.dtype)))
# Verify that the cutout uploaded correctly.
rmt.create_cutout(chan_actual, 0, x_block_pad, y_block_pad, z_block_pad, data)
# Clean up.
def _upload(f):
print("Uploading to s3:/{}/{}".format(args.bucket, args.output))
s3 = boto3.resource("s3")
f.seek(0, 0)
s3.Object(args.bucket, args.output).put(Body=f)
# Clean up.
if args.bucket and args.s3_only:
with tempfile.TemporaryFile() as f:
np.save(f, centroids_out)
_upload(f)
else:
print("Saving output")
with open(args.output, "w+b") as f:
np.save(f, centroids_out)
if args.bucket:
_upload(f)
return
def boss_push_cutout(args):
if args.config:
rmt = BossRemote(args.config)
else:
cfg = _generate_config(args.token, args)
with open("intern.cfg", "w") as f:
cfg.write(f)
rmt = BossRemote("intern.cfg")
# data is desired range
if args.bucket:
s3 = boto3.resource("s3")
with tempfile.TemporaryFile() as f:
s3.Bucket(args.bucket).download_fileobj(args.input, f)
f.seek(0, 0)
data = np.load(f)
else:
data = np.load(args.input)
numpyType = np.uint8
if args.dtype == "uint32":
numpyType = np.uint32
elif args.dtype == "uint64":
numpyType = np.uint64
if data.dtype != args.dtype:
data = data.astype(numpyType)
sources = []
if args.source:
sources.append(args.source)
COLL_NAME = args.coll
EXP_NAME = args.exp
CHAN_NAME = args.chan
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME,
COLL_NAME,
EXP_NAME,
type=args.itype,
datatype=args.dtype,
sources=sources,
)
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
# get coordinate frame to determine padding bounds
cfr = CoordinateFrameResource(args.coord)
cfr_actual = rmt.get_project(cfr)
x_min_bound = cfr_actual.x_start
x_max_bound = cfr_actual.x_stop
y_min_bound = cfr_actual.y_start
y_max_bound = cfr_actual.y_stop
z_min_bound = cfr_actual.z_start
z_max_bound = cfr_actual.z_stop
print("Data model setup.")
# Ranges use the Python convention where the number after the : is the stop
# value. Thus, x_rng specifies x values where: 0 <= x < 8.
data_shape = data.shape # with padding, will be bigger than needed
# find data cutoffs to get rid of padding
# if nmin = 0, this means that the data wasn't padded on there to begin with
xstart = args.padding if args.xmin != 0 else 0
ystart = args.padding if args.ymin != 0 else 0
zstart = args.padding if args.zmin != 0 else 0
xend = data_shape[0] - args.padding
yend = data_shape[1] - args.padding
zend = data_shape[2] - args.padding
# xstart = np.min([args.padding,args.xmin-x_min_bound])
# xend = np.max([data.shape[0]-args.padding,data.shape[0]-(x_max_bound-args.xmax)])
# ystart = np.min([args.padding,args.ymin-y_min_bound])
# yend = np.max([data.shape[1]-args.padding,data.shape[1]-(y_max_bound-args.ymax)])
# zstart = np.min([args.padding,args.zmin-z_min_bound])
# zend = np.max([data.shape[2]-args.padding,data.shape[2]-(z_max_bound-args.zmax)])
# get range which will be uploaded
x_rng = [args.xmin, args.xmax]
y_rng = [args.ymin, args.ymax]
z_rng = [args.zmin, args.zmax]
# Pipeline Data will be in X,Y,Z format
# Change to Z,Y,X for upload
data = np.transpose(data, (2, 1, 0))
data = data[zstart:zend, ystart:yend, xstart:xend]
data = data.copy(order="C")
# Verify that the cutout uploaded correctly.
attempts = 0
while attempts < 3:
try:
rmt.create_cutout(chan_actual, args.res, x_rng, y_rng, z_rng, data)
break
except HTTPError as e:
if attempts < 3:
attempts += 1
print("These are the dimensions: ")
print(data.shape)
print("This is the data type:")
print(data.dtype)
print("Specified data type was:")
print(args.dtype)
print("Specified image type")
print(args.itype)
print("Obtained HTTP error from server. Trial {}".format(attempts))
print("The error: {}".format(e))
else:
raise Exception("Failed 3 times: {}".format(e))
rmt = BossRemote('example.cfg')
COLL_NAME = 'gray'
EXP_NAME = 'alpha'
CHAN_NAME = 'ex_EM'
# Create or get a channel to write to
chan_setup = ChannelResource(CHAN_NAME,
COLL_NAME,
EXP_NAME,
'image',
'Example channel.',
datatype='uint16')
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
print('Data model setup.')
# Ranges use the Python convention where the number after the : is the stop
# value. Thus, x_rng specifies x values where: 0 <= x < 8.
x_rng = [0, 8]
y_rng = [0, 4]
z_rng = [0, 5]
# Note that the numpy matrix is in Z, Y, X order.
data = numpy.random.randint(1, 3000, (5, 4, 8))
data = data.astype(numpy.uint16)
# limitations under the License.
"""
This script sets up the data model resource used by the example scripts.
To run this example (and create new resources), you must have a user with the resource-manager role!
"""
from intern.remote.boss import BossRemote
from intern.resource.boss.resource import *
from requests import HTTPError
rmt = BossRemote('example.cfg')
coll = CollectionResource('gray', 'Collection used for examples.')
try:
rmt.get_project(coll)
except HTTPError:
rmt.create_project(coll)
coord = CoordinateFrameResource(
'StdFrame', 'Standard coordinate frame for xyz.', 0, 2048, 0, 2048, 0, 64)
try:
coord_actual = rmt.get_project(coord)
except HTTPError:
coord_actual = rmt.create_project(coord)
alpha_exp = ExperimentResource(
'alpha', 'gray', coord_actual.name, 'Alpha example experiment.',
num_time_samples=10)
try:
rmt.get_project(alpha_exp)
class ProjectServiceTest_v1(unittest.TestCase):
"""Integration tests of the Boss resource API.
"""
def setUp(self):
self.rmt = BossRemote('test.cfg', API_VER)
# Turn off SSL cert verification. This is necessary for interacting with
# developer instances of the Boss.
self.rmt.project_service.session_send_opts = {'verify': False}
self.rmt.metadata_service.session_send_opts = {'verify': False}
self.rmt.volume_service.session_send_opts = {'verify': False}
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
coll_name = 'collection2309-{}'.format(random.randint(0, 9999))
self.coll = CollectionResource(coll_name, 'bar')
coll_name_upd = '{}-{}'.format(coll_name, random.randint(0, 9999))
self.coll_upd = CollectionResource(coll_name_upd, 'latest')
cf_name = 'ProjTestFrame{}'.format(random.randint(0, 9999))
self.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.',
0, 10, -5, 5, 3, 6, 1, 1, 1,
'nanometers')
self.coord_upd = copy.copy(self.coord)
self.coord_upd.name = 'MouseFrame{}'.format(random.randint(0, 9999))
self.coord_upd.description = 'Mouse coordinate frame.'
self.exp = ExperimentResource('exp2309-2',
self.coll.name,
self.coord.name,
'my experiment',
1,
'isotropic',
time_step=2,
time_step_unit='nanoseconds')
self.exp_upd = ExperimentResource('exp2309-2a', self.coll.name,
self.coord.name,
'my first experiment', 2,
'anisotropic')
self.source_chan = ChannelResource('sourceChan', self.coll.name,
self.exp.name, 'image',
'test source channel', 0, 'uint8',
0)
self.related_chan = ChannelResource('relatedChan', self.coll.name,
self.exp.name, 'image',
'test related channel', 0, 'uint8',
0)
self.chan = ChannelResource('myChan',
self.coll.name,
self.exp.name,
'annotation',
'test annotation channel',
0,
'uint8',
0,
sources=['sourceChan'],
related=['relatedChan'])
self.chan_upd = ChannelResource('yourChan',
self.coll.name,
self.exp.name,
'annotation',
'your test annotation channel',
0,
'uint8',
1,
sources=['sourceChan'],
related=['relatedChan'])
def tearDown(self):
try:
self.rmt.delete_project(self.chan_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.chan)
except HTTPError:
pass
try:
self.rmt.delete_project(self.related_chan)
except HTTPError:
pass
try:
self.rmt.delete_project(self.source_chan)
except HTTPError:
pass
try:
self.rmt.delete_project(self.exp_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.exp)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coord_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coord)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coll_upd)
except HTTPError:
pass
try:
self.rmt.delete_project(self.coll)
except HTTPError:
pass
def test_create_coord_frame(self):
cf = self.rmt.create_project(self.coord)
self.assertEqual(self.coord.name, cf.name)
self.assertEqual(self.coord.description, cf.description)
self.assertEqual(self.coord.x_start, cf.x_start)
self.assertEqual(self.coord.x_stop, cf.x_stop)
self.assertEqual(self.coord.y_start, cf.y_start)
self.assertEqual(self.coord.y_stop, cf.y_stop)
self.assertEqual(self.coord.z_start, cf.z_start)
self.assertEqual(self.coord.z_stop, cf.z_stop)
self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size)
self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size)
self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size)
self.assertEqual(self.coord.voxel_unit, cf.voxel_unit)
def test_create_collection(self):
c = self.rmt.create_project(self.coll)
self.assertEqual(self.coll.name, c.name)
self.assertEqual(self.coll.description, c.description)
def test_create_experiment(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
self.assertEqual(self.exp.name, e.name)
self.assertEqual(self.exp.description, e.description)
self.assertEqual(self.coll.name, e.coll_name)
self.assertEqual(self.exp.coord_frame, e.coord_frame)
self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method)
self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels)
self.assertEqual(self.exp.num_time_samples, e.num_time_samples)
self.assertEqual(self.exp.time_step, e.time_step)
self.assertEqual(self.exp.time_step_unit, e.time_step_unit)
def test_create_channel(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
ch = self.rmt.create_project(self.source_chan)
self.assertEqual(self.source_chan.name, ch.name)
self.assertEqual(self.exp.name, ch.exp_name)
self.assertEqual(self.source_chan.description, ch.description)
self.assertEqual(self.coll.name, ch.coll_name)
self.assertEqual(self.source_chan.datatype, ch.datatype)
self.assertEqual(self.source_chan.default_time_sample,
ch.default_time_sample)
self.assertEqual(self.source_chan.base_resolution, ch.base_resolution)
def test_create_annotation_channel_without_source_fails(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
rel_ch = self.rmt.create_project(self.related_chan)
chan = ChannelResource('myChan',
self.coll.name,
self.exp.name,
'annotation',
'test annotation channel',
0,
'uint8',
0,
related=['relatedChan'])
with self.assertRaises(HTTPError):
self.rmt.create_project(chan)
def test_create_annotation_channel(self):
"""Annotation channels require a source channel."""
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
ch = self.rmt.create_project(self.source_chan)
rel_ch = self.rmt.create_project(self.related_chan)
ann_ch = self.rmt.create_project(self.chan)
self.assertEqual(self.chan.name, ann_ch.name)
self.assertEqual(self.exp.name, ann_ch.exp_name)
self.assertEqual(self.chan.description, ann_ch.description)
self.assertEqual(self.coll.name, ann_ch.coll_name)
self.assertEqual(self.chan.datatype, ann_ch.datatype)
self.assertEqual(self.chan.default_time_sample,
ann_ch.default_time_sample)
self.assertEqual(self.chan.base_resolution, ann_ch.base_resolution)
self.assertEqual(self.chan.sources, ann_ch.sources)
self.assertEqual(self.chan.related, ann_ch.related)
def test_get_collection(self):
coll = self.rmt.create_project(self.coll)
c = self.rmt.get_project(self.coll)
self.assertEqual(self.coll.name, c.name)
self.assertEqual(self.coll.description, c.description)
def test_get_coord_frame(self):
coord = self.rmt.create_project(self.coord)
cf = self.rmt.get_project(self.coord)
self.assertEqual(self.coord.name, cf.name)
self.assertEqual(self.coord.description, cf.description)
self.assertEqual(self.coord.x_start, cf.x_start)
self.assertEqual(self.coord.x_stop, cf.x_stop)
self.assertEqual(self.coord.y_start, cf.y_start)
self.assertEqual(self.coord.y_stop, cf.y_stop)
self.assertEqual(self.coord.z_start, cf.z_start)
self.assertEqual(self.coord.z_stop, cf.z_stop)
self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size)
self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size)
self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size)
self.assertEqual(self.coord.voxel_unit, cf.voxel_unit)
def test_get_experiment(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
exp = self.rmt.create_project(self.exp)
e = self.rmt.get_project(self.exp)
self.assertEqual(self.exp.name, e.name)
self.assertEqual(self.exp.description, e.description)
self.assertEqual(self.coll.name, e.coll_name)
self.assertEqual(self.exp.coord_frame, e.coord_frame)
self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method)
self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels)
self.assertEqual(self.exp.num_time_samples, e.num_time_samples)
self.assertEqual(self.exp.time_step, e.time_step)
self.assertEqual(self.exp.time_step_unit, e.time_step_unit)
def test_get_channel(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
chan = self.rmt.create_project(self.source_chan)
ch = self.rmt.get_project(self.source_chan)
self.assertEqual(self.source_chan.name, ch.name)
self.assertEqual(self.exp.name, ch.exp_name)
self.assertEqual(self.source_chan.description, ch.description)
self.assertEqual(self.coll.name, ch.coll_name)
self.assertEqual(self.source_chan.datatype, ch.datatype)
self.assertEqual(self.source_chan.default_time_sample,
ch.default_time_sample)
self.assertEqual(self.source_chan.base_resolution, ch.base_resolution)
def test_get_channel_helper(self):
"""
Test the helper get_channel() as opposed to getting a channel via get_project().
"""
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
chan = self.rmt.create_project(self.source_chan)
actual = self.rmt.get_channel(chan.name, chan.coll_name, chan.exp_name)
# This is not an exhaustive list of attributes, but they are the
# important ones for correct interaction with the volume service.
self.assertTrue(actual.cutout_ready)
self.assertEqual(chan.datatype, actual.datatype)
self.assertEqual(chan.default_time_sample, actual.default_time_sample)
self.assertEqual(chan.base_resolution, actual.base_resolution)
self.assertEqual(chan.downsample_status, actual.downsample_status)
self.assertEqual(chan.type, actual.type)
self.assertEqual(chan.name, actual.name)
self.assertEqual(chan.coll_name, actual.coll_name)
self.assertEqual(chan.exp_name, actual.exp_name)
def test_update_collection(self):
coll = self.rmt.create_project(self.coll)
c = self.rmt.update_project(self.coll.name, self.coll_upd)
self.assertEqual(self.coll_upd.name, c.name)
self.assertEqual(self.coll_upd.description, c.description)
def test_update_coord_frame(self):
c = self.rmt.create_project(self.coll)
coord = self.rmt.create_project(self.coord)
cf = self.rmt.update_project(self.coord.name, self.coord_upd)
self.assertEqual(self.coord_upd.name, cf.name)
self.assertEqual(self.coord_upd.description, cf.description)
def test_update_experiment(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
eup = self.rmt.update_project(self.exp.name, self.exp_upd)
self.assertEqual(self.exp_upd.name, eup.name)
self.assertEqual(self.exp_upd.description, eup.description)
self.assertEqual(self.coll.name, eup.coll_name)
self.assertEqual(self.exp_upd.coord_frame, eup.coord_frame)
self.assertEqual(self.exp_upd.hierarchy_method, eup.hierarchy_method)
self.assertEqual(self.exp_upd.num_hierarchy_levels,
eup.num_hierarchy_levels)
self.assertEqual(self.exp_upd.num_time_samples, eup.num_time_samples)
self.assertEqual(self.exp.time_step, eup.time_step)
self.assertEqual(self.exp.time_step_unit, eup.time_step_unit)
def test_update_channel(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
source_ch = self.rmt.create_project(self.source_chan)
rel_ch = self.rmt.create_project(self.related_chan)
chan = self.rmt.create_project(self.chan)
ch = self.rmt.update_project(self.chan.name, self.chan_upd)
self.assertEqual(self.chan_upd.name, ch.name)
self.assertEqual(self.exp.name, ch.exp_name)
self.assertEqual(self.chan_upd.description, ch.description)
self.assertEqual(self.coll.name, ch.coll_name)
self.assertEqual(self.chan_upd.datatype, ch.datatype)
self.assertEqual(self.chan_upd.default_time_sample,
ch.default_time_sample)
self.assertEqual(self.chan_upd.base_resolution, ch.base_resolution)
self.assertEqual(self.chan_upd.sources, ch.sources)
self.assertEqual(self.chan_upd.related, ch.related)
def test_list_collections(self):
coll = self.rmt.create_project(self.coll)
coll_list = self.rmt.list_collections()
c = [name for name in coll_list if name == self.coll.name]
self.assertEqual(1, len(c))
self.assertEqual(self.coll.name, c[0])
def test_list_coord_frames(self):
cf = self.rmt.create_project(self.coord)
cf_list = self.rmt.list_coordinate_frames()
c = [name for name in cf_list if name == self.coord.name]
self.assertEqual(1, len(c))
self.assertEqual(self.coord.name, c[0])
def test_list_experiments(self):
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
exp = self.rmt.create_project(self.exp)
exp_list = self.rmt.list_experiments(self.coll.name)
e = [name for name in exp_list if name == self.exp.name]
self.assertEqual(1, len(e))
self.assertEqual(self.exp.name, e[0])
#def test_list_channels(self):
# c = self.rmt.create_project(self.coll)
# cf = self.rmt.create_project(self.coord)
# e = self.rmt.create_project(self.exp)
# chan = self.rmt.create_project(self.chan)
# chan_list = self.rmt.list_channels(self.coll.name, self.exp.name)
# ch = [name for name in chan_list if name == self.chan.name]
# self.assertEqual(1, len(ch))
# self.assertEqual(self.chan.name, ch[0])
def test_delete_all(self):
"""Formally test delete at all levels of the data model.
Delete happens all the time in the tearDown() but specifically test
it here.
"""
c = self.rmt.create_project(self.coll)
cf = self.rmt.create_project(self.coord)
e = self.rmt.create_project(self.exp)
ch = self.rmt.create_project(self.source_chan)
self.rmt.delete_project(self.source_chan)
self.rmt.delete_project(self.exp)
self.rmt.delete_project(self.coord)
self.rmt.delete_project(self.coll)
class NeuroDataResource:
def __init__(self, host, token, collection, experiment):
self._bossRemote = BossRemote({
'protocol': 'https',
'host': host,
'token': token
})
self.collection = collection
self.experiment = experiment
self.channels = self._bossRemote.list_channels(collection, experiment)
self.channels.remove('empty') #Delete "empty" channel
self._get_coord_frame_details()
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size,
data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print(
'Error: You must supply zRange, yRange, xRange kwargs in list format'
)
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange,
zRange)
#Datatype check. Recast to original datatype if data is float64
if data.dtype == datatype:
return data
else:
return data.astype(datatype)
class NeuroDataResource:
def __init__(self,
host,
token,
collection,
experiment,
requested_channels,
x_range=None,
y_range=None,
z_range=None,
resolution=0):
self._bossRemote = BossRemote({
'protocol': 'https',
'host': host,
'token': token
})
self.collection = collection
self.experiment = experiment
self.resolution = resolution
self.channels = self._bossRemote.list_channels(collection, experiment)
if len(requested_channels) == 0:
self.requested_channels = self.channels
else:
self.requested_channels = requested_channels
self._get_coord_frame_details()
# validate range
#if not self.correct_range(z_range, y_range, x_range):
# raise Exception("Error: Inccorect dimension range")
self.x_range = x_range or [0, self.max_dimensions[2]]
self.y_range = y_range or [0, self.max_dimensions[1]]
self.z_range = z_range or [0, self.max_dimensions[0]]
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size,
data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print(
'Error: You must supply zRange, yRange, xRange kwargs in list format'
)
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource, self.resolution,
xRange, yRange, zRange)
#Datatype check. Recast to original datatype if data is float64
if data.dtype == datatype:
return data
else:
return data.astype(datatype)
def correct_range(self, z_range, y_range, x_range):
x_start, x_end = x_range
if x_start < 0 or x_end > self.max_dimensions[2]:
return False
y_start, y_end = y_range
if y_start < 0 or y_end > self.max_dimensions[1]:
return False
z_start, z_end = z_range
if z_start < 0 or z_end > self.max_dimensions[0]:
return False
return True
def boss_pull_cutout(args):
if args.config:
rmt = BossRemote(args.config)
else:
cfg = _generate_config(args.token, args)
with open("intern.cfg", "w") as f:
cfg.write(f)
rmt = BossRemote("intern.cfg")
COLL_NAME = args.coll
EXP_NAME = args.exp
CHAN_NAME = args.chan
# Create or get a channel to write to
chan_setup = ChannelResource(
CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype
)
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
# get coordinate frame to determine padding bounds
cfr = CoordinateFrameResource(args.coord)
cfr_actual = rmt.get_project(cfr)
x_min_bound = cfr_actual.x_start
x_max_bound = cfr_actual.x_stop
y_min_bound = cfr_actual.y_start
y_max_bound = cfr_actual.y_stop
z_min_bound = cfr_actual.z_start
z_max_bound = cfr_actual.z_stop
print("Data model setup.")
xmin = np.max([x_min_bound, args.xmin - args.padding])
xmax = np.min([x_max_bound, args.xmax + args.padding])
x_rng = [xmin, xmax]
ymin = np.max([y_min_bound, args.ymin - args.padding])
ymax = np.min([y_max_bound, args.ymax + args.padding])
y_rng = [ymin, ymax]
zmin = np.max([z_min_bound, args.zmin - args.padding])
zmax = np.min([z_max_bound, args.zmax + args.padding])
z_rng = [zmin, zmax]
# Verify that the cutout uploaded correctly.
attempts = 0
while attempts < 3:
try:
cutout_data = rmt.get_cutout(chan_actual, args.res, x_rng, y_rng, z_rng)
break
except HTTPError as e:
if attempts < 3:
attempts += 1
print("Obtained HTTP error from server. Trial {}".format(attempts))
else:
print("Failed 3 times: {}".format(e))
# Data will be in Z,Y,X format
# Change to X,Y,Z for pipeline
cutout_data = np.transpose(cutout_data, (2, 1, 0))
def _upload(f):
print("Uploading to s3:/{}/{}".format(args.bucket, args.output))
s3 = boto3.resource("s3")
f.seek(0, 0)
s3.Object(args.bucket, args.output).put(Body=f)
# Clean up.
if args.bucket and args.s3_only:
with tempfile.TemporaryFile() as f:
np.save(f, cutout_data)
_upload(f)
else:
with open(args.output, "w+b") as f:
np.save(f, cutout_data)
if args.bucket:
_upload(f)
class NeuroDataResource:
def __init__(self, host, token, collection, experiment):
self._bossRemote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
self.collection = collection
self.experiment = experiment
self.channels = self._bossRemote.list_channels(collection, experiment)
self.channels.remove('empty') #Delete "empty" channel
self._get_coord_frame_details()
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print('Error: You must supply zRange, yRange, xRange kwargs in list format')
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource,
0,
xRange,
yRange,
zRange)
return data
@staticmethod
def ingest_volume(host, token, channel_name, collection, experiment, volume):
"""
Assumes the collection and experiment exists in BOSS.
"""
remote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
if volume.dtype == 'uint64':
dtype = 'uint64'
img_type = 'annotation'
sources = ['empty']
else:
dtype = volume.dtype.name
img_type = 'image'
sources = []
try:
channel_resource = ChannelResource(channel_name, collection, experiment)
channel = remote.get_project(channel_resource)
except:
channel_resource = ChannelResource(channel_name, collection, experiment,
type=img_type,
sources=sources,
datatype=dtype)
channel = remote.create_project(channel_resource)
#Get max size of experiment
exp_resource = ExperimentResource(experiment, collection)
coord_frame = remote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = remote.get_project(coord_frame_resource)
y_stop, x_stop = data.y_stop, data.x_stop
for z in range(volume.shape[0]):
print('Uploading {} slice'.format(z))
remote.create_cutout(channel,
0,
(0, x_stop),
(0, y_stop),
(z, z + 1),
volume[z, :, :].reshape((-1, y_stop, x_stop)))
def main(COLL_NAME,
EXP_NAME,
COORD_FRAME,
LOCATIONS,
BF=[5, 5, 5],
CHAN_NAMES=None,
num_threads=6,
CONFIG_FILE=None):
L = genfromtxt(LOCATIONS, delimiter=',', skip_header=0,
dtype='int32').tolist()
m = morton.Morton(dimensions=3, bits=64)
Lzo = sorted([m.pack(L[i][0], L[i][1], L[i][2]) for i in range(len(L))])
loc = np.asarray([m.unpack(l) for l in Lzo])
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
TOKEN = config['Default']['token']
boss_url = ''.join((config['Default']['protocol'], '://',
config['Default']['host'], '/v1/'))
#intern
rem = BossRemote(CONFIG_FILE)
cfr = rem.get_project(CoordinateFrameResource(COORD_FRAME))
ext = {
'x': [cfr.x_start, cfr.x_stop],
'y': [cfr.y_start, cfr.y_stop],
'z': [cfr.z_start, cfr.z_stop]
}
## remove coordinates that are outside buffer area.
x_bnd = [
np.all([x[0] - BF[0] >= ext['x'][0], x[0] + BF[0] + 1 <= ext['x'][1]])
for x in loc
]
y_bnd = [
np.all([y[1] - BF[1] >= ext['y'][0], y[1] + BF[1] + 1 <= ext['y'][1]])
for y in loc
]
z_bnd = [
np.all([z[2] - BF[2] >= ext['z'][0], z[2] + BF[2] + 1 <= ext['z'][1]])
for z in loc
]
xyzbnd = np.asarray(
[np.all([x_bnd[i], y_bnd[i], z_bnd[i]]) for i in range(len(x_bnd))])
## Select only locations that are buffered away from the edge.
loc = loc[xyzbnd]
## Compute the buffered ranges
x_rng = [[x[0] - BF[0], x[0] + BF[0] + 1] for x in loc]
y_rng = [[y[1] - BF[1], y[1] + BF[1] + 1] for y in loc]
z_rng = [[z[2] - BF[2], z[2] + BF[2] + 1] for z in loc]
ChanList = []
for ch in CHAN_NAMES:
di = [{
'rem':
rem,
'ch_rsc':
ChannelResource(ch, COLL_NAME, EXP_NAME, 'image',
datatype='uint8'),
'ch':
ch,
'res':
0,
'loc':
loc[i],
'xrng':
x_rng[i],
'yrng':
y_rng[i],
'zrng':
z_rng[i],
'bf':
BF
} for i in range(len(loc))]
with ThreadPool(num_threads) as tp:
out = tp.map(toolbox.getCube, di)
print(ch) ##DEBUG
sys.stdout.flush() #DEBUG
ChanList.append(np.asarray(out))
outArray = np.asarray(ChanList)
## outArray will be a numpy array in [channel, synapse, z, y, x] order
## this is C-ordering
return (outArray, loc)
def gen_urls(args):
config = {'protocol': 'https',
'host': args.hostname,
'token': args.token}
boss = BossRemote(config)
results = []
try:
if args.collection is not None:
collections = [args.collection]
else:
collections = boss.list_collections()
for collection in collections:
if args.experiment is not None:
experiments = [args.experiment]
else:
experiments = boss.list_experiments(collection)
for experiment in experiments:
if args.channel is not None:
channels = [args.channel]
else:
channels = boss.list_channels(collection, experiment)
exp = ExperimentResource(name = experiment,
collection_name = collection)
exp = boss.get_project(exp)
coord = CoordinateFrameResource(name = exp.coord_frame)
coord = boss.get_project(coord)
for channel in channels:
ch = ChannelResource(name = channel,
experiment_name = experiment,
collection_name = collection)
ch = boss.get_project(ch)
def check_range(name, var, start, stop):
start_, stop_ = map(int, var.split(':'))
if start_ < start:
fmt = "{} range start for {}/{}/{} is less than the coordinate frame, setting to minimum"
print(fmt.format(name, collection, experiment, channel))
start_ = start
if stop_ > stop:
fmt = "{} range stop for {}/{}/{} is greater than the coordinate frame, setting to maximum"
print(fmt.format(name, collection, experiment, channel))
stop_ = stop
return '{}:{}'.format(start_, stop_)
if args.x_range:
x = check_range('X', args.x_range, coord.x_start, coord.x_stop)
else:
x = (coord.x_start, coord.x_stop, args.min, args.max)
if args.y_range:
y = check_range('Y', args.y_range, coord.y_start, coord.y_stop)
else:
y = (coord.y_start, coord.y_stop, args.min, args.max)
if args.z_range:
z = check_range('Z', args.z_range, coord.z_start, coord.z_stop)
else:
z = (coord.z_start, coord.z_stop, args.min, args.max)
# Arguments to gen_url
results.append((args.hostname,
collection,
experiment,
channel,
0, x, y, z, None))
except Exception as e:
print("Error generating URLs: {}".format(e))
return results
xmax = 8
ymax = 4
zmax = 5
tmax = 10
COLL_NAME = 'gray'
EXP_NAME = 'timeseries_test'
CHAN_NAME = 'Ch1'
COORD_FRAME = COLL_NAME + '_' + EXP_NAME
coord = CoordinateFrameResource(
COORD_FRAME, '', 0, xmax, 0, ymax, 0, zmax)
try:
coord_actual = rmt.get_project(coord)
except HTTPError:
coord_actual = rmt.create_project(coord)
# Create or get experiment
chan_setup = ExperimentResource(
EXP_NAME, COLL_NAME, coord_frame=COORD_FRAME,
num_time_samples=tmax, time_step=1)
try:
chan_actual = rmt.get_project(chan_setup)
except HTTPError:
chan_actual = rmt.create_project(chan_setup)
# Create or get a channel to write to
class bossHandler:
"""
This is a class for interacting with BOSS. It basically is a wrapper for Intern.
"""
def __init__(self, collection_name):
"""
Constructor:
Takes the following argument:
:param collection_name: name of the collection in BOSS
:type collection_name: string
"""
self.collection_name = collection_name
try:
self.rmt = BossRemote()
except:
print('Unexpected Error:', sys.exc_info()[0])
# def get_collection_list(self):
# return self.rmt.list_collections()
def list_experiments(self):
"""
:return: all the experiments available in current collection
"""
exp_list = self.rmt.list_experiments(self.collection_name)
return exp_list
def select_experiment(self, experiment_name):
"""
Select an experiment to be added to this handler
"""
tmp = ExperimentResource(collection_name=self.collection_name,
name=experiment_name)
exp = self.rmt.get_project(tmp)
self.experiment = exp
# return exp
def get_experiment(self):
"""
:return: the currently selected experiment for this handler
"""
if hasattr(self, 'experiment'):
return self.experiment
else:
raise AttributeError(
'No experiment exists. First, select an experiment using select_experiment'
)
def list_channels(self):
"""
:return: all channel in currently selected experiment
"""
return self.rmt.list_channels(self.collection_name,
self.experiment.name)
def select_channel(self, channel_name):
"""
Select a channel to be added to this handler
"""
self.channel = self.rmt.get_channel(chan_name=channel_name,
coll_name=self.collection_name,
exp_name=self.experiment.name)
def get_coordinate_frame(self):
"""
Return: current experiment's coordinate frame
"""
tmp = CoordinateFrameResource(name=self.experiment.coord_frame)
coor = self.rmt.get_project(tmp)
self.coordinate_frame = coor
return coor
def get_all(self):
"""
:return: the entire channel image data at its native resolution
"""
x_rng = [self.coordinate_frame.x_start, self.coordinate_frame.x_stop]
y_rng = [self.coordinate_frame.y_start, self.coordinate_frame.y_stop]
z_rng = [self.coordinate_frame.z_start, self.coordinate_frame.z_stop]
return self.rmt.get_cutout(self.channel, 0, x_rng, y_rng, z_rng)
def get_cutout(self, x_range, y_range, z_range, resolution=0):
"""
:return: a cutout of the image data
"""
return self.rmt.get_cutout(self.channel, resolution, x_range, y_range,
z_range)
grp_name = 'my_team'
collection_name = "my_col"
experiment_name = "my_exp"
coord_frame_name = "my_coord"
channel_name = "my_chan"
# #### SETUP
rmt = BossRemote("./boss.cfg")
print('Creating group . . .')
try:
rmt.create_group(grp_name)
except Exception as e:
# Assume group already exists if an exception raised.
print(e)
# Get the resources
collection = CollectionResource(collection_name)
collection = rmt.get_project(collection)
experiment = ExperimentResource(experiment_name, collection_name, coord_frame_name)
experiment = rmt.get_project(experiment)
channel = ChannelResource(channel_name, collection_name, experiment_name)
channel = rmt.get_project(channel)
# Add perms
perms = ['read']
rmt.add_permissions(grp_name, collection, perms)
rmt.add_permissions(grp_name, experiment, perms)
rmt.add_permissions(grp_name, channel, perms)
class InternTileProcessor(TileProcessor):
"""A Tile processor for a single image file identified by z index"""
def __init__(self):
"""Constructor to add custom class var"""
TileProcessor.__init__(self)
self.remote = None
self.channel = None
def setup(self, parameters):
""" Method to load the file for uploading data. Assumes intern token is set via environment variable or config
default file
Args:
parameters (dict): Parameters for the dataset to be processed
MUST HAVE THE CUSTOM PARAMETERS: "x_offset": offset to apply when querying the Boss
"y_offset": offset to apply when querying the Boss
"z_offset": offset to apply when querying the Boss
"x_tile": size of a tile in x dimension
"y_tile": size of a tile in y dimension
"collection": source collection
"experiment": source experiment
"channel": source channel
"resolution": source resolution
Returns:
None
"""
self.parameters = parameters
self.remote = BossRemote()
self.channel = ChannelResource(self.parameters["channel"],
self.parameters["collection"],
self.parameters["experiment"])
self.channel = self.remote.get_project(self.channel)
def process(self, file_path, x_index, y_index, z_index, t_index=0):
"""
Method to load the image file.
Args:
file_path(str): An absolute file path for the specified tile
x_index(int): The tile index in the X dimension
y_index(int): The tile index in the Y dimension
z_index(int): The tile index in the Z dimension
t_index(int): The time index
Returns:
(io.BufferedReader): A file handle for the specified tile
"""
# Compute cutout args
x_rng = [
self.parameters["x_tile"] * x_index + self.parameters["x_offset"],
self.parameters["x_tile"] * (x_index + 1) +
self.parameters["x_offset"]
]
y_rng = [
self.parameters["y_tile"] * y_index + self.parameters["y_offset"],
self.parameters["y_tile"] * (y_index + 1) +
self.parameters["y_offset"]
]
z_rng = [
z_index + self.parameters["z_offset"],
z_index + 1 + self.parameters["z_offset"]
]
if z_index + self.parameters["z_offset"] < 0:
data = np.zeros(
(self.parameters["x_tile"], self.parameters["y_tile"]),
dtype=np.int32,
order="C")
else:
# Get data
cnt = 0
while cnt < 5:
try:
data = self.remote.get_cutout(
self.channel, self.parameters["resolution"], x_rng,
y_rng, z_rng)
data = np.asarray(data, np.uint32)
break
except Exception as err:
if cnt > 5:
raise err
cnt += 1
time.sleep(10)
# Save sub-img to png and return handle
upload_img = Image.fromarray(np.squeeze(data))
output = six.BytesIO()
upload_img.save(output, format="TIFF")
# Send handle back
return output
class BossResParams:
def __init__(self, ingest_job, get_only=True):
self.ingest_job = ingest_job
self.coord_frame_name = '_'.join(
(ingest_job.coll_name, ingest_job.exp_name))
self.rmt = BossRemote(self.ingest_job.boss_config_file)
self.coll_resource = self.setup_boss_collection(get_only=get_only)
self.coord_frame_resource = self.setup_boss_coord_frame(
get_only=get_only)
self.exp_resource = self.setup_boss_experiment(get_only=get_only)
self.ch_resource = self.setup_boss_channel(get_only=get_only)
def get_boss_project(self, proj_setup, get_only):
try:
proj_actual = self.rmt.get_project(proj_setup)
except HTTPError as e:
if get_only is False:
try:
proj_actual = self.rmt.create_project(proj_setup)
except Exception as e:
print(type(e), e)
raise (e)
else:
print(type(e), e)
raise e
except Exception as e:
print(type(e), e)
raise (e)
return proj_actual
def setup_boss_collection(self, get_only=True):
coll_setup = CollectionResource(self.ingest_job.coll_name)
return self.get_boss_project(coll_setup, get_only)
def setup_boss_coord_frame(self, get_only=True):
# if we don't know the coordinate frame parameters, get the one with the same name
if get_only:
coord_setup = CoordinateFrameResource(self.coord_frame_name)
else:
coord_setup = CoordinateFrameResource(
self.coord_frame_name, '',
self.ingest_job.coord_frame_x_extent[0],
self.ingest_job.coord_frame_x_extent[1],
self.ingest_job.coord_frame_y_extent[0],
self.ingest_job.coord_frame_y_extent[1],
self.ingest_job.coord_frame_z_extent[0],
self.ingest_job.coord_frame_z_extent[1],
self.ingest_job.voxel_size[0], self.ingest_job.voxel_size[1],
self.ingest_job.voxel_size[2], self.ingest_job.voxel_unit)
coord_frame_resource = self.get_boss_project(coord_setup, get_only)
if get_only:
# matching ingest_job values to coordinate frame values (if they weren't specified, they are now populated)
self.ingest_job.voxel_size = [
coord_frame_resource.x_voxel_size,
coord_frame_resource.y_voxel_size,
coord_frame_resource.z_voxel_size
]
self.ingest_job.voxel_unit = coord_frame_resource.voxel_unit
return coord_frame_resource
def setup_boss_experiment(self, get_only=True):
# if we don't know the coordinate frame parameters, get the one with the same name
if get_only:
exp_setup = ExperimentResource(self.ingest_job.exp_name,
self.ingest_job.coll_name,
self.coord_frame_name)
else:
# if all elements are the same, isotropic, otherwise anisotropic
if len(set(self.ingest_job.voxel_size)) <= 1:
hierarchy_method = 'isotropic'
else:
hierarchy_method = 'anisotropic'
num_hierarchy_levels = self.calc_hierarchy_levels()
exp_setup = ExperimentResource(self.ingest_job.exp_name,
self.ingest_job.coll_name,
self.coord_frame_name, '',
num_hierarchy_levels,
hierarchy_method)
exp_resource = self.get_boss_project(exp_setup, get_only)
# record the offset (if there is any) into BOSS metadata field for experiment
self.record_offsets(exp_resource)
return exp_resource
def record_offsets(self, exp_resource):
if any([a > 0 for a in self.ingest_job.offsets]):
offsets_dict = {'offsets': self.ingest_job.offsets}
try:
self.rmt.create_metadata(exp_resource, offsets_dict)
except HTTPErrorList: # keys already exist
self.rmt.update_metadata(exp_resource, offsets_dict)
def setup_boss_channel(self, ch_description='', get_only=True):
ch_args = [
self.ingest_job.ch_name, self.ingest_job.coll_name,
self.ingest_job.exp_name
]
if not get_only:
ch_args.extend(
(self.ingest_job.ch_type, ch_description, 0,
self.ingest_job.boss_datatype, self.ingest_job.res))
# annotation data
if self.ingest_job.source_channel is not None:
ch_args.append([self.ingest_job.source_channel])
# checking to make sure source channel exists (creating if needed)
try:
source_args = [
self.ingest_job.source_channel,
self.ingest_job.coll_name, self.ingest_job.exp_name
]
source_setup = ChannelResource(*source_args)
self.get_boss_project(source_setup, True)
except:
self.ingest_job.send_msg(
'Creating a dummy source channel for this annotation because none exists yet'
)
source_args.extend(('image', '', 0, 'uint8', 0))
source_setup = ChannelResource(*source_args)
self.get_boss_project(source_setup, False)
ch_setup = ChannelResource(*ch_args)
ch_resource = self.get_boss_project(ch_setup, get_only)
if get_only:
self.ingest_job.boss_datatype = ch_resource.datatype
if self.ingest_job.datatype is None:
self.ingest_job.datatype = ch_resource.datatype
self.ingest_job.res = ch_resource.base_resolution
return ch_resource
def calc_hierarchy_levels(self, lowest_res=512):
img_size = [
self.coord_frame_resource.x_stop -
self.coord_frame_resource.x_start,
self.coord_frame_resource.y_stop -
self.coord_frame_resource.y_start,
self.coord_frame_resource.z_stop -
self.coord_frame_resource.z_start
]
max_xy = max(img_size[0:1])
# we add one because 0 is included in the number of downsampling levels
num_levels = max(1, math.ceil(math.log(max_xy / lowest_res, 2)) + 1)
return num_levels
