def fetch_info(self): experiment = ExperimentResource(name=self.path.dataset, collection_name=self.path.bucket) rmt = BossRemote(boss_credentials) experiment = rmt.get_project(experiment) coord_frame = CoordinateFrameResource(name=experiment.coord_frame) coord_frame = rmt.get_project(coord_frame) channel = ChannelResource(self.path.layer, self.path.bucket, self.path.dataset) channel = rmt.get_project(channel) unit_factors = { 'nanometers': 1, 'micrometers': 1e3, 'millimeters': 1e6, 'centimeters': 1e7, } unit_factor = unit_factors[coord_frame.voxel_unit] cf = coord_frame resolution = [cf.x_voxel_size, cf.y_voxel_size, cf.z_voxel_size] resolution = [int(round(_)) * unit_factor for _ in resolution] bbox = Bbox((cf.x_start, cf.y_start, cf.z_start), (cf.x_stop, cf.y_stop, cf.z_stop)) bbox.maxpt = bbox.maxpt layer_type = 'unknown' if 'type' in channel.raw: layer_type = channel.raw['type'] info = CloudVolume.create_new_info( num_channels=1, layer_type=layer_type, data_type=channel.datatype, encoding='raw', resolution=resolution, voxel_offset=bbox.minpt, volume_size=bbox.size3(), chunk_size=(512, 512, 16), # fixed in s3 implementation ) each_factor = Vec(2, 2, 1) if experiment.hierarchy_method == 'isotropic': each_factor = Vec(2, 2, 2) factor = each_factor.clone() for _ in range(1, experiment.num_hierarchy_levels): self.add_scale(factor, info=info) factor *= each_factor return info
def push_tif(): # message1 = request.form['channel'] #if message1: # print(message1) col = request.form["collection"] exp = request.form["experiment"] channel = request.form["channel"] dtype = request.form["datatype"] host,token = get_host_token() x_range = [int(request.form["xstart"]),int(request.form["xstop"])] y_range = [int(request.form["ystart"]),int(request.form["ystop"])] z_range = [int(request.form["zstart"]),int(request.form["zstop"])] remote = BossRemote('./neurodata.cfg') voxel_size = '1 1 1' voxel_unit = 'micrometers' channel_resource = ChannelResource(channel, col, exp, 'image', '', 0, dtype, 0) project = remote.get_project(channel_resource) my_file = request.form["file"] my_array = np.array(io.imread('./DATA/'+my_file)).astype(np.uint8) for z in range(z_range[0],z_range[1]): remote.create_cutout(channel_resource, 0, (x_range[0],x_range[1]), (y_range[0],y_range[1]), (z,z+1), my_array[z].reshape(-1,my_array.shape[1],my_array.shape[2])) return 'Successfully pushed'
def delete_channel(remote: BossRemote, channel, coll_name=None, exp_name=None): chan_obj = None if isinstance(channel, ChannelResource): chan_obj = channel elif isinstance(channel, str) and coll_name is not None and exp_name is not None: chan_name = channel chan_obj = remote.get_project(ChannelResource(name=chan_name, experiment_name=exp_name)) if chan_obj is not None: print('Deleting channel "{0}"...'.format(chan_obj.name)) remote.delete_project(chan_obj)
def ingest_volume(host, token, channel_name, collection, experiment, volume): """ Assumes the collection and experiment exists in BOSS. """ remote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) if volume.dtype == 'uint64': dtype = 'uint64' img_type = 'annotation' sources = ['empty'] else: dtype = volume.dtype.name img_type = 'image' sources = [] try: channel_resource = ChannelResource(channel_name, collection, experiment) channel = remote.get_project(channel_resource) except: channel_resource = ChannelResource(channel_name, collection, experiment, type=img_type, sources=sources, datatype=dtype) channel = remote.create_project(channel_resource) #Get max size of experiment exp_resource = ExperimentResource(experiment, collection) coord_frame = remote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = remote.get_project(coord_frame_resource) y_stop, x_stop = data.y_stop, data.x_stop for z in range(volume.shape[0]): print('Uploading {} slice'.format(z)) remote.create_cutout(channel, 0, (0, x_stop), (0, y_stop), (z, z + 1), volume[z, :, :].reshape((-1, y_stop, x_stop)))
def delete_coord_frame(remote: BossRemote, coord_frame): frame_obj = None # frame_name = None if isinstance(coord_frame, CoordinateFrameResource): frame_obj = coord_frame # frame_name = coord_frame.name elif isinstance(coord_frame, str): frame_name = coord_frame frame_obj = remote.get_project(CoordinateFrameResource(name=frame_name)) if frame_obj is not None: print('Deleting coordinate frame "{0}"...'.format(frame_obj.name)) remote.delete_project(frame_obj)
def delete_collection(remote: BossRemote, collection): coll_obj = None coll_name = None if isinstance(collection, CollectionResource): coll_obj = collection coll_name = collection.name elif isinstance(collection, str): coll_name = collection coll_obj = remote.get_project(CollectionResource(name=coll_name)) if coll_name is not None: print('Deleting experiments of collection "{0}"...'.format(coll_name)) exp_names = remote.list_experiments(coll_name) for n in exp_names: delete_experiment(remote, n, coll_name) if coll_obj is not None: print('Deleting collection "{0}"...'.format(coll_name)) remote.delete_project(coll_obj)
def delete_experiment(remote: BossRemote, experiment, coll_name=None): exp_obj = None exp_name = None if isinstance(experiment, ExperimentResource): exp_obj = experiment exp_name = experiment.name coll_name = experiment.coll_name elif isinstance(experiment, str) and coll_name is not None: exp_name = experiment exp_obj = remote.get_project(ExperimentResource(name=exp_name, collection_name=coll_name)) if exp_name is not None: print('Deleting channels of experiment "{0}"...'.format(exp_name)) chan_names = remote.list_channels(coll_name, exp_name) for n in chan_names: delete_channel(remote, n, exp_name, coll_name) if exp_obj is not None: print('Deleting experiment "{0}"...'.format(exp_obj.name)) remote.delete_project(exp_obj)
def boss_merge_xbrain(args): # Verify that the cutout uploaded correctly. def pull_margin_cutout(chan_actual, x_rng, y_rng, z_rng): attempts = 0 while attempts < 3: try: cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng) break except HTTPError as e: if attempts < 3: attempts += 1 print("Obtained HTTP error from server. Trial {}".format(attempts)) else: print("Failed 3 times: {}".format(e)) # Data will be in Z,Y,X format # Change to X,Y,Z for pipeline cutout_data = np.transpose(cutout_data, (2, 1, 0)) return cutout_data templatesize = args.templatesize if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") # data is desired range if args.bucket: s3 = boto3.resource("s3") with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.input, f) f.seek(0, 0) data = np.load(f) with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.centroids, f) f.seek(0, 0) centroids = np.load(f) else: data = np.load(args.input) centroids = np.load(args.centroids) COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop # coordinates of data block in original coordinate frame, before padding x_block = [args.xmin, args.xmax] y_block = [args.ymin, args.ymax] z_block = [args.zmin, args.zmax] # Coordinates of data block with padding in original coordinate frame x_block_pad = [ np.amax([args.xmin - args.padding, x_min_bound]), np.amin([args.xmax + args.padding, x_max_bound]), ] y_block_pad = [ np.amax([args.ymin - args.padding, y_min_bound]), np.amin([args.ymax + args.padding, y_max_bound]), ] z_block_pad = [ np.amax([args.zmin - args.padding, z_min_bound]), np.amin([args.zmax + args.padding, z_max_bound]), ] # Coordinates of core data block in local coordinate frame xstart = np.amin([args.padding, args.xmin - x_min_bound]) xend = np.amax( [data.shape[0] - args.padding, data.shape[0] - (x_max_bound - args.xmax)] ) ystart = np.amin([args.padding, args.ymin - y_min_bound]) yend = np.amax( [data.shape[1] - args.padding, data.shape[1] - (y_max_bound - args.ymax)] ) zstart = np.amin([args.padding, args.zmin - z_min_bound]) zend = np.amax( [data.shape[2] - args.padding, data.shape[2] - (z_max_bound - args.zmax)] ) print("Data model setup.") # Template size to decide which centroids to eliminate # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. if args.onesided: # Only merge on the max side, to prevent duplication of detection # Binarize Map data[np.where(data > 0)] = 1 # Search through centroids # On side of max values, keep anything where centroid is in padded region # On side of min values, remove anything that is partially in padded region (covered by another block) n_centers, _ = centroids.shape # n by 4 bad_inds = [] for i in range(0, n_centers): if centroids[i, 0] < xstart or centroids[i, 0] - templatesize / 2 > xend: bad_inds.append(i) elif centroids[i, 1] < ystart or centroids[i, 1] - templatesize / 2 > yend: bad_inds.append(i) elif centroids[i, 2] < zstart or centroids[i, 2] - templatesize / 2 > zend: bad_inds.append(i) centroids_out = np.delete(centroids, bad_inds, axis=0) # translate into global coordinates from local data block centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin # Eliminate any cells form data which overlap with the padding edge for ind in bad_inds: xi = np.array( [ centroids[ind, 0] - np.ceil(templatesize / 2), centroids[ind, 0] + np.ceil(templatesize / 2), ] ).astype(int) yi = np.array( [ centroids[ind, 1] - np.ceil(templatesize / 2), centroids[ind, 1] + np.ceil(templatesize / 2), ] ).astype(int) zi = np.array( [ centroids[ind, 2] - np.ceil(templatesize / 2), centroids[ind, 2] + np.ceil(templatesize / 2), ] ).astype(int) data[xi, yi, zi] = 0 # Keep any interior cells, any which overlap original boundary and not padding # Pull down existing boundary area, if area is valid # Test side 4 if ( xend < data.shape[0] ): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[1], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 5 if ( yend < data.shape[1] ): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[1], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 6 if ( zend < data.shape[2] ): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[1], z_block_pad[1]], ) data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ], margin, ) # push results over entire padded area # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = data[ xstart : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] data = np.transpose(data, (2, 1, 0)) data = data.copy(order="C").astype(eval("np.{}".format(args.dtype))) # Verify that the cutout uploaded correctly. rmt.create_cutout( chan_actual, 0, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], data, ) # Clean up. else: # Binarize Map data[np.where(data > 0)] = 1 # Search through centroids n_centers, _ = centroids.shape # n by 4 bad_inds = [] for i in range(0, n_centers): if ( centroids[i, 0] + templatesize / 2 < xstart or centroids[i, 0] - templatesize / 2 > xend ): bad_inds.append(i) elif ( centroids[i, 1] + templatesize / 2 < ystart or centroids[i, 1] - templatesize / 2 > yend ): bad_inds.append(i) elif ( centroids[i, 2] + templatesize / 2 < zstart or centroids[i, 2] - templatesize / 2 > zend ): bad_inds.append(i) centroids_out = np.delete(centroids, bad_inds, axis=0) # translate into global coordinates from local data block centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin # Eliminate any cells form data which overlap with the padding edge for ind in bad_inds: xi = np.array( [ centroids[ind, 0] - np.ceil(templatesize / 2), centroids[ind, 0] + np.ceil(templatesize / 2), ] ).astype(int) yi = np.array( [ centroids[ind, 1] - np.ceil(templatesize / 2), centroids[ind, 1] + np.ceil(templatesize / 2), ] ).astype(int) zi = np.array( [ centroids[ind, 2] - np.ceil(templatesize / 2), centroids[ind, 2] + np.ceil(templatesize / 2), ] ).astype(int) data[xi, yi, zi] = 0 # Keep any interior cells, any which overlap original boundary and not padding # Pull down existing boundary area, if area is valid # Test side 1 if ( xstart > 0 ): # There is padding on side 1 of cube [0:pad,0:ymax+2*pad,0:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block_pad[0], x_block[0]], y_block_pad, z_block_pad ) data[0:xstart, :, :] = np.maximum(data[0:xstart, :, :], margin) # Test side 2 if ( ystart > 0 ): # There is padding on side 2 of cube [pad:xmax+2*pad,0:pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block_pad[0], y_block[0]], [z_block[0], z_block_pad[1]], ) data[xstart : data.shape[0], 0:ystart, zstart : data.shape[2]] = np.maximum( data[xstart : data.shape[0], 0:ystart, zstart : data.shape[1]], margin ) # Test side 3 if ( zstart > 0 ): # There is padding on side 3 of cube [pad:xmax+2*pad,pad:ymax+2*pad,0:pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block_pad[0], z_block[0]], ) data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart] = np.maximum( data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart], margin ) # Test side 4 if ( xend < data.shape[0] ): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[1], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 5 if ( yend < data.shape[1] ): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[1], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 6 if ( zend < data.shape[2] ): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[1], z_block_pad[1]], ) data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ], margin, ) # push results over entire padded area # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = np.transpose(data, (2, 1, 0)) data = data.copy(order="C").astype(eval("np.{}".format(args.dtype))) # Verify that the cutout uploaded correctly. rmt.create_cutout(chan_actual, 0, x_block_pad, y_block_pad, z_block_pad, data) # Clean up. def _upload(f): print("Uploading to s3:/{}/{}".format(args.bucket, args.output)) s3 = boto3.resource("s3") f.seek(0, 0) s3.Object(args.bucket, args.output).put(Body=f) # Clean up. if args.bucket and args.s3_only: with tempfile.TemporaryFile() as f: np.save(f, centroids_out) _upload(f) else: print("Saving output") with open(args.output, "w+b") as f: np.save(f, centroids_out) if args.bucket: _upload(f) return
def boss_push_cutout(args): if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") # data is desired range if args.bucket: s3 = boto3.resource("s3") with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.input, f) f.seek(0, 0) data = np.load(f) else: data = np.load(args.input) numpyType = np.uint8 if args.dtype == "uint32": numpyType = np.uint32 elif args.dtype == "uint64": numpyType = np.uint64 if data.dtype != args.dtype: data = data.astype(numpyType) sources = [] if args.source: sources.append(args.source) COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype, sources=sources, ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop print("Data model setup.") # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. data_shape = data.shape # with padding, will be bigger than needed # find data cutoffs to get rid of padding # if nmin = 0, this means that the data wasn't padded on there to begin with xstart = args.padding if args.xmin != 0 else 0 ystart = args.padding if args.ymin != 0 else 0 zstart = args.padding if args.zmin != 0 else 0 xend = data_shape[0] - args.padding yend = data_shape[1] - args.padding zend = data_shape[2] - args.padding # xstart = np.min([args.padding,args.xmin-x_min_bound]) # xend = np.max([data.shape[0]-args.padding,data.shape[0]-(x_max_bound-args.xmax)]) # ystart = np.min([args.padding,args.ymin-y_min_bound]) # yend = np.max([data.shape[1]-args.padding,data.shape[1]-(y_max_bound-args.ymax)]) # zstart = np.min([args.padding,args.zmin-z_min_bound]) # zend = np.max([data.shape[2]-args.padding,data.shape[2]-(z_max_bound-args.zmax)]) # get range which will be uploaded x_rng = [args.xmin, args.xmax] y_rng = [args.ymin, args.ymax] z_rng = [args.zmin, args.zmax] # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = np.transpose(data, (2, 1, 0)) data = data[zstart:zend, ystart:yend, xstart:xend] data = data.copy(order="C") # Verify that the cutout uploaded correctly. attempts = 0 while attempts < 3: try: rmt.create_cutout(chan_actual, args.res, x_rng, y_rng, z_rng, data) break except HTTPError as e: if attempts < 3: attempts += 1 print("These are the dimensions: ") print(data.shape) print("This is the data type:") print(data.dtype) print("Specified data type was:") print(args.dtype) print("Specified image type") print(args.itype) print("Obtained HTTP error from server. Trial {}".format(attempts)) print("The error: {}".format(e)) else: raise Exception("Failed 3 times: {}".format(e))
rmt = BossRemote('example.cfg') COLL_NAME = 'gray' EXP_NAME = 'alpha' CHAN_NAME = 'ex_EM' # Create or get a channel to write to chan_setup = ChannelResource(CHAN_NAME, COLL_NAME, EXP_NAME, 'image', 'Example channel.', datatype='uint16') try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) print('Data model setup.') # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] # Note that the numpy matrix is in Z, Y, X order. data = numpy.random.randint(1, 3000, (5, 4, 8)) data = data.astype(numpy.uint16)
# limitations under the License. """ This script sets up the data model resource used by the example scripts. To run this example (and create new resources), you must have a user with the resource-manager role! """ from intern.remote.boss import BossRemote from intern.resource.boss.resource import * from requests import HTTPError rmt = BossRemote('example.cfg') coll = CollectionResource('gray', 'Collection used for examples.') try: rmt.get_project(coll) except HTTPError: rmt.create_project(coll) coord = CoordinateFrameResource( 'StdFrame', 'Standard coordinate frame for xyz.', 0, 2048, 0, 2048, 0, 64) try: coord_actual = rmt.get_project(coord) except HTTPError: coord_actual = rmt.create_project(coord) alpha_exp = ExperimentResource( 'alpha', 'gray', coord_actual.name, 'Alpha example experiment.', num_time_samples=10) try: rmt.get_project(alpha_exp)
class ProjectServiceTest_v1(unittest.TestCase): """Integration tests of the Boss resource API. """ def setUp(self): self.rmt = BossRemote('test.cfg', API_VER) # Turn off SSL cert verification. This is necessary for interacting with # developer instances of the Boss. self.rmt.project_service.session_send_opts = {'verify': False} self.rmt.metadata_service.session_send_opts = {'verify': False} self.rmt.volume_service.session_send_opts = {'verify': False} requests.packages.urllib3.disable_warnings(InsecureRequestWarning) coll_name = 'collection2309-{}'.format(random.randint(0, 9999)) self.coll = CollectionResource(coll_name, 'bar') coll_name_upd = '{}-{}'.format(coll_name, random.randint(0, 9999)) self.coll_upd = CollectionResource(coll_name_upd, 'latest') cf_name = 'ProjTestFrame{}'.format(random.randint(0, 9999)) self.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.', 0, 10, -5, 5, 3, 6, 1, 1, 1, 'nanometers') self.coord_upd = copy.copy(self.coord) self.coord_upd.name = 'MouseFrame{}'.format(random.randint(0, 9999)) self.coord_upd.description = 'Mouse coordinate frame.' self.exp = ExperimentResource('exp2309-2', self.coll.name, self.coord.name, 'my experiment', 1, 'isotropic', time_step=2, time_step_unit='nanoseconds') self.exp_upd = ExperimentResource('exp2309-2a', self.coll.name, self.coord.name, 'my first experiment', 2, 'anisotropic') self.source_chan = ChannelResource('sourceChan', self.coll.name, self.exp.name, 'image', 'test source channel', 0, 'uint8', 0) self.related_chan = ChannelResource('relatedChan', self.coll.name, self.exp.name, 'image', 'test related channel', 0, 'uint8', 0) self.chan = ChannelResource('myChan', self.coll.name, self.exp.name, 'annotation', 'test annotation channel', 0, 'uint8', 0, sources=['sourceChan'], related=['relatedChan']) self.chan_upd = ChannelResource('yourChan', self.coll.name, self.exp.name, 'annotation', 'your test annotation channel', 0, 'uint8', 1, sources=['sourceChan'], related=['relatedChan']) def tearDown(self): try: self.rmt.delete_project(self.chan_upd) except HTTPError: pass try: self.rmt.delete_project(self.chan) except HTTPError: pass try: self.rmt.delete_project(self.related_chan) except HTTPError: pass try: self.rmt.delete_project(self.source_chan) except HTTPError: pass try: self.rmt.delete_project(self.exp_upd) except HTTPError: pass try: self.rmt.delete_project(self.exp) except HTTPError: pass try: self.rmt.delete_project(self.coord_upd) except HTTPError: pass try: self.rmt.delete_project(self.coord) except HTTPError: pass try: self.rmt.delete_project(self.coll_upd) except HTTPError: pass try: self.rmt.delete_project(self.coll) except HTTPError: pass def test_create_coord_frame(self): cf = self.rmt.create_project(self.coord) self.assertEqual(self.coord.name, cf.name) self.assertEqual(self.coord.description, cf.description) self.assertEqual(self.coord.x_start, cf.x_start) self.assertEqual(self.coord.x_stop, cf.x_stop) self.assertEqual(self.coord.y_start, cf.y_start) self.assertEqual(self.coord.y_stop, cf.y_stop) self.assertEqual(self.coord.z_start, cf.z_start) self.assertEqual(self.coord.z_stop, cf.z_stop) self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size) self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size) self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size) self.assertEqual(self.coord.voxel_unit, cf.voxel_unit) def test_create_collection(self): c = self.rmt.create_project(self.coll) self.assertEqual(self.coll.name, c.name) self.assertEqual(self.coll.description, c.description) def test_create_experiment(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) self.assertEqual(self.exp.name, e.name) self.assertEqual(self.exp.description, e.description) self.assertEqual(self.coll.name, e.coll_name) self.assertEqual(self.exp.coord_frame, e.coord_frame) self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method) self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels) self.assertEqual(self.exp.num_time_samples, e.num_time_samples) self.assertEqual(self.exp.time_step, e.time_step) self.assertEqual(self.exp.time_step_unit, e.time_step_unit) def test_create_channel(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) ch = self.rmt.create_project(self.source_chan) self.assertEqual(self.source_chan.name, ch.name) self.assertEqual(self.exp.name, ch.exp_name) self.assertEqual(self.source_chan.description, ch.description) self.assertEqual(self.coll.name, ch.coll_name) self.assertEqual(self.source_chan.datatype, ch.datatype) self.assertEqual(self.source_chan.default_time_sample, ch.default_time_sample) self.assertEqual(self.source_chan.base_resolution, ch.base_resolution) def test_create_annotation_channel_without_source_fails(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) rel_ch = self.rmt.create_project(self.related_chan) chan = ChannelResource('myChan', self.coll.name, self.exp.name, 'annotation', 'test annotation channel', 0, 'uint8', 0, related=['relatedChan']) with self.assertRaises(HTTPError): self.rmt.create_project(chan) def test_create_annotation_channel(self): """Annotation channels require a source channel.""" c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) ch = self.rmt.create_project(self.source_chan) rel_ch = self.rmt.create_project(self.related_chan) ann_ch = self.rmt.create_project(self.chan) self.assertEqual(self.chan.name, ann_ch.name) self.assertEqual(self.exp.name, ann_ch.exp_name) self.assertEqual(self.chan.description, ann_ch.description) self.assertEqual(self.coll.name, ann_ch.coll_name) self.assertEqual(self.chan.datatype, ann_ch.datatype) self.assertEqual(self.chan.default_time_sample, ann_ch.default_time_sample) self.assertEqual(self.chan.base_resolution, ann_ch.base_resolution) self.assertEqual(self.chan.sources, ann_ch.sources) self.assertEqual(self.chan.related, ann_ch.related) def test_get_collection(self): coll = self.rmt.create_project(self.coll) c = self.rmt.get_project(self.coll) self.assertEqual(self.coll.name, c.name) self.assertEqual(self.coll.description, c.description) def test_get_coord_frame(self): coord = self.rmt.create_project(self.coord) cf = self.rmt.get_project(self.coord) self.assertEqual(self.coord.name, cf.name) self.assertEqual(self.coord.description, cf.description) self.assertEqual(self.coord.x_start, cf.x_start) self.assertEqual(self.coord.x_stop, cf.x_stop) self.assertEqual(self.coord.y_start, cf.y_start) self.assertEqual(self.coord.y_stop, cf.y_stop) self.assertEqual(self.coord.z_start, cf.z_start) self.assertEqual(self.coord.z_stop, cf.z_stop) self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size) self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size) self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size) self.assertEqual(self.coord.voxel_unit, cf.voxel_unit) def test_get_experiment(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) exp = self.rmt.create_project(self.exp) e = self.rmt.get_project(self.exp) self.assertEqual(self.exp.name, e.name) self.assertEqual(self.exp.description, e.description) self.assertEqual(self.coll.name, e.coll_name) self.assertEqual(self.exp.coord_frame, e.coord_frame) self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method) self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels) self.assertEqual(self.exp.num_time_samples, e.num_time_samples) self.assertEqual(self.exp.time_step, e.time_step) self.assertEqual(self.exp.time_step_unit, e.time_step_unit) def test_get_channel(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) chan = self.rmt.create_project(self.source_chan) ch = self.rmt.get_project(self.source_chan) self.assertEqual(self.source_chan.name, ch.name) self.assertEqual(self.exp.name, ch.exp_name) self.assertEqual(self.source_chan.description, ch.description) self.assertEqual(self.coll.name, ch.coll_name) self.assertEqual(self.source_chan.datatype, ch.datatype) self.assertEqual(self.source_chan.default_time_sample, ch.default_time_sample) self.assertEqual(self.source_chan.base_resolution, ch.base_resolution) def test_get_channel_helper(self): """ Test the helper get_channel() as opposed to getting a channel via get_project(). """ c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) chan = self.rmt.create_project(self.source_chan) actual = self.rmt.get_channel(chan.name, chan.coll_name, chan.exp_name) # This is not an exhaustive list of attributes, but they are the # important ones for correct interaction with the volume service. self.assertTrue(actual.cutout_ready) self.assertEqual(chan.datatype, actual.datatype) self.assertEqual(chan.default_time_sample, actual.default_time_sample) self.assertEqual(chan.base_resolution, actual.base_resolution) self.assertEqual(chan.downsample_status, actual.downsample_status) self.assertEqual(chan.type, actual.type) self.assertEqual(chan.name, actual.name) self.assertEqual(chan.coll_name, actual.coll_name) self.assertEqual(chan.exp_name, actual.exp_name) def test_update_collection(self): coll = self.rmt.create_project(self.coll) c = self.rmt.update_project(self.coll.name, self.coll_upd) self.assertEqual(self.coll_upd.name, c.name) self.assertEqual(self.coll_upd.description, c.description) def test_update_coord_frame(self): c = self.rmt.create_project(self.coll) coord = self.rmt.create_project(self.coord) cf = self.rmt.update_project(self.coord.name, self.coord_upd) self.assertEqual(self.coord_upd.name, cf.name) self.assertEqual(self.coord_upd.description, cf.description) def test_update_experiment(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) eup = self.rmt.update_project(self.exp.name, self.exp_upd) self.assertEqual(self.exp_upd.name, eup.name) self.assertEqual(self.exp_upd.description, eup.description) self.assertEqual(self.coll.name, eup.coll_name) self.assertEqual(self.exp_upd.coord_frame, eup.coord_frame) self.assertEqual(self.exp_upd.hierarchy_method, eup.hierarchy_method) self.assertEqual(self.exp_upd.num_hierarchy_levels, eup.num_hierarchy_levels) self.assertEqual(self.exp_upd.num_time_samples, eup.num_time_samples) self.assertEqual(self.exp.time_step, eup.time_step) self.assertEqual(self.exp.time_step_unit, eup.time_step_unit) def test_update_channel(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) source_ch = self.rmt.create_project(self.source_chan) rel_ch = self.rmt.create_project(self.related_chan) chan = self.rmt.create_project(self.chan) ch = self.rmt.update_project(self.chan.name, self.chan_upd) self.assertEqual(self.chan_upd.name, ch.name) self.assertEqual(self.exp.name, ch.exp_name) self.assertEqual(self.chan_upd.description, ch.description) self.assertEqual(self.coll.name, ch.coll_name) self.assertEqual(self.chan_upd.datatype, ch.datatype) self.assertEqual(self.chan_upd.default_time_sample, ch.default_time_sample) self.assertEqual(self.chan_upd.base_resolution, ch.base_resolution) self.assertEqual(self.chan_upd.sources, ch.sources) self.assertEqual(self.chan_upd.related, ch.related) def test_list_collections(self): coll = self.rmt.create_project(self.coll) coll_list = self.rmt.list_collections() c = [name for name in coll_list if name == self.coll.name] self.assertEqual(1, len(c)) self.assertEqual(self.coll.name, c[0]) def test_list_coord_frames(self): cf = self.rmt.create_project(self.coord) cf_list = self.rmt.list_coordinate_frames() c = [name for name in cf_list if name == self.coord.name] self.assertEqual(1, len(c)) self.assertEqual(self.coord.name, c[0]) def test_list_experiments(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) exp = self.rmt.create_project(self.exp) exp_list = self.rmt.list_experiments(self.coll.name) e = [name for name in exp_list if name == self.exp.name] self.assertEqual(1, len(e)) self.assertEqual(self.exp.name, e[0]) #def test_list_channels(self): # c = self.rmt.create_project(self.coll) # cf = self.rmt.create_project(self.coord) # e = self.rmt.create_project(self.exp) # chan = self.rmt.create_project(self.chan) # chan_list = self.rmt.list_channels(self.coll.name, self.exp.name) # ch = [name for name in chan_list if name == self.chan.name] # self.assertEqual(1, len(ch)) # self.assertEqual(self.chan.name, ch[0]) def test_delete_all(self): """Formally test delete at all levels of the data model. Delete happens all the time in the tearDown() but specifically test it here. """ c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) ch = self.rmt.create_project(self.source_chan) self.rmt.delete_project(self.source_chan) self.rmt.delete_project(self.exp) self.rmt.delete_project(self.coord) self.rmt.delete_project(self.coll)
class NeuroDataResource: def __init__(self, host, token, collection, experiment): self._bossRemote = BossRemote({ 'protocol': 'https', 'host': host, 'token': token }) self.collection = collection self.experiment = experiment self.channels = self._bossRemote.list_channels(collection, experiment) self.channels.remove('empty') #Delete "empty" channel self._get_coord_frame_details() def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print( 'Error: You must supply zRange, yRange, xRange kwargs in list format' ) return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange, zRange) #Datatype check. Recast to original datatype if data is float64 if data.dtype == datatype: return data else: return data.astype(datatype)
class NeuroDataResource: def __init__(self, host, token, collection, experiment, requested_channels, x_range=None, y_range=None, z_range=None, resolution=0): self._bossRemote = BossRemote({ 'protocol': 'https', 'host': host, 'token': token }) self.collection = collection self.experiment = experiment self.resolution = resolution self.channels = self._bossRemote.list_channels(collection, experiment) if len(requested_channels) == 0: self.requested_channels = self.channels else: self.requested_channels = requested_channels self._get_coord_frame_details() # validate range #if not self.correct_range(z_range, y_range, x_range): # raise Exception("Error: Inccorect dimension range") self.x_range = x_range or [0, self.max_dimensions[2]] self.y_range = y_range or [0, self.max_dimensions[1]] self.z_range = z_range or [0, self.max_dimensions[0]] def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print( 'Error: You must supply zRange, yRange, xRange kwargs in list format' ) return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, self.resolution, xRange, yRange, zRange) #Datatype check. Recast to original datatype if data is float64 if data.dtype == datatype: return data else: return data.astype(datatype) def correct_range(self, z_range, y_range, x_range): x_start, x_end = x_range if x_start < 0 or x_end > self.max_dimensions[2]: return False y_start, y_end = y_range if y_start < 0 or y_end > self.max_dimensions[1]: return False z_start, z_end = z_range if z_start < 0 or z_end > self.max_dimensions[0]: return False return True
def boss_pull_cutout(args): if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop print("Data model setup.") xmin = np.max([x_min_bound, args.xmin - args.padding]) xmax = np.min([x_max_bound, args.xmax + args.padding]) x_rng = [xmin, xmax] ymin = np.max([y_min_bound, args.ymin - args.padding]) ymax = np.min([y_max_bound, args.ymax + args.padding]) y_rng = [ymin, ymax] zmin = np.max([z_min_bound, args.zmin - args.padding]) zmax = np.min([z_max_bound, args.zmax + args.padding]) z_rng = [zmin, zmax] # Verify that the cutout uploaded correctly. attempts = 0 while attempts < 3: try: cutout_data = rmt.get_cutout(chan_actual, args.res, x_rng, y_rng, z_rng) break except HTTPError as e: if attempts < 3: attempts += 1 print("Obtained HTTP error from server. Trial {}".format(attempts)) else: print("Failed 3 times: {}".format(e)) # Data will be in Z,Y,X format # Change to X,Y,Z for pipeline cutout_data = np.transpose(cutout_data, (2, 1, 0)) def _upload(f): print("Uploading to s3:/{}/{}".format(args.bucket, args.output)) s3 = boto3.resource("s3") f.seek(0, 0) s3.Object(args.bucket, args.output).put(Body=f) # Clean up. if args.bucket and args.s3_only: with tempfile.TemporaryFile() as f: np.save(f, cutout_data) _upload(f) else: with open(args.output, "w+b") as f: np.save(f, cutout_data) if args.bucket: _upload(f)
class NeuroDataResource: def __init__(self, host, token, collection, experiment): self._bossRemote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) self.collection = collection self.experiment = experiment self.channels = self._bossRemote.list_channels(collection, experiment) self.channels.remove('empty') #Delete "empty" channel self._get_coord_frame_details() def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print('Error: You must supply zRange, yRange, xRange kwargs in list format') return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange, zRange) return data @staticmethod def ingest_volume(host, token, channel_name, collection, experiment, volume): """ Assumes the collection and experiment exists in BOSS. """ remote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) if volume.dtype == 'uint64': dtype = 'uint64' img_type = 'annotation' sources = ['empty'] else: dtype = volume.dtype.name img_type = 'image' sources = [] try: channel_resource = ChannelResource(channel_name, collection, experiment) channel = remote.get_project(channel_resource) except: channel_resource = ChannelResource(channel_name, collection, experiment, type=img_type, sources=sources, datatype=dtype) channel = remote.create_project(channel_resource) #Get max size of experiment exp_resource = ExperimentResource(experiment, collection) coord_frame = remote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = remote.get_project(coord_frame_resource) y_stop, x_stop = data.y_stop, data.x_stop for z in range(volume.shape[0]): print('Uploading {} slice'.format(z)) remote.create_cutout(channel, 0, (0, x_stop), (0, y_stop), (z, z + 1), volume[z, :, :].reshape((-1, y_stop, x_stop)))
def main(COLL_NAME, EXP_NAME, COORD_FRAME, LOCATIONS, BF=[5, 5, 5], CHAN_NAMES=None, num_threads=6, CONFIG_FILE=None): L = genfromtxt(LOCATIONS, delimiter=',', skip_header=0, dtype='int32').tolist() m = morton.Morton(dimensions=3, bits=64) Lzo = sorted([m.pack(L[i][0], L[i][1], L[i][2]) for i in range(len(L))]) loc = np.asarray([m.unpack(l) for l in Lzo]) config = configparser.ConfigParser() config.read(CONFIG_FILE) TOKEN = config['Default']['token'] boss_url = ''.join((config['Default']['protocol'], '://', config['Default']['host'], '/v1/')) #intern rem = BossRemote(CONFIG_FILE) cfr = rem.get_project(CoordinateFrameResource(COORD_FRAME)) ext = { 'x': [cfr.x_start, cfr.x_stop], 'y': [cfr.y_start, cfr.y_stop], 'z': [cfr.z_start, cfr.z_stop] } ## remove coordinates that are outside buffer area. x_bnd = [ np.all([x[0] - BF[0] >= ext['x'][0], x[0] + BF[0] + 1 <= ext['x'][1]]) for x in loc ] y_bnd = [ np.all([y[1] - BF[1] >= ext['y'][0], y[1] + BF[1] + 1 <= ext['y'][1]]) for y in loc ] z_bnd = [ np.all([z[2] - BF[2] >= ext['z'][0], z[2] + BF[2] + 1 <= ext['z'][1]]) for z in loc ] xyzbnd = np.asarray( [np.all([x_bnd[i], y_bnd[i], z_bnd[i]]) for i in range(len(x_bnd))]) ## Select only locations that are buffered away from the edge. loc = loc[xyzbnd] ## Compute the buffered ranges x_rng = [[x[0] - BF[0], x[0] + BF[0] + 1] for x in loc] y_rng = [[y[1] - BF[1], y[1] + BF[1] + 1] for y in loc] z_rng = [[z[2] - BF[2], z[2] + BF[2] + 1] for z in loc] ChanList = [] for ch in CHAN_NAMES: di = [{ 'rem': rem, 'ch_rsc': ChannelResource(ch, COLL_NAME, EXP_NAME, 'image', datatype='uint8'), 'ch': ch, 'res': 0, 'loc': loc[i], 'xrng': x_rng[i], 'yrng': y_rng[i], 'zrng': z_rng[i], 'bf': BF } for i in range(len(loc))] with ThreadPool(num_threads) as tp: out = tp.map(toolbox.getCube, di) print(ch) ##DEBUG sys.stdout.flush() #DEBUG ChanList.append(np.asarray(out)) outArray = np.asarray(ChanList) ## outArray will be a numpy array in [channel, synapse, z, y, x] order ## this is C-ordering return (outArray, loc)
def gen_urls(args): config = {'protocol': 'https', 'host': args.hostname, 'token': args.token} boss = BossRemote(config) results = [] try: if args.collection is not None: collections = [args.collection] else: collections = boss.list_collections() for collection in collections: if args.experiment is not None: experiments = [args.experiment] else: experiments = boss.list_experiments(collection) for experiment in experiments: if args.channel is not None: channels = [args.channel] else: channels = boss.list_channels(collection, experiment) exp = ExperimentResource(name = experiment, collection_name = collection) exp = boss.get_project(exp) coord = CoordinateFrameResource(name = exp.coord_frame) coord = boss.get_project(coord) for channel in channels: ch = ChannelResource(name = channel, experiment_name = experiment, collection_name = collection) ch = boss.get_project(ch) def check_range(name, var, start, stop): start_, stop_ = map(int, var.split(':')) if start_ < start: fmt = "{} range start for {}/{}/{} is less than the coordinate frame, setting to minimum" print(fmt.format(name, collection, experiment, channel)) start_ = start if stop_ > stop: fmt = "{} range stop for {}/{}/{} is greater than the coordinate frame, setting to maximum" print(fmt.format(name, collection, experiment, channel)) stop_ = stop return '{}:{}'.format(start_, stop_) if args.x_range: x = check_range('X', args.x_range, coord.x_start, coord.x_stop) else: x = (coord.x_start, coord.x_stop, args.min, args.max) if args.y_range: y = check_range('Y', args.y_range, coord.y_start, coord.y_stop) else: y = (coord.y_start, coord.y_stop, args.min, args.max) if args.z_range: z = check_range('Z', args.z_range, coord.z_start, coord.z_stop) else: z = (coord.z_start, coord.z_stop, args.min, args.max) # Arguments to gen_url results.append((args.hostname, collection, experiment, channel, 0, x, y, z, None)) except Exception as e: print("Error generating URLs: {}".format(e)) return results
xmax = 8 ymax = 4 zmax = 5 tmax = 10 COLL_NAME = 'gray' EXP_NAME = 'timeseries_test' CHAN_NAME = 'Ch1' COORD_FRAME = COLL_NAME + '_' + EXP_NAME coord = CoordinateFrameResource( COORD_FRAME, '', 0, xmax, 0, ymax, 0, zmax) try: coord_actual = rmt.get_project(coord) except HTTPError: coord_actual = rmt.create_project(coord) # Create or get experiment chan_setup = ExperimentResource( EXP_NAME, COLL_NAME, coord_frame=COORD_FRAME, num_time_samples=tmax, time_step=1) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # Create or get a channel to write to
class bossHandler: """ This is a class for interacting with BOSS. It basically is a wrapper for Intern. """ def __init__(self, collection_name): """ Constructor: Takes the following argument: :param collection_name: name of the collection in BOSS :type collection_name: string """ self.collection_name = collection_name try: self.rmt = BossRemote() except: print('Unexpected Error:', sys.exc_info()[0]) # def get_collection_list(self): # return self.rmt.list_collections() def list_experiments(self): """ :return: all the experiments available in current collection """ exp_list = self.rmt.list_experiments(self.collection_name) return exp_list def select_experiment(self, experiment_name): """ Select an experiment to be added to this handler """ tmp = ExperimentResource(collection_name=self.collection_name, name=experiment_name) exp = self.rmt.get_project(tmp) self.experiment = exp # return exp def get_experiment(self): """ :return: the currently selected experiment for this handler """ if hasattr(self, 'experiment'): return self.experiment else: raise AttributeError( 'No experiment exists. First, select an experiment using select_experiment' ) def list_channels(self): """ :return: all channel in currently selected experiment """ return self.rmt.list_channels(self.collection_name, self.experiment.name) def select_channel(self, channel_name): """ Select a channel to be added to this handler """ self.channel = self.rmt.get_channel(chan_name=channel_name, coll_name=self.collection_name, exp_name=self.experiment.name) def get_coordinate_frame(self): """ Return: current experiment's coordinate frame """ tmp = CoordinateFrameResource(name=self.experiment.coord_frame) coor = self.rmt.get_project(tmp) self.coordinate_frame = coor return coor def get_all(self): """ :return: the entire channel image data at its native resolution """ x_rng = [self.coordinate_frame.x_start, self.coordinate_frame.x_stop] y_rng = [self.coordinate_frame.y_start, self.coordinate_frame.y_stop] z_rng = [self.coordinate_frame.z_start, self.coordinate_frame.z_stop] return self.rmt.get_cutout(self.channel, 0, x_rng, y_rng, z_rng) def get_cutout(self, x_range, y_range, z_range, resolution=0): """ :return: a cutout of the image data """ return self.rmt.get_cutout(self.channel, resolution, x_range, y_range, z_range)
grp_name = 'my_team' collection_name = "my_col" experiment_name = "my_exp" coord_frame_name = "my_coord" channel_name = "my_chan" # #### SETUP rmt = BossRemote("./boss.cfg") print('Creating group . . .') try: rmt.create_group(grp_name) except Exception as e: # Assume group already exists if an exception raised. print(e) # Get the resources collection = CollectionResource(collection_name) collection = rmt.get_project(collection) experiment = ExperimentResource(experiment_name, collection_name, coord_frame_name) experiment = rmt.get_project(experiment) channel = ChannelResource(channel_name, collection_name, experiment_name) channel = rmt.get_project(channel) # Add perms perms = ['read'] rmt.add_permissions(grp_name, collection, perms) rmt.add_permissions(grp_name, experiment, perms) rmt.add_permissions(grp_name, channel, perms)
class InternTileProcessor(TileProcessor): """A Tile processor for a single image file identified by z index""" def __init__(self): """Constructor to add custom class var""" TileProcessor.__init__(self) self.remote = None self.channel = None def setup(self, parameters): """ Method to load the file for uploading data. Assumes intern token is set via environment variable or config default file Args: parameters (dict): Parameters for the dataset to be processed MUST HAVE THE CUSTOM PARAMETERS: "x_offset": offset to apply when querying the Boss "y_offset": offset to apply when querying the Boss "z_offset": offset to apply when querying the Boss "x_tile": size of a tile in x dimension "y_tile": size of a tile in y dimension "collection": source collection "experiment": source experiment "channel": source channel "resolution": source resolution Returns: None """ self.parameters = parameters self.remote = BossRemote() self.channel = ChannelResource(self.parameters["channel"], self.parameters["collection"], self.parameters["experiment"]) self.channel = self.remote.get_project(self.channel) def process(self, file_path, x_index, y_index, z_index, t_index=0): """ Method to load the image file. Args: file_path(str): An absolute file path for the specified tile x_index(int): The tile index in the X dimension y_index(int): The tile index in the Y dimension z_index(int): The tile index in the Z dimension t_index(int): The time index Returns: (io.BufferedReader): A file handle for the specified tile """ # Compute cutout args x_rng = [ self.parameters["x_tile"] * x_index + self.parameters["x_offset"], self.parameters["x_tile"] * (x_index + 1) + self.parameters["x_offset"] ] y_rng = [ self.parameters["y_tile"] * y_index + self.parameters["y_offset"], self.parameters["y_tile"] * (y_index + 1) + self.parameters["y_offset"] ] z_rng = [ z_index + self.parameters["z_offset"], z_index + 1 + self.parameters["z_offset"] ] if z_index + self.parameters["z_offset"] < 0: data = np.zeros( (self.parameters["x_tile"], self.parameters["y_tile"]), dtype=np.int32, order="C") else: # Get data cnt = 0 while cnt < 5: try: data = self.remote.get_cutout( self.channel, self.parameters["resolution"], x_rng, y_rng, z_rng) data = np.asarray(data, np.uint32) break except Exception as err: if cnt > 5: raise err cnt += 1 time.sleep(10) # Save sub-img to png and return handle upload_img = Image.fromarray(np.squeeze(data)) output = six.BytesIO() upload_img.save(output, format="TIFF") # Send handle back return output
class BossResParams: def __init__(self, ingest_job, get_only=True): self.ingest_job = ingest_job self.coord_frame_name = '_'.join( (ingest_job.coll_name, ingest_job.exp_name)) self.rmt = BossRemote(self.ingest_job.boss_config_file) self.coll_resource = self.setup_boss_collection(get_only=get_only) self.coord_frame_resource = self.setup_boss_coord_frame( get_only=get_only) self.exp_resource = self.setup_boss_experiment(get_only=get_only) self.ch_resource = self.setup_boss_channel(get_only=get_only) def get_boss_project(self, proj_setup, get_only): try: proj_actual = self.rmt.get_project(proj_setup) except HTTPError as e: if get_only is False: try: proj_actual = self.rmt.create_project(proj_setup) except Exception as e: print(type(e), e) raise (e) else: print(type(e), e) raise e except Exception as e: print(type(e), e) raise (e) return proj_actual def setup_boss_collection(self, get_only=True): coll_setup = CollectionResource(self.ingest_job.coll_name) return self.get_boss_project(coll_setup, get_only) def setup_boss_coord_frame(self, get_only=True): # if we don't know the coordinate frame parameters, get the one with the same name if get_only: coord_setup = CoordinateFrameResource(self.coord_frame_name) else: coord_setup = CoordinateFrameResource( self.coord_frame_name, '', self.ingest_job.coord_frame_x_extent[0], self.ingest_job.coord_frame_x_extent[1], self.ingest_job.coord_frame_y_extent[0], self.ingest_job.coord_frame_y_extent[1], self.ingest_job.coord_frame_z_extent[0], self.ingest_job.coord_frame_z_extent[1], self.ingest_job.voxel_size[0], self.ingest_job.voxel_size[1], self.ingest_job.voxel_size[2], self.ingest_job.voxel_unit) coord_frame_resource = self.get_boss_project(coord_setup, get_only) if get_only: # matching ingest_job values to coordinate frame values (if they weren't specified, they are now populated) self.ingest_job.voxel_size = [ coord_frame_resource.x_voxel_size, coord_frame_resource.y_voxel_size, coord_frame_resource.z_voxel_size ] self.ingest_job.voxel_unit = coord_frame_resource.voxel_unit return coord_frame_resource def setup_boss_experiment(self, get_only=True): # if we don't know the coordinate frame parameters, get the one with the same name if get_only: exp_setup = ExperimentResource(self.ingest_job.exp_name, self.ingest_job.coll_name, self.coord_frame_name) else: # if all elements are the same, isotropic, otherwise anisotropic if len(set(self.ingest_job.voxel_size)) <= 1: hierarchy_method = 'isotropic' else: hierarchy_method = 'anisotropic' num_hierarchy_levels = self.calc_hierarchy_levels() exp_setup = ExperimentResource(self.ingest_job.exp_name, self.ingest_job.coll_name, self.coord_frame_name, '', num_hierarchy_levels, hierarchy_method) exp_resource = self.get_boss_project(exp_setup, get_only) # record the offset (if there is any) into BOSS metadata field for experiment self.record_offsets(exp_resource) return exp_resource def record_offsets(self, exp_resource): if any([a > 0 for a in self.ingest_job.offsets]): offsets_dict = {'offsets': self.ingest_job.offsets} try: self.rmt.create_metadata(exp_resource, offsets_dict) except HTTPErrorList: # keys already exist self.rmt.update_metadata(exp_resource, offsets_dict) def setup_boss_channel(self, ch_description='', get_only=True): ch_args = [ self.ingest_job.ch_name, self.ingest_job.coll_name, self.ingest_job.exp_name ] if not get_only: ch_args.extend( (self.ingest_job.ch_type, ch_description, 0, self.ingest_job.boss_datatype, self.ingest_job.res)) # annotation data if self.ingest_job.source_channel is not None: ch_args.append([self.ingest_job.source_channel]) # checking to make sure source channel exists (creating if needed) try: source_args = [ self.ingest_job.source_channel, self.ingest_job.coll_name, self.ingest_job.exp_name ] source_setup = ChannelResource(*source_args) self.get_boss_project(source_setup, True) except: self.ingest_job.send_msg( 'Creating a dummy source channel for this annotation because none exists yet' ) source_args.extend(('image', '', 0, 'uint8', 0)) source_setup = ChannelResource(*source_args) self.get_boss_project(source_setup, False) ch_setup = ChannelResource(*ch_args) ch_resource = self.get_boss_project(ch_setup, get_only) if get_only: self.ingest_job.boss_datatype = ch_resource.datatype if self.ingest_job.datatype is None: self.ingest_job.datatype = ch_resource.datatype self.ingest_job.res = ch_resource.base_resolution return ch_resource def calc_hierarchy_levels(self, lowest_res=512): img_size = [ self.coord_frame_resource.x_stop - self.coord_frame_resource.x_start, self.coord_frame_resource.y_stop - self.coord_frame_resource.y_start, self.coord_frame_resource.z_stop - self.coord_frame_resource.z_start ] max_xy = max(img_size[0:1]) # we add one because 0 is included in the number of downsampling levels num_levels = max(1, math.ceil(math.log(max_xy / lowest_res, 2)) + 1) return num_levels