def set_experiment_resource(remote: BossRemote, params: dict) -> ExperimentResource: """Use the arguments in the class config to create an experiment resource object""" if 'name' not in params: params['name'] = 'experiment{0}'.format(hex(round(time.time()))[2:]) param_names = [str(p.name) for p in inspect.signature(ExperimentResource).parameters.values()] filtered_params = {k: v for k, v in list(params.items()) if k in param_names} # Filter unexpected arguments exp_resource = ExperimentResource(**filtered_params) if exp_resource.name in remote.list_experiments(exp_resource.coll_name): experiment = remote.update_project(exp_resource.name, exp_resource) print('Updated experiment {0}'.format(exp_resource.name)) else: experiment = remote.create_project(exp_resource) print('Created experiment {0}'.format(exp_resource.name)) return experiment
def set_channel_resource(remote: BossRemote, params: dict) -> ChannelResource: """Use the arguments in the class config to create a channel resource object""" if 'name' not in params: params['name'] = 'channel{0}'.format(hex(round(time.time()))[2:]) param_names = [str(p.name) for p in inspect.signature(ChannelResource).parameters.values()] filtered_params = {k: v for k, v in list(params.items()) if k in param_names} # Filter unexpected arguments chan_resource = ChannelResource(**filtered_params) if chan_resource.name in remote.list_channels(chan_resource.coll_name, chan_resource.exp_name): channel = remote.update_project(chan_resource.name, chan_resource) print('Updated channel {0}'.format(chan_resource.name)) else: channel = remote.create_project(chan_resource) print('Created channel {0}'.format(chan_resource.name)) return channel
def set_coordinate_frame_resource(remote: BossRemote, params: dict) -> CoordinateFrameResource: """Use the arguments in the class config to create a frame resource object""" if 'name' not in params: params['name'] = 'frame{0}'.format(hex(round(time.time()))[2:]) param_names = [str(p.name) for p in inspect.signature(CoordinateFrameResource).parameters.values()] filtered_params = {k: v for k, v in list(params.items()) if k in param_names} # Filter unexpected arguments frame_resource = CoordinateFrameResource(**filtered_params) if frame_resource.name in remote.list_coordinate_frames(): coordinate_frame = remote.update_project(frame_resource.name, frame_resource) print('Updated frame {0}'.format(frame_resource.name)) else: coordinate_frame = remote.create_project(frame_resource) print('Created frame {0}'.format(frame_resource.name)) return coordinate_frame
def ingest_volume(host, token, channel_name, collection, experiment, volume): """ Assumes the collection and experiment exists in BOSS. """ remote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) if volume.dtype == 'uint64': dtype = 'uint64' img_type = 'annotation' sources = ['empty'] else: dtype = volume.dtype.name img_type = 'image' sources = [] try: channel_resource = ChannelResource(channel_name, collection, experiment) channel = remote.get_project(channel_resource) except: channel_resource = ChannelResource(channel_name, collection, experiment, type=img_type, sources=sources, datatype=dtype) channel = remote.create_project(channel_resource) #Get max size of experiment exp_resource = ExperimentResource(experiment, collection) coord_frame = remote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = remote.get_project(coord_frame_resource) y_stop, x_stop = data.y_stop, data.x_stop for z in range(volume.shape[0]): print('Uploading {} slice'.format(z)) remote.create_cutout(channel, 0, (0, x_stop), (0, y_stop), (z, z + 1), volume[z, :, :].reshape((-1, y_stop, x_stop)))
coll_data = {'poc': 'Jane Doe'} local.create_metadata(Channel1, coll_data) exp_data = {'weight': '20g', 'diet': 'C2', 'date': '23-May-2016'} local.create_metadata(Collection1, exp_data) chan_new_data = {'weight': '45g', 'date': '23-May-2017'} local.update_metadata(Collection1, chan_new_data) local.delete_metadata(Channel1, ['poc']) CollectionMeta = local.list_metadata(Collection1) print(CollectionMeta) #Data processing can also be done here before re-upload #Local to Boss upload of annotated data chan_setup = get_channel('CHAN_NAME', 'COLL_NAME', 'EXP_NAME', 'image', datatype='uint16') chan = BossRemote.create_project(chan_setup) xspan = [0, 200] yspan = [0, 90] zspan = [0, 20] VolumeLmeta.astype(numpy.uint16) BossRemote.create_cutout(chan, 0, xspan, yspan, zspan, VolumeLMeta)
def boss_pull_cutout(args): if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop print("Data model setup.") xmin = np.max([x_min_bound, args.xmin - args.padding]) xmax = np.min([x_max_bound, args.xmax + args.padding]) x_rng = [xmin, xmax] ymin = np.max([y_min_bound, args.ymin - args.padding]) ymax = np.min([y_max_bound, args.ymax + args.padding]) y_rng = [ymin, ymax] zmin = np.max([z_min_bound, args.zmin - args.padding]) zmax = np.min([z_max_bound, args.zmax + args.padding]) z_rng = [zmin, zmax] # Verify that the cutout uploaded correctly. attempts = 0 while attempts < 3: try: cutout_data = rmt.get_cutout(chan_actual, args.res, x_rng, y_rng, z_rng) break except HTTPError as e: if attempts < 3: attempts += 1 print("Obtained HTTP error from server. Trial {}".format(attempts)) else: print("Failed 3 times: {}".format(e)) # Data will be in Z,Y,X format # Change to X,Y,Z for pipeline cutout_data = np.transpose(cutout_data, (2, 1, 0)) def _upload(f): print("Uploading to s3:/{}/{}".format(args.bucket, args.output)) s3 = boto3.resource("s3") f.seek(0, 0) s3.Object(args.bucket, args.output).put(Body=f) # Clean up. if args.bucket and args.s3_only: with tempfile.TemporaryFile() as f: np.save(f, cutout_data) _upload(f) else: with open(args.output, "w+b") as f: np.save(f, cutout_data) if args.bucket: _upload(f)
def boss_merge_xbrain(args): # Verify that the cutout uploaded correctly. def pull_margin_cutout(chan_actual, x_rng, y_rng, z_rng): attempts = 0 while attempts < 3: try: cutout_data = rmt.get_cutout(chan_actual, 0, x_rng, y_rng, z_rng) break except HTTPError as e: if attempts < 3: attempts += 1 print("Obtained HTTP error from server. Trial {}".format(attempts)) else: print("Failed 3 times: {}".format(e)) # Data will be in Z,Y,X format # Change to X,Y,Z for pipeline cutout_data = np.transpose(cutout_data, (2, 1, 0)) return cutout_data templatesize = args.templatesize if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") # data is desired range if args.bucket: s3 = boto3.resource("s3") with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.input, f) f.seek(0, 0) data = np.load(f) with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.centroids, f) f.seek(0, 0) centroids = np.load(f) else: data = np.load(args.input) centroids = np.load(args.centroids) COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop # coordinates of data block in original coordinate frame, before padding x_block = [args.xmin, args.xmax] y_block = [args.ymin, args.ymax] z_block = [args.zmin, args.zmax] # Coordinates of data block with padding in original coordinate frame x_block_pad = [ np.amax([args.xmin - args.padding, x_min_bound]), np.amin([args.xmax + args.padding, x_max_bound]), ] y_block_pad = [ np.amax([args.ymin - args.padding, y_min_bound]), np.amin([args.ymax + args.padding, y_max_bound]), ] z_block_pad = [ np.amax([args.zmin - args.padding, z_min_bound]), np.amin([args.zmax + args.padding, z_max_bound]), ] # Coordinates of core data block in local coordinate frame xstart = np.amin([args.padding, args.xmin - x_min_bound]) xend = np.amax( [data.shape[0] - args.padding, data.shape[0] - (x_max_bound - args.xmax)] ) ystart = np.amin([args.padding, args.ymin - y_min_bound]) yend = np.amax( [data.shape[1] - args.padding, data.shape[1] - (y_max_bound - args.ymax)] ) zstart = np.amin([args.padding, args.zmin - z_min_bound]) zend = np.amax( [data.shape[2] - args.padding, data.shape[2] - (z_max_bound - args.zmax)] ) print("Data model setup.") # Template size to decide which centroids to eliminate # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. if args.onesided: # Only merge on the max side, to prevent duplication of detection # Binarize Map data[np.where(data > 0)] = 1 # Search through centroids # On side of max values, keep anything where centroid is in padded region # On side of min values, remove anything that is partially in padded region (covered by another block) n_centers, _ = centroids.shape # n by 4 bad_inds = [] for i in range(0, n_centers): if centroids[i, 0] < xstart or centroids[i, 0] - templatesize / 2 > xend: bad_inds.append(i) elif centroids[i, 1] < ystart or centroids[i, 1] - templatesize / 2 > yend: bad_inds.append(i) elif centroids[i, 2] < zstart or centroids[i, 2] - templatesize / 2 > zend: bad_inds.append(i) centroids_out = np.delete(centroids, bad_inds, axis=0) # translate into global coordinates from local data block centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin # Eliminate any cells form data which overlap with the padding edge for ind in bad_inds: xi = np.array( [ centroids[ind, 0] - np.ceil(templatesize / 2), centroids[ind, 0] + np.ceil(templatesize / 2), ] ).astype(int) yi = np.array( [ centroids[ind, 1] - np.ceil(templatesize / 2), centroids[ind, 1] + np.ceil(templatesize / 2), ] ).astype(int) zi = np.array( [ centroids[ind, 2] - np.ceil(templatesize / 2), centroids[ind, 2] + np.ceil(templatesize / 2), ] ).astype(int) data[xi, yi, zi] = 0 # Keep any interior cells, any which overlap original boundary and not padding # Pull down existing boundary area, if area is valid # Test side 4 if ( xend < data.shape[0] ): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[1], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 5 if ( yend < data.shape[1] ): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[1], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 6 if ( zend < data.shape[2] ): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[1], z_block_pad[1]], ) data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ], margin, ) # push results over entire padded area # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = data[ xstart : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] data = np.transpose(data, (2, 1, 0)) data = data.copy(order="C").astype(eval("np.{}".format(args.dtype))) # Verify that the cutout uploaded correctly. rmt.create_cutout( chan_actual, 0, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], data, ) # Clean up. else: # Binarize Map data[np.where(data > 0)] = 1 # Search through centroids n_centers, _ = centroids.shape # n by 4 bad_inds = [] for i in range(0, n_centers): if ( centroids[i, 0] + templatesize / 2 < xstart or centroids[i, 0] - templatesize / 2 > xend ): bad_inds.append(i) elif ( centroids[i, 1] + templatesize / 2 < ystart or centroids[i, 1] - templatesize / 2 > yend ): bad_inds.append(i) elif ( centroids[i, 2] + templatesize / 2 < zstart or centroids[i, 2] - templatesize / 2 > zend ): bad_inds.append(i) centroids_out = np.delete(centroids, bad_inds, axis=0) # translate into global coordinates from local data block centroids_out[:, 0] = centroids_out[:, 0] - xstart + args.xmin centroids_out[:, 1] = centroids_out[:, 1] - ystart + args.ymin centroids_out[:, 2] = centroids_out[:, 2] - zstart + args.zmin # Eliminate any cells form data which overlap with the padding edge for ind in bad_inds: xi = np.array( [ centroids[ind, 0] - np.ceil(templatesize / 2), centroids[ind, 0] + np.ceil(templatesize / 2), ] ).astype(int) yi = np.array( [ centroids[ind, 1] - np.ceil(templatesize / 2), centroids[ind, 1] + np.ceil(templatesize / 2), ] ).astype(int) zi = np.array( [ centroids[ind, 2] - np.ceil(templatesize / 2), centroids[ind, 2] + np.ceil(templatesize / 2), ] ).astype(int) data[xi, yi, zi] = 0 # Keep any interior cells, any which overlap original boundary and not padding # Pull down existing boundary area, if area is valid # Test side 1 if ( xstart > 0 ): # There is padding on side 1 of cube [0:pad,0:ymax+2*pad,0:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block_pad[0], x_block[0]], y_block_pad, z_block_pad ) data[0:xstart, :, :] = np.maximum(data[0:xstart, :, :], margin) # Test side 2 if ( ystart > 0 ): # There is padding on side 2 of cube [pad:xmax+2*pad,0:pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block_pad[0], y_block[0]], [z_block[0], z_block_pad[1]], ) data[xstart : data.shape[0], 0:ystart, zstart : data.shape[2]] = np.maximum( data[xstart : data.shape[0], 0:ystart, zstart : data.shape[1]], margin ) # Test side 3 if ( zstart > 0 ): # There is padding on side 3 of cube [pad:xmax+2*pad,pad:ymax+2*pad,0:pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block_pad[0], z_block[0]], ) data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart] = np.maximum( data[xstart : data.shape[0], ystart : data.shape[1], 0:zstart], margin ) # Test side 4 if ( xend < data.shape[0] ): # There is padding on side 4 of cube [xmax+pad:xmax+2*pad,pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[1], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xend : data.shape[0], ystart : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 5 if ( yend < data.shape[1] ): # There is padding on side 5 of cube [pad:xmax+2*pad,ymax+pad:ymax+2*pad,pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[1], y_block_pad[1]], [z_block[0], z_block_pad[1]], ) data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], yend : data.shape[1], zstart : data.shape[2] ], margin, ) # Test side 6 if ( zend < data.shape[2] ): # There is padding on side 4 of cube [pad:xmax+2*pad,pad:ymax+2*pad,zmax+pad:zmax+2*pad] margin = pull_margin_cutout( chan_actual, [x_block[0], x_block_pad[1]], [y_block[0], y_block_pad[1]], [z_block[1], z_block_pad[1]], ) data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ] = np.maximum( data[ xstart : data.shape[0], ystart : data.shape[1], zend : data.shape[2] ], margin, ) # push results over entire padded area # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = np.transpose(data, (2, 1, 0)) data = data.copy(order="C").astype(eval("np.{}".format(args.dtype))) # Verify that the cutout uploaded correctly. rmt.create_cutout(chan_actual, 0, x_block_pad, y_block_pad, z_block_pad, data) # Clean up. def _upload(f): print("Uploading to s3:/{}/{}".format(args.bucket, args.output)) s3 = boto3.resource("s3") f.seek(0, 0) s3.Object(args.bucket, args.output).put(Body=f) # Clean up. if args.bucket and args.s3_only: with tempfile.TemporaryFile() as f: np.save(f, centroids_out) _upload(f) else: print("Saving output") with open(args.output, "w+b") as f: np.save(f, centroids_out) if args.bucket: _upload(f) return
def boss_push_cutout(args): if args.config: rmt = BossRemote(args.config) else: cfg = _generate_config(args.token, args) with open("intern.cfg", "w") as f: cfg.write(f) rmt = BossRemote("intern.cfg") # data is desired range if args.bucket: s3 = boto3.resource("s3") with tempfile.TemporaryFile() as f: s3.Bucket(args.bucket).download_fileobj(args.input, f) f.seek(0, 0) data = np.load(f) else: data = np.load(args.input) numpyType = np.uint8 if args.dtype == "uint32": numpyType = np.uint32 elif args.dtype == "uint64": numpyType = np.uint64 if data.dtype != args.dtype: data = data.astype(numpyType) sources = [] if args.source: sources.append(args.source) COLL_NAME = args.coll EXP_NAME = args.exp CHAN_NAME = args.chan # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, type=args.itype, datatype=args.dtype, sources=sources, ) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # get coordinate frame to determine padding bounds cfr = CoordinateFrameResource(args.coord) cfr_actual = rmt.get_project(cfr) x_min_bound = cfr_actual.x_start x_max_bound = cfr_actual.x_stop y_min_bound = cfr_actual.y_start y_max_bound = cfr_actual.y_stop z_min_bound = cfr_actual.z_start z_max_bound = cfr_actual.z_stop print("Data model setup.") # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. data_shape = data.shape # with padding, will be bigger than needed # find data cutoffs to get rid of padding # if nmin = 0, this means that the data wasn't padded on there to begin with xstart = args.padding if args.xmin != 0 else 0 ystart = args.padding if args.ymin != 0 else 0 zstart = args.padding if args.zmin != 0 else 0 xend = data_shape[0] - args.padding yend = data_shape[1] - args.padding zend = data_shape[2] - args.padding # xstart = np.min([args.padding,args.xmin-x_min_bound]) # xend = np.max([data.shape[0]-args.padding,data.shape[0]-(x_max_bound-args.xmax)]) # ystart = np.min([args.padding,args.ymin-y_min_bound]) # yend = np.max([data.shape[1]-args.padding,data.shape[1]-(y_max_bound-args.ymax)]) # zstart = np.min([args.padding,args.zmin-z_min_bound]) # zend = np.max([data.shape[2]-args.padding,data.shape[2]-(z_max_bound-args.zmax)]) # get range which will be uploaded x_rng = [args.xmin, args.xmax] y_rng = [args.ymin, args.ymax] z_rng = [args.zmin, args.zmax] # Pipeline Data will be in X,Y,Z format # Change to Z,Y,X for upload data = np.transpose(data, (2, 1, 0)) data = data[zstart:zend, ystart:yend, xstart:xend] data = data.copy(order="C") # Verify that the cutout uploaded correctly. attempts = 0 while attempts < 3: try: rmt.create_cutout(chan_actual, args.res, x_rng, y_rng, z_rng, data) break except HTTPError as e: if attempts < 3: attempts += 1 print("These are the dimensions: ") print(data.shape) print("This is the data type:") print(data.dtype) print("Specified data type was:") print(args.dtype) print("Specified image type") print(args.itype) print("Obtained HTTP error from server. Trial {}".format(attempts)) print("The error: {}".format(e)) else: raise Exception("Failed 3 times: {}".format(e))
COLL_NAME = 'gray' EXP_NAME = 'alpha' CHAN_NAME = 'ex_EM' # Create or get a channel to write to chan_setup = ChannelResource(CHAN_NAME, COLL_NAME, EXP_NAME, 'image', 'Example channel.', datatype='uint16') try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) print('Data model setup.') # Ranges use the Python convention where the number after the : is the stop # value. Thus, x_rng specifies x values where: 0 <= x < 8. x_rng = [0, 8] y_rng = [0, 4] z_rng = [0, 5] # Note that the numpy matrix is in Z, Y, X order. data = numpy.random.randint(1, 3000, (5, 4, 8)) data = data.astype(numpy.uint16) # Upload the cutout to the channel. rmt.create_cutout(chan_actual, 0, x_rng, y_rng, z_rng, data)
This script sets up the data model resource used by the example scripts. To run this example (and create new resources), you must have a user with the resource-manager role! """ from intern.remote.boss import BossRemote from intern.resource.boss.resource import * from requests import HTTPError rmt = BossRemote('example.cfg') coll = CollectionResource('gray', 'Collection used for examples.') try: rmt.get_project(coll) except HTTPError: rmt.create_project(coll) coord = CoordinateFrameResource( 'StdFrame', 'Standard coordinate frame for xyz.', 0, 2048, 0, 2048, 0, 64) try: coord_actual = rmt.get_project(coord) except HTTPError: coord_actual = rmt.create_project(coord) alpha_exp = ExperimentResource( 'alpha', 'gray', coord_actual.name, 'Alpha example experiment.', num_time_samples=10) try: rmt.get_project(alpha_exp) except HTTPError: rmt.create_project(alpha_exp)
class ProjectServiceTest_v1(unittest.TestCase): """Integration tests of the Boss resource API. """ def setUp(self): self.rmt = BossRemote('test.cfg', API_VER) # Turn off SSL cert verification. This is necessary for interacting with # developer instances of the Boss. self.rmt.project_service.session_send_opts = {'verify': False} self.rmt.metadata_service.session_send_opts = {'verify': False} self.rmt.volume_service.session_send_opts = {'verify': False} requests.packages.urllib3.disable_warnings(InsecureRequestWarning) coll_name = 'collection2309-{}'.format(random.randint(0, 9999)) self.coll = CollectionResource(coll_name, 'bar') coll_name_upd = '{}-{}'.format(coll_name, random.randint(0, 9999)) self.coll_upd = CollectionResource(coll_name_upd, 'latest') cf_name = 'ProjTestFrame{}'.format(random.randint(0, 9999)) self.coord = CoordinateFrameResource(cf_name, 'Test coordinate frame.', 0, 10, -5, 5, 3, 6, 1, 1, 1, 'nanometers') self.coord_upd = copy.copy(self.coord) self.coord_upd.name = 'MouseFrame{}'.format(random.randint(0, 9999)) self.coord_upd.description = 'Mouse coordinate frame.' self.exp = ExperimentResource('exp2309-2', self.coll.name, self.coord.name, 'my experiment', 1, 'isotropic', time_step=2, time_step_unit='nanoseconds') self.exp_upd = ExperimentResource('exp2309-2a', self.coll.name, self.coord.name, 'my first experiment', 2, 'anisotropic') self.source_chan = ChannelResource('sourceChan', self.coll.name, self.exp.name, 'image', 'test source channel', 0, 'uint8', 0) self.related_chan = ChannelResource('relatedChan', self.coll.name, self.exp.name, 'image', 'test related channel', 0, 'uint8', 0) self.chan = ChannelResource('myChan', self.coll.name, self.exp.name, 'annotation', 'test annotation channel', 0, 'uint8', 0, sources=['sourceChan'], related=['relatedChan']) self.chan_upd = ChannelResource('yourChan', self.coll.name, self.exp.name, 'annotation', 'your test annotation channel', 0, 'uint8', 1, sources=['sourceChan'], related=['relatedChan']) def tearDown(self): try: self.rmt.delete_project(self.chan_upd) except HTTPError: pass try: self.rmt.delete_project(self.chan) except HTTPError: pass try: self.rmt.delete_project(self.related_chan) except HTTPError: pass try: self.rmt.delete_project(self.source_chan) except HTTPError: pass try: self.rmt.delete_project(self.exp_upd) except HTTPError: pass try: self.rmt.delete_project(self.exp) except HTTPError: pass try: self.rmt.delete_project(self.coord_upd) except HTTPError: pass try: self.rmt.delete_project(self.coord) except HTTPError: pass try: self.rmt.delete_project(self.coll_upd) except HTTPError: pass try: self.rmt.delete_project(self.coll) except HTTPError: pass def test_create_coord_frame(self): cf = self.rmt.create_project(self.coord) self.assertEqual(self.coord.name, cf.name) self.assertEqual(self.coord.description, cf.description) self.assertEqual(self.coord.x_start, cf.x_start) self.assertEqual(self.coord.x_stop, cf.x_stop) self.assertEqual(self.coord.y_start, cf.y_start) self.assertEqual(self.coord.y_stop, cf.y_stop) self.assertEqual(self.coord.z_start, cf.z_start) self.assertEqual(self.coord.z_stop, cf.z_stop) self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size) self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size) self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size) self.assertEqual(self.coord.voxel_unit, cf.voxel_unit) def test_create_collection(self): c = self.rmt.create_project(self.coll) self.assertEqual(self.coll.name, c.name) self.assertEqual(self.coll.description, c.description) def test_create_experiment(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) self.assertEqual(self.exp.name, e.name) self.assertEqual(self.exp.description, e.description) self.assertEqual(self.coll.name, e.coll_name) self.assertEqual(self.exp.coord_frame, e.coord_frame) self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method) self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels) self.assertEqual(self.exp.num_time_samples, e.num_time_samples) self.assertEqual(self.exp.time_step, e.time_step) self.assertEqual(self.exp.time_step_unit, e.time_step_unit) def test_create_channel(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) ch = self.rmt.create_project(self.source_chan) self.assertEqual(self.source_chan.name, ch.name) self.assertEqual(self.exp.name, ch.exp_name) self.assertEqual(self.source_chan.description, ch.description) self.assertEqual(self.coll.name, ch.coll_name) self.assertEqual(self.source_chan.datatype, ch.datatype) self.assertEqual(self.source_chan.default_time_sample, ch.default_time_sample) self.assertEqual(self.source_chan.base_resolution, ch.base_resolution) def test_create_annotation_channel_without_source_fails(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) rel_ch = self.rmt.create_project(self.related_chan) chan = ChannelResource('myChan', self.coll.name, self.exp.name, 'annotation', 'test annotation channel', 0, 'uint8', 0, related=['relatedChan']) with self.assertRaises(HTTPError): self.rmt.create_project(chan) def test_create_annotation_channel(self): """Annotation channels require a source channel.""" c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) ch = self.rmt.create_project(self.source_chan) rel_ch = self.rmt.create_project(self.related_chan) ann_ch = self.rmt.create_project(self.chan) self.assertEqual(self.chan.name, ann_ch.name) self.assertEqual(self.exp.name, ann_ch.exp_name) self.assertEqual(self.chan.description, ann_ch.description) self.assertEqual(self.coll.name, ann_ch.coll_name) self.assertEqual(self.chan.datatype, ann_ch.datatype) self.assertEqual(self.chan.default_time_sample, ann_ch.default_time_sample) self.assertEqual(self.chan.base_resolution, ann_ch.base_resolution) self.assertEqual(self.chan.sources, ann_ch.sources) self.assertEqual(self.chan.related, ann_ch.related) def test_get_collection(self): coll = self.rmt.create_project(self.coll) c = self.rmt.get_project(self.coll) self.assertEqual(self.coll.name, c.name) self.assertEqual(self.coll.description, c.description) def test_get_coord_frame(self): coord = self.rmt.create_project(self.coord) cf = self.rmt.get_project(self.coord) self.assertEqual(self.coord.name, cf.name) self.assertEqual(self.coord.description, cf.description) self.assertEqual(self.coord.x_start, cf.x_start) self.assertEqual(self.coord.x_stop, cf.x_stop) self.assertEqual(self.coord.y_start, cf.y_start) self.assertEqual(self.coord.y_stop, cf.y_stop) self.assertEqual(self.coord.z_start, cf.z_start) self.assertEqual(self.coord.z_stop, cf.z_stop) self.assertEqual(self.coord.x_voxel_size, cf.x_voxel_size) self.assertEqual(self.coord.y_voxel_size, cf.y_voxel_size) self.assertEqual(self.coord.z_voxel_size, cf.z_voxel_size) self.assertEqual(self.coord.voxel_unit, cf.voxel_unit) def test_get_experiment(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) exp = self.rmt.create_project(self.exp) e = self.rmt.get_project(self.exp) self.assertEqual(self.exp.name, e.name) self.assertEqual(self.exp.description, e.description) self.assertEqual(self.coll.name, e.coll_name) self.assertEqual(self.exp.coord_frame, e.coord_frame) self.assertEqual(self.exp.hierarchy_method, e.hierarchy_method) self.assertEqual(self.exp.num_hierarchy_levels, e.num_hierarchy_levels) self.assertEqual(self.exp.num_time_samples, e.num_time_samples) self.assertEqual(self.exp.time_step, e.time_step) self.assertEqual(self.exp.time_step_unit, e.time_step_unit) def test_get_channel(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) chan = self.rmt.create_project(self.source_chan) ch = self.rmt.get_project(self.source_chan) self.assertEqual(self.source_chan.name, ch.name) self.assertEqual(self.exp.name, ch.exp_name) self.assertEqual(self.source_chan.description, ch.description) self.assertEqual(self.coll.name, ch.coll_name) self.assertEqual(self.source_chan.datatype, ch.datatype) self.assertEqual(self.source_chan.default_time_sample, ch.default_time_sample) self.assertEqual(self.source_chan.base_resolution, ch.base_resolution) def test_get_channel_helper(self): """ Test the helper get_channel() as opposed to getting a channel via get_project(). """ c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) chan = self.rmt.create_project(self.source_chan) actual = self.rmt.get_channel(chan.name, chan.coll_name, chan.exp_name) # This is not an exhaustive list of attributes, but they are the # important ones for correct interaction with the volume service. self.assertTrue(actual.cutout_ready) self.assertEqual(chan.datatype, actual.datatype) self.assertEqual(chan.default_time_sample, actual.default_time_sample) self.assertEqual(chan.base_resolution, actual.base_resolution) self.assertEqual(chan.downsample_status, actual.downsample_status) self.assertEqual(chan.type, actual.type) self.assertEqual(chan.name, actual.name) self.assertEqual(chan.coll_name, actual.coll_name) self.assertEqual(chan.exp_name, actual.exp_name) def test_update_collection(self): coll = self.rmt.create_project(self.coll) c = self.rmt.update_project(self.coll.name, self.coll_upd) self.assertEqual(self.coll_upd.name, c.name) self.assertEqual(self.coll_upd.description, c.description) def test_update_coord_frame(self): c = self.rmt.create_project(self.coll) coord = self.rmt.create_project(self.coord) cf = self.rmt.update_project(self.coord.name, self.coord_upd) self.assertEqual(self.coord_upd.name, cf.name) self.assertEqual(self.coord_upd.description, cf.description) def test_update_experiment(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) eup = self.rmt.update_project(self.exp.name, self.exp_upd) self.assertEqual(self.exp_upd.name, eup.name) self.assertEqual(self.exp_upd.description, eup.description) self.assertEqual(self.coll.name, eup.coll_name) self.assertEqual(self.exp_upd.coord_frame, eup.coord_frame) self.assertEqual(self.exp_upd.hierarchy_method, eup.hierarchy_method) self.assertEqual(self.exp_upd.num_hierarchy_levels, eup.num_hierarchy_levels) self.assertEqual(self.exp_upd.num_time_samples, eup.num_time_samples) self.assertEqual(self.exp.time_step, eup.time_step) self.assertEqual(self.exp.time_step_unit, eup.time_step_unit) def test_update_channel(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) source_ch = self.rmt.create_project(self.source_chan) rel_ch = self.rmt.create_project(self.related_chan) chan = self.rmt.create_project(self.chan) ch = self.rmt.update_project(self.chan.name, self.chan_upd) self.assertEqual(self.chan_upd.name, ch.name) self.assertEqual(self.exp.name, ch.exp_name) self.assertEqual(self.chan_upd.description, ch.description) self.assertEqual(self.coll.name, ch.coll_name) self.assertEqual(self.chan_upd.datatype, ch.datatype) self.assertEqual(self.chan_upd.default_time_sample, ch.default_time_sample) self.assertEqual(self.chan_upd.base_resolution, ch.base_resolution) self.assertEqual(self.chan_upd.sources, ch.sources) self.assertEqual(self.chan_upd.related, ch.related) def test_list_collections(self): coll = self.rmt.create_project(self.coll) coll_list = self.rmt.list_collections() c = [name for name in coll_list if name == self.coll.name] self.assertEqual(1, len(c)) self.assertEqual(self.coll.name, c[0]) def test_list_coord_frames(self): cf = self.rmt.create_project(self.coord) cf_list = self.rmt.list_coordinate_frames() c = [name for name in cf_list if name == self.coord.name] self.assertEqual(1, len(c)) self.assertEqual(self.coord.name, c[0]) def test_list_experiments(self): c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) exp = self.rmt.create_project(self.exp) exp_list = self.rmt.list_experiments(self.coll.name) e = [name for name in exp_list if name == self.exp.name] self.assertEqual(1, len(e)) self.assertEqual(self.exp.name, e[0]) #def test_list_channels(self): # c = self.rmt.create_project(self.coll) # cf = self.rmt.create_project(self.coord) # e = self.rmt.create_project(self.exp) # chan = self.rmt.create_project(self.chan) # chan_list = self.rmt.list_channels(self.coll.name, self.exp.name) # ch = [name for name in chan_list if name == self.chan.name] # self.assertEqual(1, len(ch)) # self.assertEqual(self.chan.name, ch[0]) def test_delete_all(self): """Formally test delete at all levels of the data model. Delete happens all the time in the tearDown() but specifically test it here. """ c = self.rmt.create_project(self.coll) cf = self.rmt.create_project(self.coord) e = self.rmt.create_project(self.exp) ch = self.rmt.create_project(self.source_chan) self.rmt.delete_project(self.source_chan) self.rmt.delete_project(self.exp) self.rmt.delete_project(self.coord) self.rmt.delete_project(self.coll)
ymax = 4 zmax = 5 tmax = 10 COLL_NAME = 'gray' EXP_NAME = 'timeseries_test' CHAN_NAME = 'Ch1' COORD_FRAME = COLL_NAME + '_' + EXP_NAME coord = CoordinateFrameResource( COORD_FRAME, '', 0, xmax, 0, ymax, 0, zmax) try: coord_actual = rmt.get_project(coord) except HTTPError: coord_actual = rmt.create_project(coord) # Create or get experiment chan_setup = ExperimentResource( EXP_NAME, COLL_NAME, coord_frame=COORD_FRAME, num_time_samples=tmax, time_step=1) try: chan_actual = rmt.get_project(chan_setup) except HTTPError: chan_actual = rmt.create_project(chan_setup) # Create or get a channel to write to chan_setup = ChannelResource( CHAN_NAME, COLL_NAME, EXP_NAME, 'image', '', datatype='uint16')
from intern.remote.boss import BossRemote from intern.resource.boss.resource import * CONFIG_FILE = "demo_cfg.json" rmt = BossRemote("./boss.cfg") # Load Configuration File with open(os.path.join("./db_configs", CONFIG_FILE), 'rt') as cfg: config = json.load(cfg) # Create a collection collection = CollectionResource(config["collection"]["name"], config["collection"]["description"]) try: collection = rmt.create_project(collection) except Exception as e: collection = rmt.get_project(collection) # Create a coord frame coord = CoordinateFrameResource(config["coordinate_frame"]["name"], config["coordinate_frame"]["description"], config["coordinate_frame"]["x_start"], config["coordinate_frame"]["x_stop"], config["coordinate_frame"]["y_start"], config["coordinate_frame"]["y_stop"], config["coordinate_frame"]["z_start"], config["coordinate_frame"]["z_stop"], config["coordinate_frame"]["voxel_size_x"], config["coordinate_frame"]["voxel_size_y"], config["coordinate_frame"]["voxel_size_z"])
coll_list = rmt.list_collections() print("Collections: {}".format(coll_list)) # For resources below the collection level, the parents must be specified. # For example to list all the channels that are part of the gray collection # and alpha experiment: chan_list = rmt.list_channels('gray', 'alpha') print("Channels in Collection - gray and Experiment - alpha: {}".format( chan_list)) # When creating a new resource, the corresponding resource object will need # the required attributes populated. # For example, to add an "image" style channel named beta to the alpha experiment referenced # in the previous example: betaChan = ChannelResource('beta', 'gray', 'alpha', 'image') newBetaChan = rmt.create_project(betaChan) # Note that the create method returns a new instance of the ChannelResource. # The new instance has its `raw` attribute populated with all the JSON data # returned by the Boss API. # Display raw channel data from the Boss API: print(newBetaChan.raw) # We forgot, to indicate that this channel is related to the omega channel. # Let's fix that by updating the channel. newBetaChan.related = ['omega'] betaChanUpdated = rmt.update_project(newBetaChan.name, newBetaChan) # Let's verify the data type was updated. print(betaChanUpdated.related)
class BossResParams: def __init__(self, ingest_job, get_only=True): self.ingest_job = ingest_job self.coord_frame_name = '_'.join( (ingest_job.coll_name, ingest_job.exp_name)) self.rmt = BossRemote(self.ingest_job.boss_config_file) self.coll_resource = self.setup_boss_collection(get_only=get_only) self.coord_frame_resource = self.setup_boss_coord_frame( get_only=get_only) self.exp_resource = self.setup_boss_experiment(get_only=get_only) self.ch_resource = self.setup_boss_channel(get_only=get_only) def get_boss_project(self, proj_setup, get_only): try: proj_actual = self.rmt.get_project(proj_setup) except HTTPError as e: if get_only is False: try: proj_actual = self.rmt.create_project(proj_setup) except Exception as e: print(type(e), e) raise (e) else: print(type(e), e) raise e except Exception as e: print(type(e), e) raise (e) return proj_actual def setup_boss_collection(self, get_only=True): coll_setup = CollectionResource(self.ingest_job.coll_name) return self.get_boss_project(coll_setup, get_only) def setup_boss_coord_frame(self, get_only=True): # if we don't know the coordinate frame parameters, get the one with the same name if get_only: coord_setup = CoordinateFrameResource(self.coord_frame_name) else: coord_setup = CoordinateFrameResource( self.coord_frame_name, '', self.ingest_job.coord_frame_x_extent[0], self.ingest_job.coord_frame_x_extent[1], self.ingest_job.coord_frame_y_extent[0], self.ingest_job.coord_frame_y_extent[1], self.ingest_job.coord_frame_z_extent[0], self.ingest_job.coord_frame_z_extent[1], self.ingest_job.voxel_size[0], self.ingest_job.voxel_size[1], self.ingest_job.voxel_size[2], self.ingest_job.voxel_unit) coord_frame_resource = self.get_boss_project(coord_setup, get_only) if get_only: # matching ingest_job values to coordinate frame values (if they weren't specified, they are now populated) self.ingest_job.voxel_size = [ coord_frame_resource.x_voxel_size, coord_frame_resource.y_voxel_size, coord_frame_resource.z_voxel_size ] self.ingest_job.voxel_unit = coord_frame_resource.voxel_unit return coord_frame_resource def setup_boss_experiment(self, get_only=True): # if we don't know the coordinate frame parameters, get the one with the same name if get_only: exp_setup = ExperimentResource(self.ingest_job.exp_name, self.ingest_job.coll_name, self.coord_frame_name) else: # if all elements are the same, isotropic, otherwise anisotropic if len(set(self.ingest_job.voxel_size)) <= 1: hierarchy_method = 'isotropic' else: hierarchy_method = 'anisotropic' num_hierarchy_levels = self.calc_hierarchy_levels() exp_setup = ExperimentResource(self.ingest_job.exp_name, self.ingest_job.coll_name, self.coord_frame_name, '', num_hierarchy_levels, hierarchy_method) exp_resource = self.get_boss_project(exp_setup, get_only) # record the offset (if there is any) into BOSS metadata field for experiment self.record_offsets(exp_resource) return exp_resource def record_offsets(self, exp_resource): if any([a > 0 for a in self.ingest_job.offsets]): offsets_dict = {'offsets': self.ingest_job.offsets} try: self.rmt.create_metadata(exp_resource, offsets_dict) except HTTPErrorList: # keys already exist self.rmt.update_metadata(exp_resource, offsets_dict) def setup_boss_channel(self, ch_description='', get_only=True): ch_args = [ self.ingest_job.ch_name, self.ingest_job.coll_name, self.ingest_job.exp_name ] if not get_only: ch_args.extend( (self.ingest_job.ch_type, ch_description, 0, self.ingest_job.boss_datatype, self.ingest_job.res)) # annotation data if self.ingest_job.source_channel is not None: ch_args.append([self.ingest_job.source_channel]) # checking to make sure source channel exists (creating if needed) try: source_args = [ self.ingest_job.source_channel, self.ingest_job.coll_name, self.ingest_job.exp_name ] source_setup = ChannelResource(*source_args) self.get_boss_project(source_setup, True) except: self.ingest_job.send_msg( 'Creating a dummy source channel for this annotation because none exists yet' ) source_args.extend(('image', '', 0, 'uint8', 0)) source_setup = ChannelResource(*source_args) self.get_boss_project(source_setup, False) ch_setup = ChannelResource(*ch_args) ch_resource = self.get_boss_project(ch_setup, get_only) if get_only: self.ingest_job.boss_datatype = ch_resource.datatype if self.ingest_job.datatype is None: self.ingest_job.datatype = ch_resource.datatype self.ingest_job.res = ch_resource.base_resolution return ch_resource def calc_hierarchy_levels(self, lowest_res=512): img_size = [ self.coord_frame_resource.x_stop - self.coord_frame_resource.x_start, self.coord_frame_resource.y_stop - self.coord_frame_resource.y_start, self.coord_frame_resource.z_stop - self.coord_frame_resource.z_start ] max_xy = max(img_size[0:1]) # we add one because 0 is included in the number of downsampling levels num_levels = max(1, math.ceil(math.log(max_xy / lowest_res, 2)) + 1) return num_levels