def set_channel_resource(remote: BossRemote, params: dict) -> ChannelResource: """Use the arguments in the class config to create a channel resource object""" if 'name' not in params: params['name'] = 'channel{0}'.format(hex(round(time.time()))[2:]) param_names = [str(p.name) for p in inspect.signature(ChannelResource).parameters.values()] filtered_params = {k: v for k, v in list(params.items()) if k in param_names} # Filter unexpected arguments chan_resource = ChannelResource(**filtered_params) if chan_resource.name in remote.list_channels(chan_resource.coll_name, chan_resource.exp_name): channel = remote.update_project(chan_resource.name, chan_resource) print('Updated channel {0}'.format(chan_resource.name)) else: channel = remote.create_project(chan_resource) print('Created channel {0}'.format(chan_resource.name)) return channel
def getChan(CONFIG_FILE, coll, exp): config = configparser.ConfigParser() config.read(CONFIG_FILE) TOKEN = config['Default']['token'] boss_url = ''.join((config['Default']['protocol'], '://', config['Default']['host'], '/v1/')) #print(boss_url) #'https://api.boss.neurodata.io/v1/' #intern rem = BossRemote(CONFIG_FILE) chan = rem.list_channels(coll, exp) out = list( filter(lambda x: re.search(r'^(?!annotation|empty|EM)', x), chan)) em = list(filter(lambda x: re.search(r'(EM*)', x), chan)) return (out + em)
def delete_experiment(remote: BossRemote, experiment, coll_name=None): exp_obj = None exp_name = None if isinstance(experiment, ExperimentResource): exp_obj = experiment exp_name = experiment.name coll_name = experiment.coll_name elif isinstance(experiment, str) and coll_name is not None: exp_name = experiment exp_obj = remote.get_project(ExperimentResource(name=exp_name, collection_name=coll_name)) if exp_name is not None: print('Deleting channels of experiment "{0}"...'.format(exp_name)) chan_names = remote.list_channels(coll_name, exp_name) for n in chan_names: delete_channel(remote, n, exp_name, coll_name) if exp_obj is not None: print('Deleting experiment "{0}"...'.format(exp_obj.name)) remote.delete_project(exp_obj)
class TestRemoteListMethods(unittest.TestCase): def setUp(self): config = { "protocol": "https", "host": "test.theboss.io", "token": "my_secret" } self.remote = BossRemote(config) def test_list_collections(self): with patch.object(Remote, 'list_project') as list_prj_fake: list_prj_fake.return_value = [] actual = self.remote.list_collections() self.assertEqual([], actual) def test_list_experiments(self): with patch.object(Remote, 'list_project') as list_prj_fake: list_prj_fake.return_value = [] actual = self.remote.list_experiments('collection_alpha') self.assertEqual([], actual) def test_list_channels(self): with patch.object(Remote, 'list_project') as list_prj_fake: list_prj_fake.return_value = [] actual = self.remote.list_channels('collection_alpha', 'experiment_beta') self.assertEqual([], actual) def test_list_coordinate_frames(self): with patch.object(Remote, 'list_project') as list_prj_fake: list_prj_fake.return_value = [] actual = self.remote.list_coordinate_frames() self.assertEqual([], actual)
class NeuroDataResource: def __init__(self, host, token, collection, experiment): self._bossRemote = BossRemote({ 'protocol': 'https', 'host': host, 'token': token }) self.collection = collection self.experiment = experiment self.channels = self._bossRemote.list_channels(collection, experiment) self.channels.remove('empty') #Delete "empty" channel self._get_coord_frame_details() def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print( 'Error: You must supply zRange, yRange, xRange kwargs in list format' ) return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange, zRange) #Datatype check. Recast to original datatype if data is float64 if data.dtype == datatype: return data else: return data.astype(datatype)
class bossHandler: """ This is a class for interacting with BOSS. It basically is a wrapper for Intern. """ def __init__(self, collection_name): """ Constructor: Takes the following argument: :param collection_name: name of the collection in BOSS :type collection_name: string """ self.collection_name = collection_name try: self.rmt = BossRemote() except: print('Unexpected Error:', sys.exc_info()[0]) # def get_collection_list(self): # return self.rmt.list_collections() def list_experiments(self): """ :return: all the experiments available in current collection """ exp_list = self.rmt.list_experiments(self.collection_name) return exp_list def select_experiment(self, experiment_name): """ Select an experiment to be added to this handler """ tmp = ExperimentResource(collection_name=self.collection_name, name=experiment_name) exp = self.rmt.get_project(tmp) self.experiment = exp # return exp def get_experiment(self): """ :return: the currently selected experiment for this handler """ if hasattr(self, 'experiment'): return self.experiment else: raise AttributeError( 'No experiment exists. First, select an experiment using select_experiment' ) def list_channels(self): """ :return: all channel in currently selected experiment """ return self.rmt.list_channels(self.collection_name, self.experiment.name) def select_channel(self, channel_name): """ Select a channel to be added to this handler """ self.channel = self.rmt.get_channel(chan_name=channel_name, coll_name=self.collection_name, exp_name=self.experiment.name) def get_coordinate_frame(self): """ Return: current experiment's coordinate frame """ tmp = CoordinateFrameResource(name=self.experiment.coord_frame) coor = self.rmt.get_project(tmp) self.coordinate_frame = coor return coor def get_all(self): """ :return: the entire channel image data at its native resolution """ x_rng = [self.coordinate_frame.x_start, self.coordinate_frame.x_stop] y_rng = [self.coordinate_frame.y_start, self.coordinate_frame.y_stop] z_rng = [self.coordinate_frame.z_start, self.coordinate_frame.z_stop] return self.rmt.get_cutout(self.channel, 0, x_rng, y_rng, z_rng) def get_cutout(self, x_range, y_range, z_range, resolution=0): """ :return: a cutout of the image data """ return self.rmt.get_cutout(self.channel, resolution, x_range, y_range, z_range)
class NeuroDataResource: def __init__(self, host, token, collection, experiment, requested_channels, x_range=None, y_range=None, z_range=None, resolution=0): self._bossRemote = BossRemote({ 'protocol': 'https', 'host': host, 'token': token }) self.collection = collection self.experiment = experiment self.resolution = resolution self.channels = self._bossRemote.list_channels(collection, experiment) if len(requested_channels) == 0: self.requested_channels = self.channels else: self.requested_channels = requested_channels self._get_coord_frame_details() # validate range #if not self.correct_range(z_range, y_range, x_range): # raise Exception("Error: Inccorect dimension range") self.x_range = x_range or [0, self.max_dimensions[2]] self.y_range = y_range or [0, self.max_dimensions[1]] self.z_range = z_range or [0, self.max_dimensions[0]] def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print( 'Error: You must supply zRange, yRange, xRange kwargs in list format' ) return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, self.resolution, xRange, yRange, zRange) #Datatype check. Recast to original datatype if data is float64 if data.dtype == datatype: return data else: return data.astype(datatype) def correct_range(self, z_range, y_range, x_range): x_start, x_end = x_range if x_start < 0 or x_end > self.max_dimensions[2]: return False y_start, y_end = y_range if y_start < 0 or y_end > self.max_dimensions[1]: return False z_start, z_end = z_range if z_start < 0 or z_end > self.max_dimensions[0]: return False return True
boss_url = 'https://api.boss.neurodata.io/v1' url_constr = boss_url + '/collection/{}/experiment/{}/channel/{}' config = configparser.ConfigParser() config.read(boss_config_file) APITOKEN = config['Default']['token'] headers = {'Authorization': 'Token {}'.format(APITOKEN)} s = requests.Session() #%% data = [] for coll in coll_list: exp_list = rmt.list_experiments(coll) for exp in exp_list: ch_list = rmt.list_channels(coll, exp) for ch in ch_list: url = '{}/downsample/{}/{}/{}'.format(boss_url, coll, exp, ch) r = s.get(url, headers=headers) try: downsample_resp = r.json() downsample_status = downsample_resp['status'] extent = downsample_resp['extent'] last_extent = extent[sorted(extent.keys())[-1]] first_extent = extent[sorted(extent.keys())[0]] data.append([ coll, exp, ch, downsample_status, ', '.join([str(x) for x in first_extent]),
from intern.remote.boss import BossRemote from intern.resource.boss.resource import * # admin token needed to list all projects rmt = BossRemote("/home/ben/Documents/travis_user_neurodata.cfg") with open("public_datasets.csv", "w") as f: f.write( "coll,exp,ch,exp_description,num_hierarchy_levels,dtype,x_start,x_stop,y_start,y_stop,z_start,z_stop\n" ) colls = rmt.list_collections() colls.remove("ben_dev") colls.remove("ZBrain") for coll in colls: exps = rmt.list_experiments(coll) for exp in exps: exp_res = rmt.get_experiment(coll, exp) coord_frame_res = rmt.get_coordinate_frame(exp_res.coord_frame) chs = rmt.list_channels(coll, exp) for ch in chs: ch_res = rmt.get_channel(ch, coll, exp) with open("public_datasets.csv", "a") as f: f.write( f"{coll},{exp},{ch},{exp_res.description},{exp_res.num_hierarchy_levels},{ch_res.datatype},{coord_frame_res.x_start},{coord_frame_res.x_stop},{coord_frame_res.y_start},{coord_frame_res.y_stop},{coord_frame_res.z_start},{coord_frame_res.z_stop}\n" )
def gen_urls(args): config = {'protocol': 'https', 'host': args.hostname, 'token': args.token} boss = BossRemote(config) results = [] try: if args.collection is not None: collections = [args.collection] else: collections = boss.list_collections() for collection in collections: if args.experiment is not None: experiments = [args.experiment] else: experiments = boss.list_experiments(collection) for experiment in experiments: if args.channel is not None: channels = [args.channel] else: channels = boss.list_channels(collection, experiment) exp = ExperimentResource(name = experiment, collection_name = collection) exp = boss.get_project(exp) coord = CoordinateFrameResource(name = exp.coord_frame) coord = boss.get_project(coord) for channel in channels: ch = ChannelResource(name = channel, experiment_name = experiment, collection_name = collection) ch = boss.get_project(ch) def check_range(name, var, start, stop): start_, stop_ = map(int, var.split(':')) if start_ < start: fmt = "{} range start for {}/{}/{} is less than the coordinate frame, setting to minimum" print(fmt.format(name, collection, experiment, channel)) start_ = start if stop_ > stop: fmt = "{} range stop for {}/{}/{} is greater than the coordinate frame, setting to maximum" print(fmt.format(name, collection, experiment, channel)) stop_ = stop return '{}:{}'.format(start_, stop_) if args.x_range: x = check_range('X', args.x_range, coord.x_start, coord.x_stop) else: x = (coord.x_start, coord.x_stop, args.min, args.max) if args.y_range: y = check_range('Y', args.y_range, coord.y_start, coord.y_stop) else: y = (coord.y_start, coord.y_stop, args.min, args.max) if args.z_range: z = check_range('Z', args.z_range, coord.z_start, coord.z_stop) else: z = (coord.z_start, coord.z_stop, args.min, args.max) # Arguments to gen_url results.append((args.hostname, collection, experiment, channel, 0, x, y, z, None)) except Exception as e: print("Error generating URLs: {}".format(e)) return results
class NeuroDataResource: def __init__(self, host, token, collection, experiment): self._bossRemote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) self.collection = collection self.experiment = experiment self.channels = self._bossRemote.list_channels(collection, experiment) self.channels.remove('empty') #Delete "empty" channel self._get_coord_frame_details() def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) def _get_channel(self, chan_name): """ Helper that gets a fully initialized ChannelResource for an *existing* channel. Args: chan_name (str): Name of channel. coll_name (str): Name of channel's collection. exp_name (str): Name of channel's experiment. Returns: (intern.resource.boss.ChannelResource) """ chan = ChannelResource(chan_name, self.collection, self.experiment) return self._bossRemote.get_project(chan) def assert_channel_exists(self, channel): return channel in self.channels def get_cutout(self, chan, zRange=None, yRange=None, xRange=None): if chan not in self.channels: print('Error: Channel Not Found in this Resource') return if zRange is None or yRange is None or xRange is None: print('Error: You must supply zRange, yRange, xRange kwargs in list format') return channel_resource = self._get_channel(chan) datatype = channel_resource.datatype data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange, zRange) return data @staticmethod def ingest_volume(host, token, channel_name, collection, experiment, volume): """ Assumes the collection and experiment exists in BOSS. """ remote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) if volume.dtype == 'uint64': dtype = 'uint64' img_type = 'annotation' sources = ['empty'] else: dtype = volume.dtype.name img_type = 'image' sources = [] try: channel_resource = ChannelResource(channel_name, collection, experiment) channel = remote.get_project(channel_resource) except: channel_resource = ChannelResource(channel_name, collection, experiment, type=img_type, sources=sources, datatype=dtype) channel = remote.create_project(channel_resource) #Get max size of experiment exp_resource = ExperimentResource(experiment, collection) coord_frame = remote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = remote.get_project(coord_frame_resource) y_stop, x_stop = data.y_stop, data.x_stop for z in range(volume.shape[0]): print('Uploading {} slice'.format(z)) remote.create_cutout(channel, 0, (0, x_stop), (0, y_stop), (z, z + 1), volume[z, :, :].reshape((-1, y_stop, x_stop)))
boss_config_file = 'neurodata.cfg' rmt = BossRemote(boss_config_file) #for making manual web requests to the boss: config = configparser.ConfigParser() config.read(boss_config_file) APITOKEN = config['Default']['token'] coll = 'weiler14' experiments_process = np.loadtxt("experiments.txt", comments="#", dtype='str',ndmin=1) s = requests.Session() for exp in experiments_process: ch_names = rmt.list_channels(coll, exp) for ch in ch_names: url = 'https://api.boss.neurodata.io/latest/downsample/{}/{}/{}/'.format( coll, exp, ch) headers = { 'Authorization': 'Token {}'.format(APITOKEN) } r = s.post(url, data = {}, headers=headers) if r.status_code != 201: print("Error: Request failed!") print("Received {} from server.".format(r.status_code)) f = tempfile.NamedTemporaryFile() f.write(r.content) import pdb; pdb.set_trace() sys.exit(1) else:
resource object also contains the parameters to place in the database. """ from intern.remote.boss import BossRemote from intern.resource.boss.resource import * rmt = BossRemote('example.cfg') # List current collections. coll_list = rmt.list_collections() print("Collections: {}".format(coll_list)) # For resources below the collection level, the parents must be specified. # For example to list all the channels that are part of the gray collection # and alpha experiment: chan_list = rmt.list_channels('gray', 'alpha') print("Channels in Collection - gray and Experiment - alpha: {}".format( chan_list)) # When creating a new resource, the corresponding resource object will need # the required attributes populated. # For example, to add an "image" style channel named beta to the alpha experiment referenced # in the previous example: betaChan = ChannelResource('beta', 'gray', 'alpha', 'image') newBetaChan = rmt.create_project(betaChan) # Note that the create method returns a new instance of the ChannelResource. # The new instance has its `raw` attribute populated with all the JSON data # returned by the Boss API. # Display raw channel data from the Boss API: