def set_channel_resource(remote: BossRemote, params: dict) -> ChannelResource:
"""Use the arguments in the class config to create a channel resource object"""
if 'name' not in params:
params['name'] = 'channel{0}'.format(hex(round(time.time()))[2:])
param_names = [str(p.name) for p in inspect.signature(ChannelResource).parameters.values()]
filtered_params = {k: v for k, v in list(params.items()) if k in param_names} # Filter unexpected arguments
chan_resource = ChannelResource(**filtered_params)
if chan_resource.name in remote.list_channels(chan_resource.coll_name, chan_resource.exp_name):
channel = remote.update_project(chan_resource.name, chan_resource)
print('Updated channel {0}'.format(chan_resource.name))
else:
channel = remote.create_project(chan_resource)
print('Created channel {0}'.format(chan_resource.name))
return channel
def getChan(CONFIG_FILE, coll, exp):
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
TOKEN = config['Default']['token']
boss_url = ''.join((config['Default']['protocol'], '://',
config['Default']['host'], '/v1/'))
#print(boss_url)
#'https://api.boss.neurodata.io/v1/'
#intern
rem = BossRemote(CONFIG_FILE)
chan = rem.list_channels(coll, exp)
out = list(
filter(lambda x: re.search(r'^(?!annotation|empty|EM)', x), chan))
em = list(filter(lambda x: re.search(r'(EM*)', x), chan))
return (out + em)
def delete_experiment(remote: BossRemote, experiment, coll_name=None):
exp_obj = None
exp_name = None
if isinstance(experiment, ExperimentResource):
exp_obj = experiment
exp_name = experiment.name
coll_name = experiment.coll_name
elif isinstance(experiment, str) and coll_name is not None:
exp_name = experiment
exp_obj = remote.get_project(ExperimentResource(name=exp_name, collection_name=coll_name))
if exp_name is not None:
print('Deleting channels of experiment "{0}"...'.format(exp_name))
chan_names = remote.list_channels(coll_name, exp_name)
for n in chan_names:
delete_channel(remote, n, exp_name, coll_name)
if exp_obj is not None:
print('Deleting experiment "{0}"...'.format(exp_obj.name))
remote.delete_project(exp_obj)
class TestRemoteListMethods(unittest.TestCase):
def setUp(self):
config = {
"protocol": "https",
"host": "test.theboss.io",
"token": "my_secret"
}
self.remote = BossRemote(config)
def test_list_collections(self):
with patch.object(Remote, 'list_project') as list_prj_fake:
list_prj_fake.return_value = []
actual = self.remote.list_collections()
self.assertEqual([], actual)
def test_list_experiments(self):
with patch.object(Remote, 'list_project') as list_prj_fake:
list_prj_fake.return_value = []
actual = self.remote.list_experiments('collection_alpha')
self.assertEqual([], actual)
def test_list_channels(self):
with patch.object(Remote, 'list_project') as list_prj_fake:
list_prj_fake.return_value = []
actual = self.remote.list_channels('collection_alpha',
'experiment_beta')
self.assertEqual([], actual)
def test_list_coordinate_frames(self):
with patch.object(Remote, 'list_project') as list_prj_fake:
list_prj_fake.return_value = []
actual = self.remote.list_coordinate_frames()
self.assertEqual([], actual)
class NeuroDataResource:
def __init__(self, host, token, collection, experiment):
self._bossRemote = BossRemote({
'protocol': 'https',
'host': host,
'token': token
})
self.collection = collection
self.experiment = experiment
self.channels = self._bossRemote.list_channels(collection, experiment)
self.channels.remove('empty') #Delete "empty" channel
self._get_coord_frame_details()
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size,
data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print(
'Error: You must supply zRange, yRange, xRange kwargs in list format'
)
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource, 0, xRange, yRange,
zRange)
#Datatype check. Recast to original datatype if data is float64
if data.dtype == datatype:
return data
else:
return data.astype(datatype)
class bossHandler:
"""
This is a class for interacting with BOSS. It basically is a wrapper for Intern.
"""
def __init__(self, collection_name):
"""
Constructor:
Takes the following argument:
:param collection_name: name of the collection in BOSS
:type collection_name: string
"""
self.collection_name = collection_name
try:
self.rmt = BossRemote()
except:
print('Unexpected Error:', sys.exc_info()[0])
# def get_collection_list(self):
# return self.rmt.list_collections()
def list_experiments(self):
"""
:return: all the experiments available in current collection
"""
exp_list = self.rmt.list_experiments(self.collection_name)
return exp_list
def select_experiment(self, experiment_name):
"""
Select an experiment to be added to this handler
"""
tmp = ExperimentResource(collection_name=self.collection_name,
name=experiment_name)
exp = self.rmt.get_project(tmp)
self.experiment = exp
# return exp
def get_experiment(self):
"""
:return: the currently selected experiment for this handler
"""
if hasattr(self, 'experiment'):
return self.experiment
else:
raise AttributeError(
'No experiment exists. First, select an experiment using select_experiment'
)
def list_channels(self):
"""
:return: all channel in currently selected experiment
"""
return self.rmt.list_channels(self.collection_name,
self.experiment.name)
def select_channel(self, channel_name):
"""
Select a channel to be added to this handler
"""
self.channel = self.rmt.get_channel(chan_name=channel_name,
coll_name=self.collection_name,
exp_name=self.experiment.name)
def get_coordinate_frame(self):
"""
Return: current experiment's coordinate frame
"""
tmp = CoordinateFrameResource(name=self.experiment.coord_frame)
coor = self.rmt.get_project(tmp)
self.coordinate_frame = coor
return coor
def get_all(self):
"""
:return: the entire channel image data at its native resolution
"""
x_rng = [self.coordinate_frame.x_start, self.coordinate_frame.x_stop]
y_rng = [self.coordinate_frame.y_start, self.coordinate_frame.y_stop]
z_rng = [self.coordinate_frame.z_start, self.coordinate_frame.z_stop]
return self.rmt.get_cutout(self.channel, 0, x_rng, y_rng, z_rng)
def get_cutout(self, x_range, y_range, z_range, resolution=0):
"""
:return: a cutout of the image data
"""
return self.rmt.get_cutout(self.channel, resolution, x_range, y_range,
z_range)
class NeuroDataResource:
def __init__(self,
host,
token,
collection,
experiment,
requested_channels,
x_range=None,
y_range=None,
z_range=None,
resolution=0):
self._bossRemote = BossRemote({
'protocol': 'https',
'host': host,
'token': token
})
self.collection = collection
self.experiment = experiment
self.resolution = resolution
self.channels = self._bossRemote.list_channels(collection, experiment)
if len(requested_channels) == 0:
self.requested_channels = self.channels
else:
self.requested_channels = requested_channels
self._get_coord_frame_details()
# validate range
#if not self.correct_range(z_range, y_range, x_range):
# raise Exception("Error: Inccorect dimension range")
self.x_range = x_range or [0, self.max_dimensions[2]]
self.y_range = y_range or [0, self.max_dimensions[1]]
self.z_range = z_range or [0, self.max_dimensions[0]]
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size,
data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print(
'Error: You must supply zRange, yRange, xRange kwargs in list format'
)
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource, self.resolution,
xRange, yRange, zRange)
#Datatype check. Recast to original datatype if data is float64
if data.dtype == datatype:
return data
else:
return data.astype(datatype)
def correct_range(self, z_range, y_range, x_range):
x_start, x_end = x_range
if x_start < 0 or x_end > self.max_dimensions[2]:
return False
y_start, y_end = y_range
if y_start < 0 or y_end > self.max_dimensions[1]:
return False
z_start, z_end = z_range
if z_start < 0 or z_end > self.max_dimensions[0]:
return False
return True
boss_url = 'https://api.boss.neurodata.io/v1'
url_constr = boss_url + '/collection/{}/experiment/{}/channel/{}'
config = configparser.ConfigParser()
config.read(boss_config_file)
APITOKEN = config['Default']['token']
headers = {'Authorization': 'Token {}'.format(APITOKEN)}
s = requests.Session()
#%%
data = []
for coll in coll_list:
exp_list = rmt.list_experiments(coll)
for exp in exp_list:
ch_list = rmt.list_channels(coll, exp)
for ch in ch_list:
url = '{}/downsample/{}/{}/{}'.format(boss_url, coll, exp, ch)
r = s.get(url, headers=headers)
try:
downsample_resp = r.json()
downsample_status = downsample_resp['status']
extent = downsample_resp['extent']
last_extent = extent[sorted(extent.keys())[-1]]
first_extent = extent[sorted(extent.keys())[0]]
data.append([
coll, exp, ch, downsample_status,
', '.join([str(x) for x in first_extent]),
from intern.remote.boss import BossRemote
from intern.resource.boss.resource import *
# admin token needed to list all projects
rmt = BossRemote("/home/ben/Documents/travis_user_neurodata.cfg")
with open("public_datasets.csv", "w") as f:
f.write(
"coll,exp,ch,exp_description,num_hierarchy_levels,dtype,x_start,x_stop,y_start,y_stop,z_start,z_stop\n"
)
colls = rmt.list_collections()
colls.remove("ben_dev")
colls.remove("ZBrain")
for coll in colls:
exps = rmt.list_experiments(coll)
for exp in exps:
exp_res = rmt.get_experiment(coll, exp)
coord_frame_res = rmt.get_coordinate_frame(exp_res.coord_frame)
chs = rmt.list_channels(coll, exp)
for ch in chs:
ch_res = rmt.get_channel(ch, coll, exp)
with open("public_datasets.csv", "a") as f:
f.write(
f"{coll},{exp},{ch},{exp_res.description},{exp_res.num_hierarchy_levels},{ch_res.datatype},{coord_frame_res.x_start},{coord_frame_res.x_stop},{coord_frame_res.y_start},{coord_frame_res.y_stop},{coord_frame_res.z_start},{coord_frame_res.z_stop}\n"
)
def gen_urls(args):
config = {'protocol': 'https',
'host': args.hostname,
'token': args.token}
boss = BossRemote(config)
results = []
try:
if args.collection is not None:
collections = [args.collection]
else:
collections = boss.list_collections()
for collection in collections:
if args.experiment is not None:
experiments = [args.experiment]
else:
experiments = boss.list_experiments(collection)
for experiment in experiments:
if args.channel is not None:
channels = [args.channel]
else:
channels = boss.list_channels(collection, experiment)
exp = ExperimentResource(name = experiment,
collection_name = collection)
exp = boss.get_project(exp)
coord = CoordinateFrameResource(name = exp.coord_frame)
coord = boss.get_project(coord)
for channel in channels:
ch = ChannelResource(name = channel,
experiment_name = experiment,
collection_name = collection)
ch = boss.get_project(ch)
def check_range(name, var, start, stop):
start_, stop_ = map(int, var.split(':'))
if start_ < start:
fmt = "{} range start for {}/{}/{} is less than the coordinate frame, setting to minimum"
print(fmt.format(name, collection, experiment, channel))
start_ = start
if stop_ > stop:
fmt = "{} range stop for {}/{}/{} is greater than the coordinate frame, setting to maximum"
print(fmt.format(name, collection, experiment, channel))
stop_ = stop
return '{}:{}'.format(start_, stop_)
if args.x_range:
x = check_range('X', args.x_range, coord.x_start, coord.x_stop)
else:
x = (coord.x_start, coord.x_stop, args.min, args.max)
if args.y_range:
y = check_range('Y', args.y_range, coord.y_start, coord.y_stop)
else:
y = (coord.y_start, coord.y_stop, args.min, args.max)
if args.z_range:
z = check_range('Z', args.z_range, coord.z_start, coord.z_stop)
else:
z = (coord.z_start, coord.z_stop, args.min, args.max)
# Arguments to gen_url
results.append((args.hostname,
collection,
experiment,
channel,
0, x, y, z, None))
except Exception as e:
print("Error generating URLs: {}".format(e))
return results
class NeuroDataResource:
def __init__(self, host, token, collection, experiment):
self._bossRemote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
self.collection = collection
self.experiment = experiment
self.channels = self._bossRemote.list_channels(collection, experiment)
self.channels.remove('empty') #Delete "empty" channel
self._get_coord_frame_details()
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size)
def _get_channel(self, chan_name):
"""
Helper that gets a fully initialized ChannelResource for an *existing* channel.
Args:
chan_name (str): Name of channel.
coll_name (str): Name of channel's collection.
exp_name (str): Name of channel's experiment.
Returns:
(intern.resource.boss.ChannelResource)
"""
chan = ChannelResource(chan_name, self.collection, self.experiment)
return self._bossRemote.get_project(chan)
def assert_channel_exists(self, channel):
return channel in self.channels
def get_cutout(self, chan, zRange=None, yRange=None, xRange=None):
if chan not in self.channels:
print('Error: Channel Not Found in this Resource')
return
if zRange is None or yRange is None or xRange is None:
print('Error: You must supply zRange, yRange, xRange kwargs in list format')
return
channel_resource = self._get_channel(chan)
datatype = channel_resource.datatype
data = self._bossRemote.get_cutout(channel_resource,
0,
xRange,
yRange,
zRange)
return data
@staticmethod
def ingest_volume(host, token, channel_name, collection, experiment, volume):
"""
Assumes the collection and experiment exists in BOSS.
"""
remote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
if volume.dtype == 'uint64':
dtype = 'uint64'
img_type = 'annotation'
sources = ['empty']
else:
dtype = volume.dtype.name
img_type = 'image'
sources = []
try:
channel_resource = ChannelResource(channel_name, collection, experiment)
channel = remote.get_project(channel_resource)
except:
channel_resource = ChannelResource(channel_name, collection, experiment,
type=img_type,
sources=sources,
datatype=dtype)
channel = remote.create_project(channel_resource)
#Get max size of experiment
exp_resource = ExperimentResource(experiment, collection)
coord_frame = remote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = remote.get_project(coord_frame_resource)
y_stop, x_stop = data.y_stop, data.x_stop
for z in range(volume.shape[0]):
print('Uploading {} slice'.format(z))
remote.create_cutout(channel,
0,
(0, x_stop),
(0, y_stop),
(z, z + 1),
volume[z, :, :].reshape((-1, y_stop, x_stop)))
boss_config_file = 'neurodata.cfg'
rmt = BossRemote(boss_config_file)
#for making manual web requests to the boss:
config = configparser.ConfigParser()
config.read(boss_config_file)
APITOKEN = config['Default']['token']
coll = 'weiler14'
experiments_process = np.loadtxt("experiments.txt", comments="#", dtype='str',ndmin=1)
s = requests.Session()
for exp in experiments_process:
ch_names = rmt.list_channels(coll, exp)
for ch in ch_names:
url = 'https://api.boss.neurodata.io/latest/downsample/{}/{}/{}/'.format(
coll, exp, ch)
headers = {
'Authorization': 'Token {}'.format(APITOKEN)
}
r = s.post(url, data = {}, headers=headers)
if r.status_code != 201:
print("Error: Request failed!")
print("Received {} from server.".format(r.status_code))
f = tempfile.NamedTemporaryFile()
f.write(r.content)
import pdb; pdb.set_trace()
sys.exit(1)
else:
resource object also contains the parameters to place in the database.
"""
from intern.remote.boss import BossRemote
from intern.resource.boss.resource import *
rmt = BossRemote('example.cfg')
# List current collections.
coll_list = rmt.list_collections()
print("Collections: {}".format(coll_list))
# For resources below the collection level, the parents must be specified.
# For example to list all the channels that are part of the gray collection
# and alpha experiment:
chan_list = rmt.list_channels('gray', 'alpha')
print("Channels in Collection - gray and Experiment - alpha: {}".format(
chan_list))
# When creating a new resource, the corresponding resource object will need
# the required attributes populated.
# For example, to add an "image" style channel named beta to the alpha experiment referenced
# in the previous example:
betaChan = ChannelResource('beta', 'gray', 'alpha', 'image')
newBetaChan = rmt.create_project(betaChan)
# Note that the create method returns a new instance of the ChannelResource.
# The new instance has its `raw` attribute populated with all the JSON data
# returned by the Boss API.
# Display raw channel data from the Boss API:
