def get_voxel_unit(self,
channel: ChannelResource,
resolution: int = 0) -> str:
experiment = self.get_project(
ExperimentResource(channel.exp_name, channel.coll_name))
# Get the coordinate frame:
cf = self.get_project(CoordinateFrameResource(experiment.coord_frame))
return cf.voxel_unit
def get_voxel_size(self,
channel: ChannelResource,
resolution: int = 0) -> Tuple[float, float, float]:
experiment = self.get_project(
ExperimentResource(channel.exp_name, channel.coll_name))
# Get the coordinate frame:
cf = self.get_project(CoordinateFrameResource(experiment.coord_frame))
return (cf.x_voxel_size, cf.y_voxel_size, cf.z_voxel_size)
def _get_coord_frame_details(self):
exp_resource = ExperimentResource(self.experiment, self.collection)
coord_frame = self._bossRemote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = self._bossRemote.get_project(coord_frame_resource)
self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size)
def get_coord_frame_details(rmt, coll, exp):
exp_resource = ExperimentResource(exp, coll)
coord_frame = rmt.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = rmt.get_project(coord_frame_resource)
max_dimensions = (data.z_stop, data.y_stop, data.x_stop)
voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size)
return max_dimensions, voxel_size
def fetch_info(self):
experiment = ExperimentResource(name=self.path.dataset,
collection_name=self.path.bucket)
rmt = BossRemote(boss_credentials)
experiment = rmt.get_project(experiment)
coord_frame = CoordinateFrameResource(name=experiment.coord_frame)
coord_frame = rmt.get_project(coord_frame)
channel = ChannelResource(self.path.layer, self.path.bucket,
self.path.dataset)
channel = rmt.get_project(channel)
unit_factors = {
'nanometers': 1,
'micrometers': 1e3,
'millimeters': 1e6,
'centimeters': 1e7,
}
unit_factor = unit_factors[coord_frame.voxel_unit]
cf = coord_frame
resolution = [cf.x_voxel_size, cf.y_voxel_size, cf.z_voxel_size]
resolution = [int(round(_)) * unit_factor for _ in resolution]
bbox = Bbox((cf.x_start, cf.y_start, cf.z_start),
(cf.x_stop, cf.y_stop, cf.z_stop))
bbox.maxpt = bbox.maxpt
layer_type = 'unknown'
if 'type' in channel.raw:
layer_type = channel.raw['type']
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=layer_type,
data_type=channel.datatype,
encoding='raw',
resolution=resolution,
voxel_offset=bbox.minpt,
volume_size=bbox.size3(),
chunk_size=(512, 512, 16), # fixed in s3 implementation
)
each_factor = Vec(2, 2, 1)
if experiment.hierarchy_method == 'isotropic':
each_factor = Vec(2, 2, 2)
factor = each_factor.clone()
for _ in range(1, experiment.num_hierarchy_levels):
self.add_scale(factor, info=info)
factor *= each_factor
return info
def setup_experiment_boss(remote, collection, experiment):
"""
Get experiment and coordinate frame information from the boss.
"""
exp_setup = ExperimentResource(experiment, collection)
try:
exp_actual = remote.get_project(exp_setup)
coord_setup = CoordinateFrameResource(exp_actual.coord_frame)
coord_actual = remote.get_project(coord_setup)
return (exp_setup, coord_actual)
except Exception as e:
print(e.message)
def get_shape(self,
channel: ChannelResource,
resolution: int = 0) -> Tuple[int, int, int]:
# Get the experiment:
experiment = self.get_project(
ExperimentResource(channel.exp_name, channel.coll_name))
# Get the coordinate frame:
cf = self.get_project(CoordinateFrameResource(experiment.coord_frame))
# Return the bounds of the coordinate frame:
return (
(cf.z_stop - cf.z_start),
int((cf.y_stop - cf.y_start) / (2**resolution)),
int((cf.x_stop - cf.x_start) / (2**resolution)),
)
def ingest_volume(host, token, channel_name, collection, experiment, volume):
"""
Assumes the collection and experiment exists in BOSS.
"""
remote = BossRemote({'protocol': 'https',
'host': host,
'token': token})
if volume.dtype == 'uint64':
dtype = 'uint64'
img_type = 'annotation'
sources = ['empty']
else:
dtype = volume.dtype.name
img_type = 'image'
sources = []
try:
channel_resource = ChannelResource(channel_name, collection, experiment)
channel = remote.get_project(channel_resource)
except:
channel_resource = ChannelResource(channel_name, collection, experiment,
type=img_type,
sources=sources,
datatype=dtype)
channel = remote.create_project(channel_resource)
#Get max size of experiment
exp_resource = ExperimentResource(experiment, collection)
coord_frame = remote.get_project(exp_resource).coord_frame
coord_frame_resource = CoordinateFrameResource(coord_frame)
data = remote.get_project(coord_frame_resource)
y_stop, x_stop = data.y_stop, data.x_stop
for z in range(volume.shape[0]):
print('Uploading {} slice'.format(z))
remote.create_cutout(channel,
0,
(0, x_stop),
(0, y_stop),
(z, z + 1),
volume[z, :, :].reshape((-1, y_stop, x_stop)))
def __init__(self,
resource: Union[BossResource, str],
remote: BossRemote = None):
self._remote = remote or BossRemote(_DEFAULT_BOSS_OPTIONS)
if isinstance(resource, str):
resource = resource.split("://")[-1]
path_items = resource.split("/")
if len(path_items) == 1:
self._resource = self._remote.get_project(
CollectionResource(path_items[0]))
elif len(path_items) == 2:
self._resource = self._remote.get_project(
ExperimentResource(path_items[1], path_items[0]))
elif len(path_items) == 3:
self._resource = self._remote.get_project(
ChannelResource(path_items[2], path_items[0],
path_items[1]))
else:
raise ValueError(f"Invalid resource path: {resource}")
else:
self._resource = resource
def setUpClass(cls):
'''Initialized once, before any class tests are run'''
boss = cls.boss = Remote()
fixtures = {}
fixtures['coordframe'] = CoordinateFrameResource(COORD_FRAME)
fixtures['collection'] = CollectionResource(COLLECTION)
fixtures['experiment'] = ExperimentResource(EXPERIMENT,
COLLECTION,
coord_frame=COORD_FRAME)
fixtures['channel'] = ChannelResource(CHANNEL, COLLECTION, EXPERIMENT,
'image')
fixtures['ann_channel'] = ChannelResource(ANN_CHANNEL,
COLLECTION,
EXPERIMENT,
'annotation',
sources=[CHANNEL])
cls.fixtures = fixtures
for resource_name in [
'coordframe', 'collection', 'experiment', 'channel',
'ann_channel'
]:
boss.create_project(fixtures[resource_name])
def setUp(self):
self.er = ExperimentResource('foo', 'bar', 'coords')
class TestExperimentResource(unittest.TestCase):
def setUp(self):
self.er = ExperimentResource('foo', 'bar', 'coords')
def test_not_valid_volume(self):
self.assertFalse(self.er.valid_volume())
def test_get_route(self):
self.assertEqual(
'{}/experiment/{}'.format(self.er.coll_name, self.er.name),
self.er.get_route())
def test_get_list_route(self):
self.assertEqual('{}/experiment/'.format(self.er.coll_name),
self.er.get_list_route())
def test_hierarchy_method_setter(self):
exp = 'anisotropic'
self.er.hierarchy_method = exp
self.assertEqual(exp, self.er.hierarchy_method)
def test_validate_hierarchy_method_aniso(self):
exp = 'anisotropic'
self.assertEqual(exp, self.er.validate_hierarchy_method(exp))
def test_validate_hierarchy_method_iso(self):
exp = 'isotropic'
self.assertEqual(exp, self.er.validate_hierarchy_method(exp))
def test_validate_hierarchy_method_bad(self):
with self.assertRaises(ValueError):
self.er.validate_hierarchy_method('slice')
def test_validate_time_units_blank(self):
exp = ''
self.assertEqual(exp, self.er.validate_time_units(''))
def test_validate_time_units_ns(self):
exp = 'nanoseconds'
self.assertEqual(exp, self.er.validate_time_units(exp))
def test_validate_time_units_micro(self):
exp = 'microseconds'
self.assertEqual(exp, self.er.validate_time_units(exp))
def test_validate_time_units_ms(self):
exp = 'milliseconds'
self.assertEqual(exp, self.er.validate_time_units(exp))
def test_validate_time_units_s(self):
exp = 'seconds'
self.assertEqual(exp, self.er.validate_time_units(exp))
def test_validate_time_units_bad(self):
with self.assertRaises(ValueError):
self.er.validate_time_units('secs')
def __init__(
self,
channel: Union[ChannelResource, Tuple, str],
resolution: int = 0,
volume_provider: VolumeProvider = None,
axis_order: str = AxisOrder.ZYX,
create_new: bool = False,
description: Optional[str] = None,
dtype: Optional[str] = None,
extents: Optional[Tuple[int, int, int]] = None,
voxel_size: Optional[Tuple[int, int, int]] = None,
voxel_unit: Optional[str] = None,
downsample_levels: int = 6,
downsample_method: Optional[str] = "anisotropic",
coordinate_frame_name: Optional[str] = None,
coordinate_frame_desc: Optional[str] = None,
collection_desc: Optional[str] = None,
experiment_desc: Optional[str] = None,
source_channel: Optional[str] = None,
boss_config: Optional[dict] = None,
) -> None:
"""
Construct a new intern-backed array.
Arguments:
channel (intern.resource.boss.ChannelResource): The channel from
which data will be downloaded.
resolution (int: 0): The native resolution or MIP to use
volume_provider (VolumeProvider): The remote-like to use
axis_order (str = AxisOrder.ZYX): The axis-ordering to use for data
cutouts. Defaults to ZYX. DOES NOT affect the `voxel_size` or
`extents` arguments to this constructor.
create_new (bool: False): Whether to create new Resources if they
do not exist. Does not work with public token.
dtype (str): Only required if `create_new = True`. Specifies the
numpy-style datatype for this new dataset (e.g. "uint8").
description (str): Only required if `create_new = True`. Sets the
description for the newly-created collection, experiment,
channel, and coordframe resources.
extents: Optional[Tuple[int, int, int]]: Only required if
`create_new = True`. Specifies the total dataset extents of
this new dataset, in ZYX order.
voxel_size: Optional[Tuple[int, int, int]]: Only required if
`create_new = True`. Specifies the voxel dimensions of this new
dataset, in ZYX order.
voxel_unit: Optional[str]: Only required if `create_new = True`.
Specifies the voxel-dimension unit. For example, "nanometers".
downsample_levels (int: 6): The number of downsample levels.
downsample_method (Optional[str]): The type of downsample to use.
If unset, defaults to 'anisotropic'.
coordinate_frame_name (Optional[str]): If set, the name to use for
the newly created coordinate frame. If not set, the name of the
coordinate frame will be chosen automatically.
coordinate_frame_desc (Optional[str]): If set, the description text
to use for the newly created coordinate frame. If not set, the
description will be chosen automatically.
collection_desc (Optional[str]): The description text to use for a
newly created collection. If not set, the description will be
chosen automatically.
experiment_desc (Optional[str]): The description text to use for a
newly created experiment. If not set, the description will be
chosen automatically.
source_channel (Optional[str]): The channel to use as the source
for this new channel, if `create_new` is True and this is
going to be an annotation channel (dtype!=uint8).
boss_config (Optional[dict]): The BossRemote configuration dict to
use in order to authenticate with a BossDB remote. This option
is mutually exclusive with the VolumeProvider configuration. If
the `volume_provider` arg is set, this will be ignored.
"""
self.axis_order = axis_order
# Handle inferring the remote from the channel:
volume_provider = volume_provider or _infer_volume_provider(channel)
if volume_provider is None:
if boss_config:
volume_provider = _BossDBVolumeProvider(
BossRemote(boss_config))
else:
volume_provider = _BossDBVolumeProvider()
# Handle custom Remote:
self.volume_provider = volume_provider
if create_new:
if self.volume_provider.get_vp_type() != "bossdb":
raise ValueError(
"Creating new resources with a non-bossdb volume provider is not currently supported."
)
# We'll need at least `extents` and `voxel_size`.
description = description or "Created with intern"
dtype = dtype or "uint8"
if extents is None:
raise ValueError(
"If `create_new` is True, you must specify the extents of the new coordinate frame as a [x, y, z]."
)
if voxel_size is None:
raise ValueError(
"If `create_new` is True, you must specify the voxel_size of the new coordinate frame as a [x, y, z]."
)
uri = _parse_bossdb_uri(channel)
# create collection if it doesn't exist:
try:
# Try to get an existing collection:
collection = self.volume_provider.get_project(
CollectionResource(uri.collection))
except:
# Create the collection:
collection = CollectionResource(uri.collection,
description=collection_desc
or description)
self.volume_provider.create_project(collection)
# create coordframe if it doesn't exist:
try:
# Try to get an existing coordframe:
coordframe = self.volume_provider.get_project(
CoordinateFrameResource(
coordinate_frame_name
or f"CF_{uri.collection}_{uri.experiment}"))
except:
# Default to nanometers if a voxel unit isn't provided
voxel_unit = voxel_unit or "nanometers"
# Create the coordframe:
coordframe = CoordinateFrameResource(
coordinate_frame_name
or f"CF_{uri.collection}_{uri.experiment}",
description=coordinate_frame_desc or description,
x_start=0,
y_start=0,
z_start=0,
x_stop=extents[2],
y_stop=extents[1],
z_stop=extents[0],
x_voxel_size=voxel_size[2],
y_voxel_size=voxel_size[1],
z_voxel_size=voxel_size[0],
voxel_unit=voxel_unit,
)
self.volume_provider.create_project(coordframe)
# create experiment if it doesn't exist:
try:
# Try to get an existing experiment:
experiment = self.volume_provider.get_project(
ExperimentResource(uri.experiment, uri.collection))
except:
# Create the experiment:
experiment = ExperimentResource(
uri.experiment,
uri.collection,
description=experiment_desc or description,
coord_frame=coordframe.name,
num_hierarchy_levels=downsample_levels,
hierarchy_method=downsample_method,
)
self.volume_provider.create_project(experiment)
# create channel if it doesn't exist:
try:
# Try to get an existing channel:
channel = self.volume_provider.get_project(
ChannelResource(uri.channel, uri.collection,
uri.experiment))
except:
# Create the channel:
channel = ChannelResource(
uri.channel,
uri.collection,
uri.experiment,
description=description,
type="image"
if dtype in ["uint8", "uint16"] else "annotation",
datatype=dtype,
sources=[source_channel] if source_channel else [],
)
self.volume_provider.create_project(channel)
self.resolution = resolution
# If the channel is set as a Resource, then use that resource.
if isinstance(channel, ChannelResource):
self._channel = channel
# If it is set as a string, then parse the channel and generate an
# intern.Resource from a bossDB URI.
else: # if isinstance(channel, str):
uri = {
"BossDB": _parse_bossdb_uri,
"CloudVolumeOpenData": _parse_cloudvolume_uri,
}[self.volume_provider.get_vp_type()](channel)
self.resolution = (uri.resolution if not (uri.resolution is None)
else self.resolution)
self._channel = self.volume_provider.get_channel(
uri.channel, uri.collection, uri.experiment)
# Set col/exp/chan based upon the channel or URI provided.
self.collection_name = self._channel.coll_name
self.experiment_name = self._channel.exp_name
self.channel_name = self._channel.name
# Create a pointer to the metadata for the channel.
self._channel_metadata = Metadata(self._channel)
def get_available_resolutions(self, channel: Resource) -> List[int]:
experiment = self.get_project(
ExperimentResource(channel.exp_name, channel.coll_name))
return list(range(experiment.num_hierarchy_levels))