def get_voxel_unit(self, channel: ChannelResource, resolution: int = 0) -> str: experiment = self.get_project( ExperimentResource(channel.exp_name, channel.coll_name)) # Get the coordinate frame: cf = self.get_project(CoordinateFrameResource(experiment.coord_frame)) return cf.voxel_unit
def get_voxel_size(self, channel: ChannelResource, resolution: int = 0) -> Tuple[float, float, float]: experiment = self.get_project( ExperimentResource(channel.exp_name, channel.coll_name)) # Get the coordinate frame: cf = self.get_project(CoordinateFrameResource(experiment.coord_frame)) return (cf.x_voxel_size, cf.y_voxel_size, cf.z_voxel_size)
def _get_coord_frame_details(self): exp_resource = ExperimentResource(self.experiment, self.collection) coord_frame = self._bossRemote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = self._bossRemote.get_project(coord_frame_resource) self.max_dimensions = (data.z_stop, data.y_stop, data.x_stop) self.voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size)
def get_coord_frame_details(rmt, coll, exp): exp_resource = ExperimentResource(exp, coll) coord_frame = rmt.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = rmt.get_project(coord_frame_resource) max_dimensions = (data.z_stop, data.y_stop, data.x_stop) voxel_size = (data.z_voxel_size, data.y_voxel_size, data.x_voxel_size) return max_dimensions, voxel_size
def fetch_info(self): experiment = ExperimentResource(name=self.path.dataset, collection_name=self.path.bucket) rmt = BossRemote(boss_credentials) experiment = rmt.get_project(experiment) coord_frame = CoordinateFrameResource(name=experiment.coord_frame) coord_frame = rmt.get_project(coord_frame) channel = ChannelResource(self.path.layer, self.path.bucket, self.path.dataset) channel = rmt.get_project(channel) unit_factors = { 'nanometers': 1, 'micrometers': 1e3, 'millimeters': 1e6, 'centimeters': 1e7, } unit_factor = unit_factors[coord_frame.voxel_unit] cf = coord_frame resolution = [cf.x_voxel_size, cf.y_voxel_size, cf.z_voxel_size] resolution = [int(round(_)) * unit_factor for _ in resolution] bbox = Bbox((cf.x_start, cf.y_start, cf.z_start), (cf.x_stop, cf.y_stop, cf.z_stop)) bbox.maxpt = bbox.maxpt layer_type = 'unknown' if 'type' in channel.raw: layer_type = channel.raw['type'] info = CloudVolume.create_new_info( num_channels=1, layer_type=layer_type, data_type=channel.datatype, encoding='raw', resolution=resolution, voxel_offset=bbox.minpt, volume_size=bbox.size3(), chunk_size=(512, 512, 16), # fixed in s3 implementation ) each_factor = Vec(2, 2, 1) if experiment.hierarchy_method == 'isotropic': each_factor = Vec(2, 2, 2) factor = each_factor.clone() for _ in range(1, experiment.num_hierarchy_levels): self.add_scale(factor, info=info) factor *= each_factor return info
def setup_experiment_boss(remote, collection, experiment): """ Get experiment and coordinate frame information from the boss. """ exp_setup = ExperimentResource(experiment, collection) try: exp_actual = remote.get_project(exp_setup) coord_setup = CoordinateFrameResource(exp_actual.coord_frame) coord_actual = remote.get_project(coord_setup) return (exp_setup, coord_actual) except Exception as e: print(e.message)
def get_shape(self, channel: ChannelResource, resolution: int = 0) -> Tuple[int, int, int]: # Get the experiment: experiment = self.get_project( ExperimentResource(channel.exp_name, channel.coll_name)) # Get the coordinate frame: cf = self.get_project(CoordinateFrameResource(experiment.coord_frame)) # Return the bounds of the coordinate frame: return ( (cf.z_stop - cf.z_start), int((cf.y_stop - cf.y_start) / (2**resolution)), int((cf.x_stop - cf.x_start) / (2**resolution)), )
def ingest_volume(host, token, channel_name, collection, experiment, volume): """ Assumes the collection and experiment exists in BOSS. """ remote = BossRemote({'protocol': 'https', 'host': host, 'token': token}) if volume.dtype == 'uint64': dtype = 'uint64' img_type = 'annotation' sources = ['empty'] else: dtype = volume.dtype.name img_type = 'image' sources = [] try: channel_resource = ChannelResource(channel_name, collection, experiment) channel = remote.get_project(channel_resource) except: channel_resource = ChannelResource(channel_name, collection, experiment, type=img_type, sources=sources, datatype=dtype) channel = remote.create_project(channel_resource) #Get max size of experiment exp_resource = ExperimentResource(experiment, collection) coord_frame = remote.get_project(exp_resource).coord_frame coord_frame_resource = CoordinateFrameResource(coord_frame) data = remote.get_project(coord_frame_resource) y_stop, x_stop = data.y_stop, data.x_stop for z in range(volume.shape[0]): print('Uploading {} slice'.format(z)) remote.create_cutout(channel, 0, (0, x_stop), (0, y_stop), (z, z + 1), volume[z, :, :].reshape((-1, y_stop, x_stop)))
def __init__(self, resource: Union[BossResource, str], remote: BossRemote = None): self._remote = remote or BossRemote(_DEFAULT_BOSS_OPTIONS) if isinstance(resource, str): resource = resource.split("://")[-1] path_items = resource.split("/") if len(path_items) == 1: self._resource = self._remote.get_project( CollectionResource(path_items[0])) elif len(path_items) == 2: self._resource = self._remote.get_project( ExperimentResource(path_items[1], path_items[0])) elif len(path_items) == 3: self._resource = self._remote.get_project( ChannelResource(path_items[2], path_items[0], path_items[1])) else: raise ValueError(f"Invalid resource path: {resource}") else: self._resource = resource
def setUpClass(cls): '''Initialized once, before any class tests are run''' boss = cls.boss = Remote() fixtures = {} fixtures['coordframe'] = CoordinateFrameResource(COORD_FRAME) fixtures['collection'] = CollectionResource(COLLECTION) fixtures['experiment'] = ExperimentResource(EXPERIMENT, COLLECTION, coord_frame=COORD_FRAME) fixtures['channel'] = ChannelResource(CHANNEL, COLLECTION, EXPERIMENT, 'image') fixtures['ann_channel'] = ChannelResource(ANN_CHANNEL, COLLECTION, EXPERIMENT, 'annotation', sources=[CHANNEL]) cls.fixtures = fixtures for resource_name in [ 'coordframe', 'collection', 'experiment', 'channel', 'ann_channel' ]: boss.create_project(fixtures[resource_name])
def setUp(self): self.er = ExperimentResource('foo', 'bar', 'coords')
class TestExperimentResource(unittest.TestCase): def setUp(self): self.er = ExperimentResource('foo', 'bar', 'coords') def test_not_valid_volume(self): self.assertFalse(self.er.valid_volume()) def test_get_route(self): self.assertEqual( '{}/experiment/{}'.format(self.er.coll_name, self.er.name), self.er.get_route()) def test_get_list_route(self): self.assertEqual('{}/experiment/'.format(self.er.coll_name), self.er.get_list_route()) def test_hierarchy_method_setter(self): exp = 'anisotropic' self.er.hierarchy_method = exp self.assertEqual(exp, self.er.hierarchy_method) def test_validate_hierarchy_method_aniso(self): exp = 'anisotropic' self.assertEqual(exp, self.er.validate_hierarchy_method(exp)) def test_validate_hierarchy_method_iso(self): exp = 'isotropic' self.assertEqual(exp, self.er.validate_hierarchy_method(exp)) def test_validate_hierarchy_method_bad(self): with self.assertRaises(ValueError): self.er.validate_hierarchy_method('slice') def test_validate_time_units_blank(self): exp = '' self.assertEqual(exp, self.er.validate_time_units('')) def test_validate_time_units_ns(self): exp = 'nanoseconds' self.assertEqual(exp, self.er.validate_time_units(exp)) def test_validate_time_units_micro(self): exp = 'microseconds' self.assertEqual(exp, self.er.validate_time_units(exp)) def test_validate_time_units_ms(self): exp = 'milliseconds' self.assertEqual(exp, self.er.validate_time_units(exp)) def test_validate_time_units_s(self): exp = 'seconds' self.assertEqual(exp, self.er.validate_time_units(exp)) def test_validate_time_units_bad(self): with self.assertRaises(ValueError): self.er.validate_time_units('secs')
def __init__( self, channel: Union[ChannelResource, Tuple, str], resolution: int = 0, volume_provider: VolumeProvider = None, axis_order: str = AxisOrder.ZYX, create_new: bool = False, description: Optional[str] = None, dtype: Optional[str] = None, extents: Optional[Tuple[int, int, int]] = None, voxel_size: Optional[Tuple[int, int, int]] = None, voxel_unit: Optional[str] = None, downsample_levels: int = 6, downsample_method: Optional[str] = "anisotropic", coordinate_frame_name: Optional[str] = None, coordinate_frame_desc: Optional[str] = None, collection_desc: Optional[str] = None, experiment_desc: Optional[str] = None, source_channel: Optional[str] = None, boss_config: Optional[dict] = None, ) -> None: """ Construct a new intern-backed array. Arguments: channel (intern.resource.boss.ChannelResource): The channel from which data will be downloaded. resolution (int: 0): The native resolution or MIP to use volume_provider (VolumeProvider): The remote-like to use axis_order (str = AxisOrder.ZYX): The axis-ordering to use for data cutouts. Defaults to ZYX. DOES NOT affect the `voxel_size` or `extents` arguments to this constructor. create_new (bool: False): Whether to create new Resources if they do not exist. Does not work with public token. dtype (str): Only required if `create_new = True`. Specifies the numpy-style datatype for this new dataset (e.g. "uint8"). description (str): Only required if `create_new = True`. Sets the description for the newly-created collection, experiment, channel, and coordframe resources. extents: Optional[Tuple[int, int, int]]: Only required if `create_new = True`. Specifies the total dataset extents of this new dataset, in ZYX order. voxel_size: Optional[Tuple[int, int, int]]: Only required if `create_new = True`. Specifies the voxel dimensions of this new dataset, in ZYX order. voxel_unit: Optional[str]: Only required if `create_new = True`. Specifies the voxel-dimension unit. For example, "nanometers". downsample_levels (int: 6): The number of downsample levels. downsample_method (Optional[str]): The type of downsample to use. If unset, defaults to 'anisotropic'. coordinate_frame_name (Optional[str]): If set, the name to use for the newly created coordinate frame. If not set, the name of the coordinate frame will be chosen automatically. coordinate_frame_desc (Optional[str]): If set, the description text to use for the newly created coordinate frame. If not set, the description will be chosen automatically. collection_desc (Optional[str]): The description text to use for a newly created collection. If not set, the description will be chosen automatically. experiment_desc (Optional[str]): The description text to use for a newly created experiment. If not set, the description will be chosen automatically. source_channel (Optional[str]): The channel to use as the source for this new channel, if `create_new` is True and this is going to be an annotation channel (dtype!=uint8). boss_config (Optional[dict]): The BossRemote configuration dict to use in order to authenticate with a BossDB remote. This option is mutually exclusive with the VolumeProvider configuration. If the `volume_provider` arg is set, this will be ignored. """ self.axis_order = axis_order # Handle inferring the remote from the channel: volume_provider = volume_provider or _infer_volume_provider(channel) if volume_provider is None: if boss_config: volume_provider = _BossDBVolumeProvider( BossRemote(boss_config)) else: volume_provider = _BossDBVolumeProvider() # Handle custom Remote: self.volume_provider = volume_provider if create_new: if self.volume_provider.get_vp_type() != "bossdb": raise ValueError( "Creating new resources with a non-bossdb volume provider is not currently supported." ) # We'll need at least `extents` and `voxel_size`. description = description or "Created with intern" dtype = dtype or "uint8" if extents is None: raise ValueError( "If `create_new` is True, you must specify the extents of the new coordinate frame as a [x, y, z]." ) if voxel_size is None: raise ValueError( "If `create_new` is True, you must specify the voxel_size of the new coordinate frame as a [x, y, z]." ) uri = _parse_bossdb_uri(channel) # create collection if it doesn't exist: try: # Try to get an existing collection: collection = self.volume_provider.get_project( CollectionResource(uri.collection)) except: # Create the collection: collection = CollectionResource(uri.collection, description=collection_desc or description) self.volume_provider.create_project(collection) # create coordframe if it doesn't exist: try: # Try to get an existing coordframe: coordframe = self.volume_provider.get_project( CoordinateFrameResource( coordinate_frame_name or f"CF_{uri.collection}_{uri.experiment}")) except: # Default to nanometers if a voxel unit isn't provided voxel_unit = voxel_unit or "nanometers" # Create the coordframe: coordframe = CoordinateFrameResource( coordinate_frame_name or f"CF_{uri.collection}_{uri.experiment}", description=coordinate_frame_desc or description, x_start=0, y_start=0, z_start=0, x_stop=extents[2], y_stop=extents[1], z_stop=extents[0], x_voxel_size=voxel_size[2], y_voxel_size=voxel_size[1], z_voxel_size=voxel_size[0], voxel_unit=voxel_unit, ) self.volume_provider.create_project(coordframe) # create experiment if it doesn't exist: try: # Try to get an existing experiment: experiment = self.volume_provider.get_project( ExperimentResource(uri.experiment, uri.collection)) except: # Create the experiment: experiment = ExperimentResource( uri.experiment, uri.collection, description=experiment_desc or description, coord_frame=coordframe.name, num_hierarchy_levels=downsample_levels, hierarchy_method=downsample_method, ) self.volume_provider.create_project(experiment) # create channel if it doesn't exist: try: # Try to get an existing channel: channel = self.volume_provider.get_project( ChannelResource(uri.channel, uri.collection, uri.experiment)) except: # Create the channel: channel = ChannelResource( uri.channel, uri.collection, uri.experiment, description=description, type="image" if dtype in ["uint8", "uint16"] else "annotation", datatype=dtype, sources=[source_channel] if source_channel else [], ) self.volume_provider.create_project(channel) self.resolution = resolution # If the channel is set as a Resource, then use that resource. if isinstance(channel, ChannelResource): self._channel = channel # If it is set as a string, then parse the channel and generate an # intern.Resource from a bossDB URI. else: # if isinstance(channel, str): uri = { "BossDB": _parse_bossdb_uri, "CloudVolumeOpenData": _parse_cloudvolume_uri, }[self.volume_provider.get_vp_type()](channel) self.resolution = (uri.resolution if not (uri.resolution is None) else self.resolution) self._channel = self.volume_provider.get_channel( uri.channel, uri.collection, uri.experiment) # Set col/exp/chan based upon the channel or URI provided. self.collection_name = self._channel.coll_name self.experiment_name = self._channel.exp_name self.channel_name = self._channel.name # Create a pointer to the metadata for the channel. self._channel_metadata = Metadata(self._channel)
def get_available_resolutions(self, channel: Resource) -> List[int]: experiment = self.get_project( ExperimentResource(channel.exp_name, channel.coll_name)) return list(range(experiment.num_hierarchy_levels))