def wrapper(src, dst, *args, **kwargs): logger.debug(f"save_metadata wrapper: src {src}, dst {dst}") from pprint import pprint pprint(args) pprint(kwargs) result = func(src, dst, *args, **kwargs) ds = dataset_from_uri(dst, mode="r+") if ds.supports_metadata(): for param in ["kind", "name"]: if not ds.has_attr(param): print(f"Setting param {param} {fname}") ds.set_attr(param, fname) for k, v in kwargs.items(): print(f"Setting key value {k}, {v}") ds.set_attr(k, v) if type(src) == list: src_id = [dataset_from_uri(s, mode="r").id for s in src] else: src_id = dataset_from_uri(src, mode="r").id ds.set_attr("source", src_id) else: logger.debug("Dataset doesn't support metadata.") result = dataset_repr(ds) ds.close() logger.info("+ Computed: {}".format(fname)) return result
def annotate_regions( workspace: String, level: String, region: DataURI, r: IntList, label: Int, full: SmartBoolean, parent_level: String, parent_label_idx: Int, bb: IntList, viewer_order: tuple, ): from survos2.api.annotate import annotate_regions ds = get_level(workspace, level, full) region = dataset_from_uri(region, mode="r") from survos2.frontend.frontend import get_level_from_server if parent_level != "-1" and parent_level != -1: parent_arr, parent_annotations_dataset = get_level_from_server( {"level_id": parent_level}, retrieval_mode="volume" ) parent_arr = parent_arr & 15 # print(f"Using parent dataset for masking {parent_annotations_dataset}") parent_mask = parent_arr == parent_label_idx else: # print("Not masking using parent level") parent_arr = None parent_mask = None logger.debug(f"BB in annotate_regions {bb}") anno = annotate_regions( ds, region, r=r, label=label, parent_mask=parent_mask, bb=bb, viewer_order=viewer_order, ) def pass_through(x): return x dst = DataModel.g.dataset_uri(level, group="annotations") map_blocks(pass_through, anno, out=dst, normalize=False) modified_ds = dataset_from_uri(dst, mode="r") modified = [1] modified_ds.set_attr("modified", modified)
def upload(body, request, response): print(f"Request: {request}") print(f"Response: {response}") encoded_array = body['file'] array_shape = body['shape'] anno_id = body['name'] print(f"shape {array_shape} name {anno_id}") level_arr = np.frombuffer(encoded_array, dtype="uint32") print(f"level_arr: {level_arr.shape}") from ast import literal_eval level_arr.shape = literal_eval(array_shape) print(f"Uploaded feature of shape {level_arr.shape}") dst = DataModel.g.dataset_uri(anno_id, group="annotations") with DatasetManager(dst, out=dst, dtype="uint32", fillvalue=0) as DM: DM.out[:] = level_arr modified_ds = dataset_from_uri(dst, mode="r") modified = [1] modified_ds.set_attr("modified", modified)
def prepare_features(features, roi_crop, resample_amt): """Calculate filters on image volume to generate features for survos segmentation Arguments: features {list of string} -- list of feature uri roi_crop {tuple of int} -- tuple defining a bounding box for cropping the image volume resample_amt {float} -- amount to scale the input volume Returns: features -- dataclass containing the processed image layers, and a stack made from them """ # features_stack = [] filtered_layers = [] for i, feature in enumerate(features): logger.info(f"Loading feature number {i}: {os.path.basename(feature)}") data = dataset_from_uri(feature, mode="r") data = data[roi_crop[0]:roi_crop[1], roi_crop[2]:roi_crop[3], roi_crop[4]:roi_crop[5], ] data = scipy.ndimage.zoom(data, resample_amt, order=1) logger.info(f"Cropped and resampled feature shape: {data.shape}") filtered_layers.append(data) # features_stack.append(data[...].ravel()) # features_stack = np.stack(features_stack, axis=1) dataset_feats, features_stack = prepare_prediction_features( filtered_layers) features = SRFeatures(filtered_layers, dataset_feats, features_stack) return features
def test_data(data_func): dataset = dataset_from_uri(data_func()) data = asnparray(dataset) assert data.shape == data_func.__shape__ assert data.dtype == np.float32 assert data.max() == 1.0 assert data.min() == 0.0 dataset.close()
def wrapper(src, dst, *args, **kwargs): result = func(src, dst, *args, **kwargs) ds = dataset_from_uri(dst, mode='r+') if ds.supports_metadata(): for param in ['kind', 'name']: if not ds.has_attr(param): ds.set_attr(param, fname) for k, v in kwargs.items(): ds.set_attr(k, v) if type(src) == list: src_id = [dataset_from_uri(s, mode='r').id for s in src] else: src_id = dataset_from_uri(src, mode='r').id ds.set_attr('source', src_id) result = dataset_repr(ds) ds.close() logger.info('+ Computed: {}'.format(fname)) return result
def load_metadata(source: "Dataset URI to load in HDF5, MRC or SuRVoS format"): from survos2.io import dataset_from_uri, supports_metadata if supports_metadata(source): source = dataset_from_uri(source) logger.info(format_yaml(source.metadata())) source.close() else: logger.info("Dataset `{}` has no metadata.".format(source))
def annotate_regions(workspace: String, level: String, region: DataURI, r: IntList, label: Int, full: SmartBoolean = False): from survos2.api.annotations.annotate import annotate_regions as annotate ds = get_level(workspace, level, full) region = dataset_from_uri(region, mode='r') annotate(ds, region, r=r, label=label)
def add_data(workspace: String, dataset: String): import dask.array as da from survos2.improc.utils import optimal_chunksize ws = get(workspace) with dataset_from_uri(dataset, mode='r') as data: chunk_size = optimal_chunksize(data, Config['computing.chunk_size']) data = da.from_array(data, chunks=chunk_size) data -= da.min(data) data /= da.max(data) ds = ws.add_data(data) logger.info(type(ds)) return ds
def view_data( source: "Dataset URI to load in HDF5, MRC or SuRVoS format", boundaries: "Boundaries to show on top of the `source`" = None, overlay: "Overlay dataset to show on top of the `source`" = None, bcolor: "Color of the overlaid boundaries" = "#000099", balpha: "Alpha of the overlaid boundaries" = 0.7, oalpha: "Overlay alpha." = 0.5, ): """ Visualizes a 3D volume with sliders. Allows to overlay a segmentation / other image and to overlay boundaries, from either supervoxels or other boundary extraction method. """ from survos2.io import dataset_from_uri from survos2.volume_utils import view logger.info(f"Loading source volume {source}") source = dataset_from_uri(source) if boundaries: logger.info("Loading boundaries") boundaries = dataset_from_uri(boundaries) if overlay: logger.info("Loading overlay") overlay = dataset_from_uri(overlay) view( source, boundaries=boundaries, overlay=overlay, bcolor=bcolor, balpha=balpha, oalpha=oalpha, ) source.close() if boundaries: boundaries.close() if overlay: overlay.close()
def pull_anno(roi_fname: String): roi_ws = ws.get(roi_fname) ds = ws.get_dataset(roi_fname, '001_level', group="annotations") roi_parts = roi_fname.split("_") z_min = int(roi_parts[-6]) z_max = int(roi_parts[-5]) x_min = int(roi_parts[-4]) x_max = int(roi_parts[-3]) y_min = int(roi_parts[-2]) y_max = int(roi_parts[-1]) dst = DataModel.g.dataset_uri('001_level', group="annotations") main_anno = dataset_from_uri(dst, mode="rw") main_anno[z_min:z_max, x_min:x_max, y_min:y_max] = ds[:]
def annotate_voxels( workspace: String, level: String, slice_idx: Int, yy: IntList, xx: IntList, label: Int, full: SmartBoolean, parent_level: String, parent_label_idx: Int, viewer_order: tuple, three_dim: SmartBoolean, brush_size: Int, centre_point: tuple ): from survos2.api.annotate import annotate_voxels ds = get_level(workspace, level, full) from survos2.frontend.frontend import get_level_from_server if parent_level != "-1" and parent_level != -1: parent_arr, parent_annotations_dataset = get_level_from_server( {"level_id": parent_level}, retrieval_mode="volume" ) parent_arr = parent_arr & 15 parent_mask = parent_arr == parent_label_idx else: parent_arr = None parent_mask = None logger.info(f"slice_idx {slice_idx} Viewer order: {viewer_order}") annotate_voxels( ds, slice_idx=slice_idx, yy=yy, xx=xx, label=label, parent_mask=parent_mask, viewer_order=viewer_order, three_dim = three_dim, brush_size=brush_size, centre_point=centre_point ) dst = DataModel.g.dataset_uri(level, group="annotations") modified_ds = dataset_from_uri(dst, mode="r") modified = [1] modified_ds.set_attr("modified", modified)
def prepare_supervoxels( supervoxels: List[str], filtered_stack: np.ndarray, roi_crop: np.ndarray, resample_amt: float, ) -> SRData: """Load supervoxels from file, then generate supervoxel features from a features stack and the supervoxel rag,then bundle as SRData and return. Args: supervoxels (List[str]): list of supervoxels filtered_stack (np.ndarray): stack of filters roi_crop (np.ndarray): roi to crop to resample_amt (float): zoom level Returns: [SRData]: superregions dataclass object """ logger.debug(f"Loading supervoxel file {supervoxels[0]}") logger.debug(f"Roi crop {roi_crop}") supervoxel_vol = dataset_from_uri(supervoxels[0], mode="r") supervoxel_vol = np.array(supervoxel_vol).astype(np.uint32, copy=False) supervoxel_vol = np.nan_to_num(supervoxel_vol) supervoxel_proc = supervoxel_vol[roi_crop[0]:roi_crop[1], roi_crop[2]:roi_crop[3], roi_crop[4]:roi_crop[5]].astype( np.uint32, copy=False) supervoxel_proc = scipy.ndimage.zoom(supervoxel_proc, resample_amt, order=1) logger.debug( f"Loading Supervoxel {os.path.basename(supervoxels[0])} with shape {supervoxel_proc.shape}" ) supervoxel_features = rmeans(filtered_stack, supervoxel_proc) supervoxel_rag = create_rag(np.array(supervoxel_proc).astype(np.uint32), connectivity=6) supervoxel_features = [] supervoxel_rag = [] superregions = SRData(supervoxel_proc, supervoxel_features, supervoxel_rag) return superregions
def get_entities_metadata(src: DataURI): ds = dataset_from_uri(src, mode="r")[:] with DatasetManager(src, out=None, dtype="float32", fillvalue=0) as DM: ds_objects = DM.sources[0] logger.debug(f"Using dataset {ds_objects}") entities_metadata = { 'fullname': ds_objects.get_metadata("fullname"), 'scale': ds_objects.get_metadata("scale"), 'offset': ds_objects.get_metadata("offset"), 'crop_start': ds_objects.get_metadata("crop_start"), 'crop_end': ds_objects.get_metadata("crop_end") } return entities_metadata
def add_data(workspace: String, data_fname: String): import dask.array as da from survos2.improc.utils import optimal_chunksize ws = get(workspace) logger.info(f"Adding data to workspace {ws}") with dataset_from_uri(data_fname, mode="r") as data: chunk_size = optimal_chunksize(data, Config["computing.chunk_size"]) logger.debug( f'Calculating optimal chunk size using chunk_size {Config["computing.chunk_size"]}: {chunk_size}' ) data = da.from_array(data, chunks=chunk_size) data -= da.min(data) data /= da.max(data) ds = ws.add_data(data) # ds.set_attr("chunk_size", chunk_size) return ds
def test_workspace(): ws = Workspace(".") workspace_fpath = "./newws1" ws = ws.create(workspace_fpath) data_fname = "./tmp/testvol_4x4x4b.h5" with dataset_from_uri(data_fname, mode="r") as data: chunk_size = optimal_chunksize(data, Config["computing.chunk_size"]) data = da.from_array(data, chunks=chunk_size) data -= da.min(data) data /= da.max(data) ds = ws.add_data(data) # ds.set_attr("chunk_size", chunk_size) ws.add_dataset("testds", "float32") assert ws.exists(workspace_fpath) assert ws.has_data() assert ws.available_datasets() == ['testds'] ws.add_session('newsesh') assert ws.has_session('newsesh') ws.delete()
def get_slice(src:DataURI, slice_idx:Int): ds = dataset_from_uri(src, mode='r') data = ds[slice_idx] return encode_numpy(data)
def get_crop(src: DataURI, roi: IntList): logger.debug("Getting anno crop") ds = dataset_from_uri(src, mode="r") data = ds[roi[0] : roi[1], roi[2] : roi[3], roi[4] : roi[5]] return encode_numpy(data)
def get_slice(src: DataURI, slice_idx: Int, order: tuple): ds = dataset_from_uri(src, mode="r")[:] ds = np.transpose(ds, order) data = ds[slice_idx] return encode_numpy(data)
def get_volume(src: DataURI): logger.debug("Getting annotation volume") ds = dataset_from_uri(src, mode="r") data = ds[:] return encode_numpy(data)
def get_slice(src: DataURI, slice_idx: Int, order: tuple): order = np.array(order) ds = dataset_from_uri(src, mode="r")[:] ds = np.transpose(ds, order).astype(np.float32) data = ds[slice_idx] return encode_numpy_slice(data.astype(np.float32))