def __get_spec(self, array_key): info = self.__get_info(array_key) roi_min = info['Extended']['MinPoint'] if roi_min is not None: roi_min = Coordinate(roi_min[::-1]) roi_max = info['Extended']['MaxPoint'] if roi_max is not None: roi_max = Coordinate(roi_max[::-1]) data_roi = Roi(offset=roi_min, shape=(roi_max - roi_min)) data_dims = Coordinate(data_roi.get_shape()) if self.ndims is None: self.ndims = len(data_dims) else: assert self.ndims == len(data_dims) if array_key in self.array_specs: spec = self.array_specs[array_key].copy() else: spec = ArraySpec() if spec.voxel_size is None: spec.voxel_size = Coordinate(info['Extended']['VoxelSize']) if spec.roi is None: spec.roi = data_roi * spec.voxel_size data_dtype = dvision.DVIDDataInstance(self.hostname, self.port, self.uuid, self.datasets[array_key]).dtype if spec.dtype is not None: assert spec.dtype == data_dtype, ( "dtype %s provided in array_specs for %s, " "but differs from instance %s dtype %s" % (self.array_specs[array_key].dtype, array_key, self.datasets[array_key], data_dtype)) else: spec.dtype = data_dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning( "WARNING: You didn't set 'interpolatable' for %s. " "Based on the dtype %s, it has been set to %s. " "This might not be what you want.", array_key, spec.dtype, spec.interpolatable) return spec
def __read_spec(self, array_key, data_file, ds_name): dataset = data_file[ds_name] if array_key in self.array_specs: spec = self.array_specs[array_key].copy() else: spec = ArraySpec() if spec.voxel_size is None: voxel_size = self._get_voxel_size(dataset) if voxel_size is None: voxel_size = Coordinate((1, ) * len(dataset.shape)) logger.warning( "WARNING: File %s does not contain resolution information " "for %s (dataset %s), voxel size has been set to %s. This " "might not be what you want.", self.filename, array_key, ds_name, spec.voxel_size) spec.voxel_size = voxel_size self.ndims = len(spec.voxel_size) if spec.roi is None: offset = self._get_offset(dataset) if offset is None: offset = Coordinate((0, ) * self.ndims) if self.channels_first: shape = Coordinate(dataset.shape[-self.ndims:]) else: shape = Coordinate(dataset.shape[:self.ndims]) spec.roi = Roi(offset, shape * spec.voxel_size) if spec.dtype is not None: assert spec.dtype == dataset.dtype, ( "dtype %s provided in array_specs for %s, " "but differs from dataset %s dtype %s" % (self.array_specs[array_key].dtype, array_key, ds_name, dataset.dtype)) else: spec.dtype = dataset.dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning( "WARNING: You didn't set 'interpolatable' for %s " "(dataset %s). Based on the dtype %s, it has been " "set to %s. This might not be what you want.", array_key, ds_name, spec.dtype, spec.interpolatable) return spec
def __read_spec(self, cv): # dataset = data_file[ds_name] dims = Coordinate(cv.bounds.maxpt[::-1]) * self._get_voxel_size(cv) if self.ndims is None: self.ndims = cv.bounds.ndim else: assert self.ndims == len(dims) if self.array_spec is not None: spec = self.array_spec.copy() else: spec = ArraySpec() if spec.voxel_size is None: voxel_size = self._get_voxel_size(cv) if voxel_size is None: voxel_size = Coordinate((1,) * self.ndims) logger.warning( "WARNING: File %s does not contain resolution information " "for %s , voxel size has been set to %s. This " "might not be what you want.", self.filename, array_key, spec.voxel_size) spec.voxel_size = voxel_size if spec.roi is None: offset = self._get_offset(cv) if offset is None: offset = Coordinate((0,) * self.ndims) spec.roi = Roi(offset, dims * spec.voxel_size) if spec.dtype is not None: assert spec.dtype == cv.dtype, ( "dtype %s provided in array_specs for %s, " "but differs from cloudvolume dtype %s" % (self.array_spec.dtype, self.array_key, dataset.dtype)) else: spec.dtype = cv.dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning("WARNING: You didn't set 'interpolatable' for %s " ". Based on the dtype %s, it has been " "set to %s. This might not be what you want.", self.array_key, spec.dtype, spec.interpolatable) return spec
def __get_mask_spec(self, mask_key): # create initial array spec if mask_key in self.array_specs: spec = self.array_specs[mask_key].copy() else: spec = ArraySpec() # get voxel size if spec.voxel_size is None: voxel_size = None for array_key in self.datasets: if voxel_size is None: voxel_size = self.spec[array_key].voxel_size else: assert voxel_size == self.spec[array_key].voxel_size, ( "No voxel size was given for mask %s, and the voxel " "sizes of the volumes %s are not all the same. I don't " "know what voxel size to use to create the mask." % (mask_key, self.datasets.keys())) spec.voxel_size = voxel_size # get ROI if spec.roi is None: for array_key in self.datasets: roi = self.spec[array_key].roi if spec.roi is None: spec.roi = roi.copy() else: spec.roi = roi.union(spec.roi) # set interpolatable if spec.interpolatable is None: spec.interpolatable = False # set datatype if spec.dtype is not None and spec.dtype != np.uint8: logger.warn( "Ignoring dtype in array_spec for %s, only np.uint8 " "is allowed for masks.", mask_key) spec.dtype = np.uint8 return spec
def __read_spec(self, array_key, data_file, ds_name): dataset = data_file[ds_name] if array_key in self.array_specs: spec = self.array_specs[array_key].copy() else: spec = ArraySpec() if spec.voxel_size is None: voxel_size = self._get_voxel_size(dataset) if voxel_size is None: voxel_size = Coordinate((1, ) * len(dataset.shape)) spec.voxel_size = voxel_size self.ndims = len(spec.voxel_size) if spec.roi is None: offset = self._get_offset(dataset) if offset is None: offset = Coordinate((0, ) * self.ndims) if self.data_format == 'channels_first': shape = Coordinate(dataset.shape[-self.ndims:]) if self.data_format == 'channels_last': shape = Coordinate(dataset.shape[0:self.ndims]) spec.roi = Roi(offset, shape * spec.voxel_size) if spec.dtype is not None: assert spec.dtype == dataset.dtype, ( "dtype %s provided in array_specs for %s, " "but differs from dataset %s dtype %s" % (self.array_specs[array_key].dtype, array_key, ds_name, dataset.dtype)) else: spec.dtype = dataset.dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] return spec
def __read_spec(self, headers): num_files = len(headers) assert num_files > 0 common_header = headers[0] for header in headers: for attr in ['imagesize_tczyx', 'pixelspacing_tczyx']: assert (common_header[attr] == header[attr]).all(), ( "Headers of provided KLB files differ in attribute %s"%attr) assert common_header['datatype'] == header['datatype'], ( "Headers of provided KLB files differ in attribute datatype") size = Coordinate(common_header['imagesize_tczyx']) voxel_size = Coordinate(common_header['pixelspacing_tczyx']) dtype = common_header['datatype'] # strip leading 1 dimensions while size[0] == 1 and len(size) > 1: size = size[1:] voxel_size = voxel_size[1:] # append num_files dimension if num_files > 1: size = (num_files,) + size voxel_size = (1,) + voxel_size dims = Coordinate(size) self.ndims = len(dims) if self.array_spec is not None: spec = self.array_spec else: spec = ArraySpec() if spec.voxel_size is None: spec.voxel_size = Coordinate(voxel_size) if spec.roi is None: offset = Coordinate((0,)*self.ndims) spec.roi = Roi(offset, dims*spec.voxel_size) if spec.dtype is not None: assert spec.dtype == dtype, ( "dtype %s provided in array_specs for %s, but differs from " "dataset dtype %s"%( self.array_specs[self.array].dtype, self.array, dataset.dtype)) else: spec.dtype = dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning("WARNING: You didn't set 'interpolatable' for %s. " "Based on the dtype %s, it has been set to %s. " "This might not be what you want.", self.array, spec.dtype, spec.interpolatable) return spec
def __read_spec(self, array_key, base_folder, ds_name): js_path = os.path.join(base_folder, "%s.json" % ds_name) if not os.path.exists(js_path): raise RuntimeError("%s.json not in %s" % (ds_name, base_folder)) dataset_spec = json.load(open(js_path, 'r')) self.sec_dir = dataset_spec['sections_dir'] self.sections = dataset_spec['sections'] dims = np.asarray(dataset_spec['shape']) self.tile_shape = np.asarray(dataset_spec['tile_size']) # last two dims are assumed to be rows x columns dims[-2:] *= self.tile_shape if self.ndims is None: self.ndims = len(dims) else: assert self.ndims == len(dims) if array_key in self.array_specs: spec = self.array_specs[array_key].copy() else: spec = ArraySpec() if spec.voxel_size is None: if 'resolution' in dataset_spec: spec.voxel_size = Coordinate(dataset_spec['resolution']) else: spec.voxel_size = Coordinate((1, ) * self.ndims) logger.warning( "WARNING: File %s does not contain resolution information " "for %s (dataset %s), voxel size has been set to %s. This " "might not be what you want.", self.filename, array_key, ds_name, spec.voxel_size) if spec.roi is None: if 'offset' in dataset_spec: offset = Coordinate(dataset_spec['offset']) else: offset = Coordinate((0, ) * self.ndims) spec.roi = Roi(offset, dims * spec.voxel_size) if spec.dtype is not None: assert spec.dtype == dataset.dtype, ( "dtype %s provided in array_specs for %s, " "but differs from dataset %s dtype %s" % (self.array_specs[array_key].dtype, array_key, ds_name, dataset_spec['dtype'])) else: spec.dtype = dataset_spec['dtype'] if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning( "WARNING: You didn't set 'interpolatable' for %s " "(dataset %s). Based on the dtype %s, it has been " "set to %s. This might not be what you want.", array_key, ds_name, spec.dtype, spec.interpolatable) return spec