def __get_spec(self, array_key):
info = self.__get_info(array_key)
roi_min = info['Extended']['MinPoint']
if roi_min is not None:
roi_min = Coordinate(roi_min[::-1])
roi_max = info['Extended']['MaxPoint']
if roi_max is not None:
roi_max = Coordinate(roi_max[::-1])
data_roi = Roi(offset=roi_min, shape=(roi_max - roi_min))
data_dims = Coordinate(data_roi.get_shape())
if self.ndims is None:
self.ndims = len(data_dims)
else:
assert self.ndims == len(data_dims)
if array_key in self.array_specs:
spec = self.array_specs[array_key].copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
spec.voxel_size = Coordinate(info['Extended']['VoxelSize'])
if spec.roi is None:
spec.roi = data_roi * spec.voxel_size
data_dtype = dvision.DVIDDataInstance(self.hostname, self.port,
self.uuid,
self.datasets[array_key]).dtype
if spec.dtype is not None:
assert spec.dtype == data_dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from instance %s dtype %s" %
(self.array_specs[array_key].dtype, array_key,
self.datasets[array_key], data_dtype))
else:
spec.dtype = data_dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning(
"WARNING: You didn't set 'interpolatable' for %s. "
"Based on the dtype %s, it has been set to %s. "
"This might not be what you want.", array_key, spec.dtype,
spec.interpolatable)
return spec
def __read_spec(self, array_key, data_file, ds_name):
dataset = data_file[ds_name]
if array_key in self.array_specs:
spec = self.array_specs[array_key].copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
voxel_size = self._get_voxel_size(dataset)
if voxel_size is None:
voxel_size = Coordinate((1, ) * len(dataset.shape))
logger.warning(
"WARNING: File %s does not contain resolution information "
"for %s (dataset %s), voxel size has been set to %s. This "
"might not be what you want.", self.filename, array_key,
ds_name, spec.voxel_size)
spec.voxel_size = voxel_size
self.ndims = len(spec.voxel_size)
if spec.roi is None:
offset = self._get_offset(dataset)
if offset is None:
offset = Coordinate((0, ) * self.ndims)
if self.channels_first:
shape = Coordinate(dataset.shape[-self.ndims:])
else:
shape = Coordinate(dataset.shape[:self.ndims])
spec.roi = Roi(offset, shape * spec.voxel_size)
if spec.dtype is not None:
assert spec.dtype == dataset.dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from dataset %s dtype %s" %
(self.array_specs[array_key].dtype, array_key, ds_name,
dataset.dtype))
else:
spec.dtype = dataset.dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning(
"WARNING: You didn't set 'interpolatable' for %s "
"(dataset %s). Based on the dtype %s, it has been "
"set to %s. This might not be what you want.", array_key,
ds_name, spec.dtype, spec.interpolatable)
return spec
def __read_spec(self, cv):
# dataset = data_file[ds_name]
dims = Coordinate(cv.bounds.maxpt[::-1]) * self._get_voxel_size(cv)
if self.ndims is None:
self.ndims = cv.bounds.ndim
else:
assert self.ndims == len(dims)
if self.array_spec is not None:
spec = self.array_spec.copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
voxel_size = self._get_voxel_size(cv)
if voxel_size is None:
voxel_size = Coordinate((1,) * self.ndims)
logger.warning(
"WARNING: File %s does not contain resolution information "
"for %s , voxel size has been set to %s. This "
"might not be what you want.",
self.filename, array_key, spec.voxel_size)
spec.voxel_size = voxel_size
if spec.roi is None:
offset = self._get_offset(cv)
if offset is None:
offset = Coordinate((0,) * self.ndims)
spec.roi = Roi(offset, dims * spec.voxel_size)
if spec.dtype is not None:
assert spec.dtype == cv.dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from cloudvolume dtype %s" %
(self.array_spec.dtype,
self.array_key, dataset.dtype))
else:
spec.dtype = cv.dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning("WARNING: You didn't set 'interpolatable' for %s "
". Based on the dtype %s, it has been "
"set to %s. This might not be what you want.",
self.array_key, spec.dtype,
spec.interpolatable)
return spec
def __get_mask_spec(self, mask_key):
# create initial array spec
if mask_key in self.array_specs:
spec = self.array_specs[mask_key].copy()
else:
spec = ArraySpec()
# get voxel size
if spec.voxel_size is None:
voxel_size = None
for array_key in self.datasets:
if voxel_size is None:
voxel_size = self.spec[array_key].voxel_size
else:
assert voxel_size == self.spec[array_key].voxel_size, (
"No voxel size was given for mask %s, and the voxel "
"sizes of the volumes %s are not all the same. I don't "
"know what voxel size to use to create the mask." %
(mask_key, self.datasets.keys()))
spec.voxel_size = voxel_size
# get ROI
if spec.roi is None:
for array_key in self.datasets:
roi = self.spec[array_key].roi
if spec.roi is None:
spec.roi = roi.copy()
else:
spec.roi = roi.union(spec.roi)
# set interpolatable
if spec.interpolatable is None:
spec.interpolatable = False
# set datatype
if spec.dtype is not None and spec.dtype != np.uint8:
logger.warn(
"Ignoring dtype in array_spec for %s, only np.uint8 "
"is allowed for masks.", mask_key)
spec.dtype = np.uint8
return spec
def __read_spec(self, array_key, data_file, ds_name):
dataset = data_file[ds_name]
if array_key in self.array_specs:
spec = self.array_specs[array_key].copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
voxel_size = self._get_voxel_size(dataset)
if voxel_size is None:
voxel_size = Coordinate((1, ) * len(dataset.shape))
spec.voxel_size = voxel_size
self.ndims = len(spec.voxel_size)
if spec.roi is None:
offset = self._get_offset(dataset)
if offset is None:
offset = Coordinate((0, ) * self.ndims)
if self.data_format == 'channels_first':
shape = Coordinate(dataset.shape[-self.ndims:])
if self.data_format == 'channels_last':
shape = Coordinate(dataset.shape[0:self.ndims])
spec.roi = Roi(offset, shape * spec.voxel_size)
if spec.dtype is not None:
assert spec.dtype == dataset.dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from dataset %s dtype %s" %
(self.array_specs[array_key].dtype, array_key, ds_name,
dataset.dtype))
else:
spec.dtype = dataset.dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
return spec
def __read_spec(self, headers):
num_files = len(headers)
assert num_files > 0
common_header = headers[0]
for header in headers:
for attr in ['imagesize_tczyx', 'pixelspacing_tczyx']:
assert (common_header[attr] == header[attr]).all(), (
"Headers of provided KLB files differ in attribute %s"%attr)
assert common_header['datatype'] == header['datatype'], (
"Headers of provided KLB files differ in attribute datatype")
size = Coordinate(common_header['imagesize_tczyx'])
voxel_size = Coordinate(common_header['pixelspacing_tczyx'])
dtype = common_header['datatype']
# strip leading 1 dimensions
while size[0] == 1 and len(size) > 1:
size = size[1:]
voxel_size = voxel_size[1:]
# append num_files dimension
if num_files > 1:
size = (num_files,) + size
voxel_size = (1,) + voxel_size
dims = Coordinate(size)
self.ndims = len(dims)
if self.array_spec is not None:
spec = self.array_spec
else:
spec = ArraySpec()
if spec.voxel_size is None:
spec.voxel_size = Coordinate(voxel_size)
if spec.roi is None:
offset = Coordinate((0,)*self.ndims)
spec.roi = Roi(offset, dims*spec.voxel_size)
if spec.dtype is not None:
assert spec.dtype == dtype, (
"dtype %s provided in array_specs for %s, but differs from "
"dataset dtype %s"%(
self.array_specs[self.array].dtype, self.array,
dataset.dtype))
else:
spec.dtype = dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning("WARNING: You didn't set 'interpolatable' for %s. "
"Based on the dtype %s, it has been set to %s. "
"This might not be what you want.",
self.array, spec.dtype, spec.interpolatable)
return spec
def __read_spec(self, array_key, base_folder, ds_name):
js_path = os.path.join(base_folder, "%s.json" % ds_name)
if not os.path.exists(js_path):
raise RuntimeError("%s.json not in %s" % (ds_name, base_folder))
dataset_spec = json.load(open(js_path, 'r'))
self.sec_dir = dataset_spec['sections_dir']
self.sections = dataset_spec['sections']
dims = np.asarray(dataset_spec['shape'])
self.tile_shape = np.asarray(dataset_spec['tile_size'])
# last two dims are assumed to be rows x columns
dims[-2:] *= self.tile_shape
if self.ndims is None:
self.ndims = len(dims)
else:
assert self.ndims == len(dims)
if array_key in self.array_specs:
spec = self.array_specs[array_key].copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
if 'resolution' in dataset_spec:
spec.voxel_size = Coordinate(dataset_spec['resolution'])
else:
spec.voxel_size = Coordinate((1, ) * self.ndims)
logger.warning(
"WARNING: File %s does not contain resolution information "
"for %s (dataset %s), voxel size has been set to %s. This "
"might not be what you want.", self.filename, array_key,
ds_name, spec.voxel_size)
if spec.roi is None:
if 'offset' in dataset_spec:
offset = Coordinate(dataset_spec['offset'])
else:
offset = Coordinate((0, ) * self.ndims)
spec.roi = Roi(offset, dims * spec.voxel_size)
if spec.dtype is not None:
assert spec.dtype == dataset.dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from dataset %s dtype %s" %
(self.array_specs[array_key].dtype, array_key, ds_name,
dataset_spec['dtype']))
else:
spec.dtype = dataset_spec['dtype']
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning(
"WARNING: You didn't set 'interpolatable' for %s "
"(dataset %s). Based on the dtype %s, it has been "
"set to %s. This might not be what you want.", array_key,
ds_name, spec.dtype, spec.interpolatable)
return spec
