def get_datasets(dataset, origin):
# output_shape = (40, 30, 80)
# input_shape = (100, 110, 120)
output_shape = (2, 3, 4)
input_shape = tuple([x + 2 for x in output_shape])
borders = tuple([(in_ - out_) / 2
for (in_, out_) in zip(input_shape, output_shape)])
input_slices = tuple(
[slice(x, x + l) for x, l in zip(origin, input_shape)])
output_slices = tuple([
slice(x + b, x + b + l)
for x, b, l in zip(origin, borders, output_shape)
])
expected_dataset = dict()
data_slices = [slice(0, l) for l in dataset['data'].shape]
data_slices[-3:] = input_slices
data_slices = tuple(data_slices)
expected_data_array = np.array(dataset['data'][data_slices],
dtype=np.float32)
expected_data_array = expected_data_array.reshape((1, ) + input_shape)
expected_data_array /= (2.0**8)
expected_dataset['data'] = expected_data_array
components_slices = [slice(0, l) for l in dataset['components'].shape]
components_slices[-3:] = output_slices
components_slices = tuple(components_slices)
expected_components_array = np.array(
dataset['components'][components_slices]).reshape((1, ) +
output_shape)
if type(dataset['components']) is DVIDDataInstance:
print("Is DVIDDataInstance...")
print("uniques before:", np.unique(expected_components_array))
dvid_uuid = dataset['components'].uuid
body_names_to_exclude = dataset.get('body_names_to_exclude')
good_bodies = get_good_components(dvid_uuid, body_names_to_exclude)
expected_components_array = \
replace_array_except_whitelist(expected_components_array, 0, good_bodies)
print("uniques after:", np.unique(expected_components_array))
expected_dataset['components'] = expected_components_array
components_for_affinity_generation = expected_components_array.reshape(
output_shape)
expected_label = malis.seg_to_affgraph(
components_for_affinity_generation, malis.mknhood3d())
expected_dataset['label'] = expected_label
if type(dataset['components']) is DVIDDataInstance:
expected_mask = np.array(expected_components_array > 0).astype(
np.uint8)
else:
expected_mask = np.ones(shape=(1, ) + output_shape, dtype=np.uint8)
expected_dataset['mask'] = expected_mask
numpy_dataset = get_numpy_dataset(dataset, input_slices, output_slices,
True)
return numpy_dataset, expected_dataset
def get_datasets(dataset, origin):
# output_shape = (40, 30, 80)
# input_shape = (100, 110, 120)
output_shape = (2,3,4)
input_shape = tuple([x + 2 for x in output_shape])
borders = tuple([(in_ - out_) / 2 for (in_, out_) in zip(input_shape, output_shape)])
input_slices = tuple([slice(x, x + l) for x, l in zip(origin, input_shape)])
output_slices = tuple([slice(x + b, x + b + l) for x, b, l in zip(origin, borders, output_shape)])
expected_dataset = dict()
data_slices = [slice(0, l) for l in dataset['data'].shape]
data_slices[-3:] = input_slices
data_slices = tuple(data_slices)
expected_data_array = np.array(dataset['data'][data_slices], dtype=np.float32)
expected_data_array = expected_data_array.reshape((1,) + input_shape)
expected_data_array /= (2.0 ** 8)
expected_dataset['data'] = expected_data_array
components_slices = [slice(0, l) for l in dataset['components'].shape]
components_slices[-3:] = output_slices
components_slices = tuple(components_slices)
expected_components_array = np.array(dataset['components'][components_slices]).reshape((1,) + output_shape)
if type(dataset['components']) is DVIDDataInstance:
print("Is DVIDDataInstance...")
print("uniques before:", np.unique(expected_components_array))
dvid_uuid = dataset['components'].uuid
body_names_to_exclude = dataset.get('body_names_to_exclude')
good_bodies = get_good_components(dvid_uuid, body_names_to_exclude)
expected_components_array = \
replace_array_except_whitelist(expected_components_array, 0, good_bodies)
print("uniques after:", np.unique(expected_components_array))
expected_dataset['components'] = expected_components_array
components_for_affinity_generation = expected_components_array.reshape(output_shape)
expected_label = malis.seg_to_affgraph(components_for_affinity_generation, malis.mknhood3d())
expected_dataset['label'] = expected_label
if type(dataset['components']) is DVIDDataInstance:
expected_mask = np.array(expected_components_array > 0).astype(np.uint8)
else:
expected_mask = np.ones(shape=(1,) + output_shape, dtype=np.uint8)
expected_dataset['mask'] = expected_mask
numpy_dataset = get_numpy_dataset(dataset, input_slices, output_slices, True)
return numpy_dataset, expected_dataset
def get_outputs(original_dataset, output_slice):
output_shape = tuple([slice_.stop - slice_.start for slice_ in output_slice])
n_spatial_dimensions = len(output_slice)
components_shape = (1,) + output_shape
mask_shape = (1,) + output_shape
affinities_shape = (n_spatial_dimensions,) + output_shape
component_slices = [slice(0, l) for l in original_dataset['components'].shape]
component_slices[-n_spatial_dimensions:] = output_slice
logger.debug("component_slices: {}".format(component_slices))
components_array = get_zero_padded_array_slice(original_dataset['components'], component_slices)
source_class = type(original_dataset['components'])
components_are_from_dvid = source_class in dvid_classes
exclude_strings = original_dataset.get('body_names_to_exclude', [])
if exclude_strings and components_are_from_dvid:
dvid_uuid = original_dataset['components'].uuid
components_to_keep = get_good_components(dvid_uuid, exclude_strings)
logger.debug("components before: {}".format(list(np.unique(components_array))))
components_array = replace_array_except_whitelist(components_array, 0, components_to_keep)
logger.debug("components after: {}".format(list(np.unique(components_array))))
minimum_component_size = original_dataset.get('minimum_component_size', 0)
if minimum_component_size > 0:
components_array = replace_infrequent_values(components_array, minimum_component_size, 0)
component_erosion_steps = original_dataset.get('component_erosion_steps', 0)
if component_erosion_steps > 0:
components_array = erode_value_blobs(
components_array,
steps=component_erosion_steps,
values_to_ignore=(0,))
components_for_malis = components_array.reshape(output_shape)
affinities_from_components = malis.seg_to_affgraph(
components_for_malis,
original_dataset['nhood'])
components_array, _ = malis.connected_components_affgraph(
affinities_from_components,
original_dataset['nhood'])
components_array = shift_up_component_values(components_array)
components_array = components_array.reshape(components_shape)
if 'label' in original_dataset:
label_shape = original_dataset['label'].shape
label_slices = [slice(0, l) for l in label_shape]
label_slices[-n_spatial_dimensions:] = output_slice
affinities_array = get_zero_padded_array_slice(original_dataset['label'], label_slices)
else:
# compute affinities from components
logger.debug("Computing affinity labels from components because 'label' wasn't provided in data source.")
affinities_array = affinities_from_components
assert affinities_array.shape == affinities_shape, \
"affinities_array.shape is {actual} but should be {desired}".format(
actual=str(affinities_array.shape), desired=str(affinities_shape))
if 'mask' in original_dataset:
mask_array = get_zero_padded_array_slice(original_dataset['mask'], output_slice)
else:
if components_are_from_dvid:
# infer mask values: 1 if component is nonzero, 0 otherwise
mask_array = np.not_equal(components_array, 0)
logger.debug("No mask provided. Setting to 1 where components != 0.")
else:
# assume no masking
mask_array = np.ones_like(components_array, dtype=np.uint8)
logger.debug("No mask provided. Setting to 1 where outputs exist.")
mask_dilation_steps = original_dataset.get('mask_dilation_steps', 0)
if mask_dilation_steps > 0:
mask_array = ndimage.binary_dilation(mask_array, iterations=mask_dilation_steps)
mask_array = mask_array.astype(np.uint8)
mask_array = mask_array.reshape(mask_shape)
return components_array, affinities_array, mask_array
def get_outputs(original_dataset, output_slice):
output_shape = tuple(
[slice_.stop - slice_.start for slice_ in output_slice])
n_spatial_dimensions = len(output_slice)
components_shape = (1, ) + output_shape
mask_shape = (1, ) + output_shape
affinities_shape = (n_spatial_dimensions, ) + output_shape
component_slices = [
slice(0, l) for l in original_dataset['components'].shape
]
component_slices[-n_spatial_dimensions:] = output_slice
logger.debug("component_slices: {}".format(component_slices))
components_array = get_zero_padded_array_slice(
original_dataset['components'], component_slices)
source_class = type(original_dataset['components'])
components_are_from_dvid = source_class in dvid_classes
exclude_strings = original_dataset.get('body_names_to_exclude', [])
if exclude_strings and components_are_from_dvid:
dvid_uuid = original_dataset['components'].uuid
components_to_keep = get_good_components(dvid_uuid, exclude_strings)
logger.debug("components before: {}".format(
list(np.unique(components_array))))
components_array = replace_array_except_whitelist(
components_array, 0, components_to_keep)
logger.debug("components after: {}".format(
list(np.unique(components_array))))
minimum_component_size = original_dataset.get('minimum_component_size', 0)
if minimum_component_size > 0:
components_array = replace_infrequent_values(components_array,
minimum_component_size, 0)
component_erosion_steps = original_dataset.get('component_erosion_steps',
0)
if component_erosion_steps > 0:
components_array = erode_value_blobs(components_array,
steps=component_erosion_steps,
values_to_ignore=(0, ))
components_for_malis = components_array.reshape(output_shape)
affinities_from_components = malis.seg_to_affgraph(
components_for_malis, original_dataset['nhood'])
components_array, _ = malis.connected_components_affgraph(
affinities_from_components, original_dataset['nhood'])
components_array = shift_up_component_values(components_array)
components_array = components_array.reshape(components_shape)
if 'label' in original_dataset:
label_shape = original_dataset['label'].shape
label_slices = [slice(0, l) for l in label_shape]
label_slices[-n_spatial_dimensions:] = output_slice
affinities_array = get_zero_padded_array_slice(
original_dataset['label'], label_slices)
else:
# compute affinities from components
logger.debug(
"Computing affinity labels from components because 'label' wasn't provided in data source."
)
affinities_array = affinities_from_components
assert affinities_array.shape == affinities_shape, \
"affinities_array.shape is {actual} but should be {desired}".format(
actual=str(affinities_array.shape), desired=str(affinities_shape))
if 'mask' in original_dataset:
mask_array = get_zero_padded_array_slice(original_dataset['mask'],
output_slice)
else:
if components_are_from_dvid:
# infer mask values: 1 if component is nonzero, 0 otherwise
mask_array = np.not_equal(components_array, 0)
logger.debug(
"No mask provided. Setting to 1 where components != 0.")
else:
# assume no masking
mask_array = np.ones_like(components_array, dtype=np.uint8)
logger.debug("No mask provided. Setting to 1 where outputs exist.")
mask_dilation_steps = original_dataset.get('mask_dilation_steps', 0)
if mask_dilation_steps > 0:
mask_array = ndimage.binary_dilation(mask_array,
iterations=mask_dilation_steps)
mask_array = mask_array.astype(np.uint8)
mask_array = mask_array.reshape(mask_shape)
return components_array, affinities_array, mask_array
