def get_datasets(dataset, origin): # output_shape = (40, 30, 80) # input_shape = (100, 110, 120) output_shape = (2, 3, 4) input_shape = tuple([x + 2 for x in output_shape]) borders = tuple([(in_ - out_) / 2 for (in_, out_) in zip(input_shape, output_shape)]) input_slices = tuple( [slice(x, x + l) for x, l in zip(origin, input_shape)]) output_slices = tuple([ slice(x + b, x + b + l) for x, b, l in zip(origin, borders, output_shape) ]) expected_dataset = dict() data_slices = [slice(0, l) for l in dataset['data'].shape] data_slices[-3:] = input_slices data_slices = tuple(data_slices) expected_data_array = np.array(dataset['data'][data_slices], dtype=np.float32) expected_data_array = expected_data_array.reshape((1, ) + input_shape) expected_data_array /= (2.0**8) expected_dataset['data'] = expected_data_array components_slices = [slice(0, l) for l in dataset['components'].shape] components_slices[-3:] = output_slices components_slices = tuple(components_slices) expected_components_array = np.array( dataset['components'][components_slices]).reshape((1, ) + output_shape) if type(dataset['components']) is DVIDDataInstance: print("Is DVIDDataInstance...") print("uniques before:", np.unique(expected_components_array)) dvid_uuid = dataset['components'].uuid body_names_to_exclude = dataset.get('body_names_to_exclude') good_bodies = get_good_components(dvid_uuid, body_names_to_exclude) expected_components_array = \ replace_array_except_whitelist(expected_components_array, 0, good_bodies) print("uniques after:", np.unique(expected_components_array)) expected_dataset['components'] = expected_components_array components_for_affinity_generation = expected_components_array.reshape( output_shape) expected_label = malis.seg_to_affgraph( components_for_affinity_generation, malis.mknhood3d()) expected_dataset['label'] = expected_label if type(dataset['components']) is DVIDDataInstance: expected_mask = np.array(expected_components_array > 0).astype( np.uint8) else: expected_mask = np.ones(shape=(1, ) + output_shape, dtype=np.uint8) expected_dataset['mask'] = expected_mask numpy_dataset = get_numpy_dataset(dataset, input_slices, output_slices, True) return numpy_dataset, expected_dataset
def get_datasets(dataset, origin): # output_shape = (40, 30, 80) # input_shape = (100, 110, 120) output_shape = (2,3,4) input_shape = tuple([x + 2 for x in output_shape]) borders = tuple([(in_ - out_) / 2 for (in_, out_) in zip(input_shape, output_shape)]) input_slices = tuple([slice(x, x + l) for x, l in zip(origin, input_shape)]) output_slices = tuple([slice(x + b, x + b + l) for x, b, l in zip(origin, borders, output_shape)]) expected_dataset = dict() data_slices = [slice(0, l) for l in dataset['data'].shape] data_slices[-3:] = input_slices data_slices = tuple(data_slices) expected_data_array = np.array(dataset['data'][data_slices], dtype=np.float32) expected_data_array = expected_data_array.reshape((1,) + input_shape) expected_data_array /= (2.0 ** 8) expected_dataset['data'] = expected_data_array components_slices = [slice(0, l) for l in dataset['components'].shape] components_slices[-3:] = output_slices components_slices = tuple(components_slices) expected_components_array = np.array(dataset['components'][components_slices]).reshape((1,) + output_shape) if type(dataset['components']) is DVIDDataInstance: print("Is DVIDDataInstance...") print("uniques before:", np.unique(expected_components_array)) dvid_uuid = dataset['components'].uuid body_names_to_exclude = dataset.get('body_names_to_exclude') good_bodies = get_good_components(dvid_uuid, body_names_to_exclude) expected_components_array = \ replace_array_except_whitelist(expected_components_array, 0, good_bodies) print("uniques after:", np.unique(expected_components_array)) expected_dataset['components'] = expected_components_array components_for_affinity_generation = expected_components_array.reshape(output_shape) expected_label = malis.seg_to_affgraph(components_for_affinity_generation, malis.mknhood3d()) expected_dataset['label'] = expected_label if type(dataset['components']) is DVIDDataInstance: expected_mask = np.array(expected_components_array > 0).astype(np.uint8) else: expected_mask = np.ones(shape=(1,) + output_shape, dtype=np.uint8) expected_dataset['mask'] = expected_mask numpy_dataset = get_numpy_dataset(dataset, input_slices, output_slices, True) return numpy_dataset, expected_dataset
def get_outputs(original_dataset, output_slice): output_shape = tuple([slice_.stop - slice_.start for slice_ in output_slice]) n_spatial_dimensions = len(output_slice) components_shape = (1,) + output_shape mask_shape = (1,) + output_shape affinities_shape = (n_spatial_dimensions,) + output_shape component_slices = [slice(0, l) for l in original_dataset['components'].shape] component_slices[-n_spatial_dimensions:] = output_slice logger.debug("component_slices: {}".format(component_slices)) components_array = get_zero_padded_array_slice(original_dataset['components'], component_slices) source_class = type(original_dataset['components']) components_are_from_dvid = source_class in dvid_classes exclude_strings = original_dataset.get('body_names_to_exclude', []) if exclude_strings and components_are_from_dvid: dvid_uuid = original_dataset['components'].uuid components_to_keep = get_good_components(dvid_uuid, exclude_strings) logger.debug("components before: {}".format(list(np.unique(components_array)))) components_array = replace_array_except_whitelist(components_array, 0, components_to_keep) logger.debug("components after: {}".format(list(np.unique(components_array)))) minimum_component_size = original_dataset.get('minimum_component_size', 0) if minimum_component_size > 0: components_array = replace_infrequent_values(components_array, minimum_component_size, 0) component_erosion_steps = original_dataset.get('component_erosion_steps', 0) if component_erosion_steps > 0: components_array = erode_value_blobs( components_array, steps=component_erosion_steps, values_to_ignore=(0,)) components_for_malis = components_array.reshape(output_shape) affinities_from_components = malis.seg_to_affgraph( components_for_malis, original_dataset['nhood']) components_array, _ = malis.connected_components_affgraph( affinities_from_components, original_dataset['nhood']) components_array = shift_up_component_values(components_array) components_array = components_array.reshape(components_shape) if 'label' in original_dataset: label_shape = original_dataset['label'].shape label_slices = [slice(0, l) for l in label_shape] label_slices[-n_spatial_dimensions:] = output_slice affinities_array = get_zero_padded_array_slice(original_dataset['label'], label_slices) else: # compute affinities from components logger.debug("Computing affinity labels from components because 'label' wasn't provided in data source.") affinities_array = affinities_from_components assert affinities_array.shape == affinities_shape, \ "affinities_array.shape is {actual} but should be {desired}".format( actual=str(affinities_array.shape), desired=str(affinities_shape)) if 'mask' in original_dataset: mask_array = get_zero_padded_array_slice(original_dataset['mask'], output_slice) else: if components_are_from_dvid: # infer mask values: 1 if component is nonzero, 0 otherwise mask_array = np.not_equal(components_array, 0) logger.debug("No mask provided. Setting to 1 where components != 0.") else: # assume no masking mask_array = np.ones_like(components_array, dtype=np.uint8) logger.debug("No mask provided. Setting to 1 where outputs exist.") mask_dilation_steps = original_dataset.get('mask_dilation_steps', 0) if mask_dilation_steps > 0: mask_array = ndimage.binary_dilation(mask_array, iterations=mask_dilation_steps) mask_array = mask_array.astype(np.uint8) mask_array = mask_array.reshape(mask_shape) return components_array, affinities_array, mask_array
def get_outputs(original_dataset, output_slice): output_shape = tuple( [slice_.stop - slice_.start for slice_ in output_slice]) n_spatial_dimensions = len(output_slice) components_shape = (1, ) + output_shape mask_shape = (1, ) + output_shape affinities_shape = (n_spatial_dimensions, ) + output_shape component_slices = [ slice(0, l) for l in original_dataset['components'].shape ] component_slices[-n_spatial_dimensions:] = output_slice logger.debug("component_slices: {}".format(component_slices)) components_array = get_zero_padded_array_slice( original_dataset['components'], component_slices) source_class = type(original_dataset['components']) components_are_from_dvid = source_class in dvid_classes exclude_strings = original_dataset.get('body_names_to_exclude', []) if exclude_strings and components_are_from_dvid: dvid_uuid = original_dataset['components'].uuid components_to_keep = get_good_components(dvid_uuid, exclude_strings) logger.debug("components before: {}".format( list(np.unique(components_array)))) components_array = replace_array_except_whitelist( components_array, 0, components_to_keep) logger.debug("components after: {}".format( list(np.unique(components_array)))) minimum_component_size = original_dataset.get('minimum_component_size', 0) if minimum_component_size > 0: components_array = replace_infrequent_values(components_array, minimum_component_size, 0) component_erosion_steps = original_dataset.get('component_erosion_steps', 0) if component_erosion_steps > 0: components_array = erode_value_blobs(components_array, steps=component_erosion_steps, values_to_ignore=(0, )) components_for_malis = components_array.reshape(output_shape) affinities_from_components = malis.seg_to_affgraph( components_for_malis, original_dataset['nhood']) components_array, _ = malis.connected_components_affgraph( affinities_from_components, original_dataset['nhood']) components_array = shift_up_component_values(components_array) components_array = components_array.reshape(components_shape) if 'label' in original_dataset: label_shape = original_dataset['label'].shape label_slices = [slice(0, l) for l in label_shape] label_slices[-n_spatial_dimensions:] = output_slice affinities_array = get_zero_padded_array_slice( original_dataset['label'], label_slices) else: # compute affinities from components logger.debug( "Computing affinity labels from components because 'label' wasn't provided in data source." ) affinities_array = affinities_from_components assert affinities_array.shape == affinities_shape, \ "affinities_array.shape is {actual} but should be {desired}".format( actual=str(affinities_array.shape), desired=str(affinities_shape)) if 'mask' in original_dataset: mask_array = get_zero_padded_array_slice(original_dataset['mask'], output_slice) else: if components_are_from_dvid: # infer mask values: 1 if component is nonzero, 0 otherwise mask_array = np.not_equal(components_array, 0) logger.debug( "No mask provided. Setting to 1 where components != 0.") else: # assume no masking mask_array = np.ones_like(components_array, dtype=np.uint8) logger.debug("No mask provided. Setting to 1 where outputs exist.") mask_dilation_steps = original_dataset.get('mask_dilation_steps', 0) if mask_dilation_steps > 0: mask_array = ndimage.binary_dilation(mask_array, iterations=mask_dilation_steps) mask_array = mask_array.astype(np.uint8) mask_array = mask_array.reshape(mask_shape) return components_array, affinities_array, mask_array