def generate_skels(prefix, output_file_path, segmentation_file_path, resolution): if not os.path.exists("../meta"): os.mkdir("../meta") meta_path = "../meta/" + prefix + ".meta" segmentation = dataIO.ReadH5File(segmentation_file_path, "main").astype(np.int64) grid = [len(segmentation), len(segmentation[0]), len(segmentation[0][0])] with open(meta_path, "w") as meta_file: meta_file.write("# resolution in nm\n") meta_file.write(f"{resolution[0]}x{resolution[1]}x{resolution[2]}\n") meta_file.write("# segmentation filename\n") meta_file.write(f"{segmentation_file_path} main\n") meta_file.write("# grid size\n") meta_file.write(f"{grid[0]}x{grid[1]}x{grid[2]}\n") seg2seg.DownsampleMapping(prefix, segmentation) generate_skeletons.TopologicalThinning(prefix, segmentation) generate_skeletons.FindEndpointVectors(prefix) skeletons = ReadSkeletons(prefix) with open(output_file_path, 'wb') as fp: pickle.dump(skeletons, fp)
def CollectExamples(prefix, width, radius, subset): # get the parent directory with all of the featuers parent_directory = 'features/biological/edges-{}nm-{}x{}x{}'.format( radius, width[IB_Z + 1], width[IB_Y + 1], width[IB_X + 1]) positive_filename = '{}/{}/positives/{}-examples.h5'.format( parent_directory, subset, prefix) if os.path.exists(positive_filename): positive_examples = dataIO.ReadH5File(positive_filename, 'main') else: positive_examples = np.zeros( (0, width[IB_Z + 1], width[IB_Y + 1], width[IB_X + 1])) negative_filename = '{}/{}/negatives/{}-examples.h5'.format( parent_directory, subset, prefix) if os.path.exists(negative_filename): negative_examples = dataIO.ReadH5File(negative_filename, 'main') else: negative_examples = np.zeros( (0, width[IB_Z + 1], width[IB_Y + 1], width[IB_X + 1])) unknowns_filename = '{}/{}/unknowns/{}-examples.h5'.format( parent_directory, subset, prefix) if os.path.exists(unknowns_filename): unknowns_example = dataIO.ReadH5File(unknowns_filename, 'main') else: unknowns_example = np.zeros( (0, width[IB_Z + 1], width[IB_Y + 1], width[IB_X + 1])) # concatenate all of the examples together examples = np.concatenate( (positive_examples, negative_examples, unknowns_example), axis=0) # add in information needed for forward inference [regions masked out for training and validation] forward_positive_filename = '{}/forward/positives/{}-examples.h5'.format( parent_directory, prefix) if os.path.exists(forward_positive_filename): forward_positive_examples = dataIO.ReadH5File( forward_positive_filename, 'main') examples = np.concatenate((examples, forward_positive_examples), axis=0) forward_negative_filename = '{}/forward/negatives/{}-examples.h5'.format( parent_directory, prefix) if os.path.exists(forward_negative_filename): forward_negative_examples = dataIO.ReadH5File( forward_negative_filename, 'main') examples = np.concatenate((examples, forward_negative_examples), axis=0) forward_unknowns_filename = '{}/forward/unknowns/{}-examples.h5'.format( parent_directory, prefix) if os.path.exists(forward_unknowns_filename): forward_unknowns_examples = dataIO.ReadH5File( forward_unknowns_filename, 'main') examples = np.concatenate((examples, forward_unknowns_examples), axis=0) return examples, positive_examples.shape[0], negative_examples.shape[0]
def EdgeGenerator(parameters, width, radius, subset): # get the directories corresponding to this radius and subset positive_directory = 'features/biological/edges-{}nm-{}x{}x{}/{}/positives'.format( radius, width[IB_Z + 1], width[IB_Y + 1], width[IB_X + 1], subset) negative_directory = 'features/biological/edges-{}nm-{}x{}x{}/{}/negatives'.format( radius, width[IB_Z + 1], width[IB_Y + 1], width[IB_X + 1], subset) # get all the positive candidate filenames positive_filenames = os.listdir(positive_directory) positive_candidates = [] for positive_filename in positive_filenames: if not positive_filename[-3:] == '.h5': continue positive_candidates.append( dataIO.ReadH5File( '{}/{}'.format(positive_directory, positive_filename), 'main')) positive_candidates = np.concatenate(positive_candidates, axis=0) # get all the negative candidate filenames negative_filenames = os.listdir(negative_directory) negative_candidates = [] for negative_filename in negative_filenames: if not negative_filename[-3:] == '.h5': continue negative_candidates.append( dataIO.ReadH5File( '{}/{}'.format(negative_directory, negative_filename), 'main')) negative_candidates = np.concatenate(negative_candidates, axis=0) # create easy access to the numbers of candidates npositive_candidates = positive_candidates.shape[0] nnegative_candidates = negative_candidates.shape[0] batch_size = parameters['batch_size'] examples = np.zeros((batch_size, width[0], width[IB_Z + 1], width[IB_Y + 1], width[IB_X + 1]), dtype=np.float32) labels = np.zeros(batch_size, dtype=np.float32) positive_order = range(npositive_candidates) negative_order = range(nnegative_candidates) random.shuffle(positive_order) random.shuffle(negative_order) positive_index = 0 negative_index = 0 while True: for iv in range(batch_size // 2): positive_candidate = positive_candidates[ positive_order[positive_index]] negative_candidate = negative_candidates[ negative_order[negative_index]] examples[2 * iv, :, :, :, :] = AugmentFeature( positive_candidate, width) labels[2 * iv] = True examples[2 * iv + 1, :, :, :, :] = AugmentFeature( negative_candidate, width) labels[2 * iv + 1] = False positive_index += 1 if positive_index == npositive_candidates: random.shuffle(positive_order) positive_index = 0 negative_index += 1 if negative_index == nnegative_candidates: random.shuffle(negative_order) negative_index = 0 yield (examples, labels)