def __call__(self, prefix: str): logging.info(f'aggregate skeletons with prefix of {prefix}') id2filenames = defaultdict(list) for filename in self.fragments_storage.list_files(prefix=prefix): filename = os.path.basename(filename) # `match` implies the beginning (^). `search` matches whole string matches = re.search(r'(\d+):', filename) if not matches: continue # skeleton ID skl_id = int(matches.group(0)[:-1]) id2filenames[skl_id].append(filename) for skl_id, filenames in id2filenames.items(): logging.info(f'skeleton id: {skl_id}') frags = self.fragments_storage.get(filenames) frags = [ PrecomputedSkeleton.from_precomputed(x['content']) for x in frags ] skel = PrecomputedSkeleton.simple_merge(frags).consolidate() skel = kimimaro.postprocess(skel, dust_threshold=1000, tick_threshold=3500) self.output_storage.put( file_path=str(skl_id), content=skel.to_precomputed(), ) # the last few hundred files will not be uploaded without sleeping! sleep(0.01)
def process_skeletons(self, unfused_skeletons, in_place=False): skeletons = {} if in_place: skeletons = unfused_skeletons for label in tqdm(unfused_skeletons.keys(), desc="Postprocessing", disable=(not self.progress)): skels = unfused_skeletons[label] skel = PrecomputedSkeleton.simple_merge(skels) skel.id = label skel.extra_attributes = [ attr for attr in skel.extra_attributes \ if attr['data_type'] == 'float32' ] if self.max_cable_length is not None and skel.cable_length( ) > self.max_cable_length: skeletons[label] = skel.to_precomputed() else: skeletons[label] = kimimaro.postprocess( skel, dust_threshold=self.dust_threshold, # voxels tick_threshold=self.tick_threshold, # nm ).to_precomputed() return skeletons
def execute(self): corgie_logger.info(f"Merging skeletons at {self.dst_path}") fragment_filenames = self.cf.list(prefix=self.prefix, flat=True) skeleton_files = self.cf.get(fragment_filenames) skeletons = defaultdict(list) for skeleton_file in skeleton_files: try: colon_index = skeleton_file["path"].index(":") except ValueError: # File is full skeleton, not fragment continue seg_id = skeleton_file["path"][0:colon_index] skeleton_fragment = pickle.loads(skeleton_file["content"]) if not skeleton_fragment.empty(): skeletons[seg_id].append(skeleton_fragment) for seg_id, skeleton_fragments in skeletons.items(): skeleton = PrecomputedSkeleton.simple_merge( skeleton_fragments).consolidate() skeleton = kimimaro.postprocess(skeleton, self.dust_threshold, self.tick_threshold) skeleton.id = int(seg_id) self.cf.put(path=seg_id, content=skeleton.to_precomputed(), compress="gzip") corgie_logger.info(f"Finished skeleton {seg_id}")
def process_skeletons(self, locations, cv): skeletons = {} for label, locs in locations.items(): skel = PrecomputedSkeleton.simple_merge( self.get_unfused(label, locs, cv) ) skel.id = label skel.extra_attributes = [ attr for attr in skel.extra_attributes \ if attr['data_type'] == 'float32' ] skeletons[label] = kimimaro.postprocess( skel, dust_threshold=self.dust_threshold, # voxels tick_threshold=self.tick_threshold, # nm ).to_precomputed() return skeletons
def execute(self): self.vol = CloudVolume(self.cloudpath, mip=self.mip, cdn_cache=False) fragment_filenames = self.get_filenames() skels = self.get_skeletons_by_segid(fragment_filenames) skeletons = [] for segid, frags in skels.items(): skeleton = self.fuse_skeletons(frags) skeleton = kimimaro.postprocess(skeleton, self.dust_threshold, self.tick_threshold) skeleton.id = segid skeletons.append(skeleton) self.vol.skeleton.upload(skeletons) if self.delete_fragments: with Storage(self.cloudpath, progress=True) as stor: stor.delete_files(fragment_filenames)
def process_skeletons(self, locations, cv): filenames = set(itertools.chain(*locations.values())) labels = set(locations.keys()) unfused_skeletons = self.get_unfused(labels, filenames, cv) skeletons = {} for label, skels in tqdm(unfused_skeletons.items(), desc="Postprocessing", disable=(not self.progress)): skel = PrecomputedSkeleton.simple_merge(skels) skel.id = label skel.extra_attributes = [ attr for attr in skel.extra_attributes \ if attr['data_type'] == 'float32' ] skeletons[label] = kimimaro.postprocess( skel, dust_threshold=self.dust_threshold, # voxels tick_threshold=self.tick_threshold, # nm ).to_precomputed() return skeletons
def execute(self): self.vol = CloudVolume(self.cloudpath, cdn_cache=False) self.vol.mip = self.vol.skeleton.meta.mip fragment_filenames = self.get_filenames() skels = self.get_skeletons_by_segid(fragment_filenames) skeletons = [] for segid, frags in skels.items(): skeleton = self.fuse_skeletons(frags) if self.max_cable_length is None or skel.cable_length( ) <= self.max_cable_length: skeleton = kimimaro.postprocess(skeleton, self.dust_threshold, self.tick_threshold) skeleton.id = segid skeletons.append(skeleton) self.vol.skeleton.upload(skeletons) if self.delete_fragments: cf = CloudFiles(self.cloudpath, progress=True) cf.delete(fragment_filenames)
def complex_merge(skel): return kimimaro.postprocess( skel, dust_threshold=1000, # voxels tick_threshold=1300, # nm )
for label in all_ids: print("----------------------------------------------------") print("Combining skeletons for label " + str(label)) all_skels = [] for bz in bz_range: for by in by_range: for bx in bx_range: fname = output_folder + "skel_out-label{:09d}-{:04d}z-{:04d}y-{:04d}x.swc".format(label,bz,by,bx) if os.path.exists(fname): skel_read = readSkelFromFile(fname, bz, by, bx, bsize, anisotropy) print("Adding part from block " + str((bz,by,bx)) + " with " + str(len(skel_read.vertices)) + " vertices") all_skels.insert(0, skel_read) skel_joined = kimimaro.join_close_components(all_skels, radius=1500) # 1500 units threshold print("Skeleton joined with " + str(len(skel_joined.vertices)) + " vertices") skel_final = kimimaro.postprocess(skel_joined, dust_threshold=1000, tick_threshold=3500) print("Skeleton final with " + str(len(skel_final.vertices)) + " vertices") writeFinalSkeletonToFile(skel_final, label, output_folder) try: print("xmin, xmax: " + str(np.min(skel_final.vertices[:,0])) + ", " + str(np.max(skel_final.vertices[:,0]))) print("ymin, ymax: " + str(np.min(skel_final.vertices[:,1])) + ", " + str(np.max(skel_final.vertices[:,1]))) print("zmin, zmax: " + str(np.min(skel_final.vertices[:,2])) + ", " + str(np.max(skel_final.vertices[:,2]))) except: print("not able to print") print("total time combine: " + str(time.time()-start_time_combine)) print("total time thinning: " + str(total_time_thinning))
def ConnectSkeletons(data): for label in range(0, data.NLabels()): # print("--------------------- \n processing label {}".format(label)) # start timing statistics total_time = time.time() read_time = time.time() # list to store skeletons of different blocks all_skels = [] # iterate over all blocks and read in skeletons for respective label for iz in range(data.StartZ(), data.EndZ()): for iy in range(data.StartY(), data.EndY()): for ix in range(data.StartX(), data.EndX()): # get the location for the temporary directory tmp_directory = data.TempBlockDirectory(iz, iy, ix) fname = tmp_directory + "/skel_out-label{:09d}-{:04d}z-{:04d}y-{:04d}x.swc".format( label, iz, iy, ix) if os.path.exists(fname): skel_read = readSkelFromFile(fname, iz, iy, ix, data.BlockSize(), data.Resolution()) all_skels.insert(0, skel_read) # print("Adding part from block " + str((iz,iy,ix)) + " with " + str(len(skel_read.vertices)) + " vertices") read_time = time.time() - read_time # if label not present, skip if len(all_skels) == 0: continue # output directory for final skeletons out_dir = data.SkeletonOutputDirectory() # join skeleton components to one skeleton join_time = time.time() skel_joined = kimimaro.join_close_components( all_skels, radius=1500) # 1500 units threshold join_time = time.time() - join_time # writeSkeletonToFile(skel_joined, label, out_dir, "joined") # postprocess and connect skeleton parts postprocess_time = time.time() skel_final = kimimaro.postprocess(skel_joined, dust_threshold=1000, tick_threshold=0) postprocess_time = time.time() - postprocess_time # check if it actually contains vertices if skel_final.vertices.shape[1] == 0: continue # write final skeleton to file write_time = time.time() writeSkeletonToFile(skel_final, label, out_dir, "final") write_time = time.time() - write_time total_time = time.time() - total_time print('Read Time: {:0.2f} seconds.'.format(read_time)) print('Join Time: {:0.2f} seconds.'.format(join_time)) print('Postprocess Time: {:0.2f} seconds.'.format(postprocess_time)) print('Write Time: {:0.2f} seconds.'.format(write_time)) print('Total Time: {:0.2f} seconds.'.format(total_time)) # output timing statistics timing_directory = '{}/skeletons-combine'.format( data.TimingDirectory()) if not os.path.exists(timing_directory): os.makedirs(timing_directory, exist_ok=True) timing_filename = '{}/{:016d}.txt'.format(timing_directory, label) with open(timing_filename, 'w') as fd: fd.write('Read Time: {:0.2f} seconds.\n'.format(read_time)) fd.write('Join Time: {:0.2f} seconds.\n'.format(join_time)) fd.write('Postprocess Time: {:0.2f} seconds.\n'.format( postprocess_time)) fd.write('Write Time: {:0.2f} seconds.\n'.format(write_time)) fd.write('Total Time: {:0.2f} seconds.\n'.format(total_time))