def generate_subcell_kd_from_proba(co, chunk_size=None, transf_func_kd_overlay=None, load_cellorganelles_from_kd_overlaycubes=False, cube_of_interest_bb=None, log=None, **kwargs): """ Generate KnossosDatasets for given subcellular structure key (e.g. 'mi'). The required initial data format is a chunkdataset located at ``"{}/chunkdatasets/{}/".format(global_params.config.working_dir, co)``. Resulting KD will be stored at ``"{}/knossosdatasets/{}_seg/".format(global_params.config.working_dir, co)``. See :func:`~syconn.extraction.object_extraction_wrapper.from_probabilities_to_kd` for details of the conversion process from the initial probability map to the SV segmentation. Default: thresholding and connected components, thresholds are set via the `config.ini` file, check ``syconn.global_params.config.entries["Probathresholds"]`` of an initialized :calss:`~syconn.handler.config.DynConfig` object. Parameters ---------- co : str chunk_size : Tuple transf_func_kd_overlay : callable load_cellorganelles_from_kd_overlaycubes : bool cube_of_interest_bb : Tuple[Tuple[int]] or np.ndarray log : logger """ if chunk_size is None: chunk_size = [512, 512, 512] if log is None: log = log_extraction kd = basics.kd_factory(global_params.config.kd_seg_path) if cube_of_interest_bb is None: cube_of_interest_bb = [np.zeros(3, dtype=np.int), kd.boundary] size = cube_of_interest_bb[1] - cube_of_interest_bb[0] + 1 offset = cube_of_interest_bb[0] cd_dir = "{}/chunkdatasets/{}/".format(global_params.config.working_dir, co) cd = chunky.ChunkDataset() cd.initialize(kd, kd.boundary, chunk_size, cd_dir, box_coords=[0, 0, 0], fit_box_size=True, list_of_coords=[]) log.info('Started object extraction of cellular organelles "{}" from ' '{} chunks.'.format(co, len(cd.chunk_dict))) prob_kd_path_dict = {co: getattr(global_params.config, 'kd_{}_path'.format(co))} # This creates a SegmentationDataset of type 'co' prob_thresh = global_params.config.entries["Probathresholds"][co] # get probability threshold # `from_probabilities_to_objects` will export a KD at `path`, remove if already existing path = global_params.config.kd_organelle_seg_paths[co] if os.path.isdir(path): log.debug('Found existing KD at {}. Removing it now.'.format(path)) shutil.rmtree(path) target_kd = knossosdataset.KnossosDataset() scale = np.array(global_params.config.entries["Dataset"]["scaling"], dtype=np.float32) target_kd.initialize_without_conf(path, kd.boundary, scale, kd.experiment_name, mags=[1, ]) target_kd = knossosdataset.KnossosDataset() target_kd.initialize_from_knossos_path(path) from_probabilities_to_kd(cd, co, # membrane_kd_path=global_params.config.kd_barrier_path, # TODO: currently does not exist prob_kd_path_dict=prob_kd_path_dict, thresholds=[prob_thresh], hdf5names=[co], n_max_co_processes=None, target_kd=target_kd, debug=False, size=size, offset=offset, load_from_kd_overlaycubes=load_cellorganelles_from_kd_overlaycubes, transf_func_kd_overlay=transf_func_kd_overlay[co], log=log, **kwargs)
def _extract_synapse_type_thread(args): paths = args[0] obj_version = args[1] working_dir = args[2] kd_asym_path = args[3] kd_sym_path = args[4] trafo_dict_path = args[5] if trafo_dict_path is not None: with open(trafo_dict_path, "rb") as f: trafo_dict = pkl.load(f) else: trafo_dict = None kd_asym = knossosdataset.KnossosDataset() kd_asym.initialize_from_knossos_path(kd_asym_path) kd_sym = knossosdataset.KnossosDataset() kd_sym.initialize_from_knossos_path(kd_sym_path) seg_dataset = segmentation.SegmentationDataset("syn_ssv", version=obj_version, working_dir=working_dir) for p in paths: this_attr_dc = AttributeDict(p + "/attr_dict.pkl", read_only=False, disable_locking=True) for so_id in this_attr_dc.keys(): so = seg_dataset.get_segmentation_object(so_id) so.attr_dict = this_attr_dc[so_id] so.load_voxel_list() vxl = so.voxel_list if trafo_dict is not None: vxl -= trafo_dict[so_id] vxl = vxl[:, [1, 0, 2]] # TODO: remvoe try-except try: asym_prop = np.mean(kd_asym.from_raw_cubes_to_list(vxl)) sym_prop = np.mean(kd_sym.from_raw_cubes_to_list(vxl)) except: log_proc.error("Failed to read raw cubes during synapse type " "extraction.") sym_prop = 0 asym_prop = 0 if sym_prop + asym_prop == 0: sym_ratio = -1 else: sym_ratio = sym_prop / float(asym_prop + sym_prop) so.attr_dict["syn_type_sym_ratio"] = sym_ratio syn_sign = -1 if sym_ratio > global_params.sym_thresh else 1 so.attr_dict["syn_sign"] = syn_sign this_attr_dc[so_id] = so.attr_dict this_attr_dc.push()
def swc_to_mask(f_cube, f_swc, f_output): ''' f_cube: ../mag1/knossos.conf f_swc: ../*.swc f_output: output dir ''' if not os.path.exists(f_output): os.makedirs(f_output) f_cube = os.path.abspath(f_cube) f_output = os.path.abspath(f_output) f_swc = os.path.abspath(f_swc) f_tiff_output = os.path.join(f_output, 'stack') f_mask = os.path.join(f_output, 'output_mask') f_v3d = os.path.join(f_output, 'output_mask.v3draw') kd = knossosdataset.KnossosDataset() kd.initialize_from_knossos_path(f_cube) print(kd._experiment_name) print(kd.boundary) x, y, z = kd.boundary #z,x,y = kd.boundary command = r'vaa3d -x swc_to_maskimage_sphere -f swc_to_maskimage -i {} -p {} {} {} -o {}.tiff'.format( f_swc, str(x), str(y), str(z), f_mask) print(command) #fiji_command = r'run("Convert...", "input={} output={} output_format=TIFF interpolation=Bilinear scale=1"); '.format(f_output, f_output) # print(fiji_command) #f_fiji = os.path.join(f_output, 'tmp.ijm') f_log = os.path.join(f_output, 'log.txt') # with open(f_fiji, 'w') as f: # f.write(fiji_command+'\n') #run_fiji_command = r'fiji -macro {}'.format(f_fiji) subprocess.call(command, shell=True)
def labels_to_knossos(f_cube, f_anno, f_ext, mode): '''converts segmentated mask back into knossos segmentation channel''' kd = knossosdataset.KnossosDataset() kd.initialize_from_knossos_path(f_cube) #raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0]) print(kd._experiment_name) print(kd.boundary) if mode == 'to': # print(overlay.shape) # print(np.unique(overlay[:])) new_overlay = omni_read(f_ext) new_overlay = np.rollaxis(new_overlay, 2, 0) new_overlay = np.rollaxis(new_overlay, 2, 1) print(new_overlay.shape) print(np.unique(new_overlay)) new_kzip = kd.from_matrix_to_cubes(offset=( 0, 0, 0), data=new_overlay, kzip_path=os.path.join(os.path.dirname(f_anno), 'new')) elif mode == 'from': overlay = kd.from_kzip_to_matrix(path=f_anno, size=kd.boundary, offset=[ 0, 0, 0], mag=1, verbose=True, alt_exp_name_kzip_path_mode=True) print(np.unique(overlay[:])) rolled_overlay = np.rollaxis(overlay, 2, 0) rolled_overlay = np.rollaxis(rolled_overlay, 2, 1) f_out = os.path.dirname(f_ext) if not os.path.exists(f_out): os.makedirs(f_out) tifffile.imsave(f_ext, np.uint32(rolled_overlay))
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--cube', default= '/mnt/md0/XRay/Knossos_measurements/WT_colon_380/knossos_cube/mag1/knossos.conf' ) parser.add_argument( '--anno', default= '/mnt/md0/XRay/Knossos_measurements/WT_colon_380/annotation/muscle_skeleton_annotation-20180113T1805.139.k.zip' ) args = parser.parse_args() print(args.cube, args.anno) root_dir = '.' f_knossos = args.cube f_overlay = args.anno f_out = os.path.join(root_dir, 'output') if not os.path.exists(f_out): os.makedirs(f_out) kd = knossosdataset.KnossosDataset() kd.initialize_from_knossos_path(f_knossos) #raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0]) print(kd._experiment_name) print(kd.boundary) #overlay= kd.from_kzip_to_matrix(path=f_overlay,size=kd.boundary, offset=[0, 0, 0], mag=1, verbose=True, alt_exp_name_kzip_path_mode=False) sk = skeleton.Skeleton() sk.fromNml(f_overlay) result_dict = sweep_sk(sk) result_dict = get_stat(sk, result_dict) convert_to_csv(result_dict, f_out) sys.exit()
def _export_sd_to_knossosdataset_thread(args): block_loc = args[0] so_ids = args[1] obj_type = args[2] version = args[3] working_dir = args[4] kd_path = args[5] block_edge_length = args[6] block_size = np.array([block_edge_length] * 3, dtype=np.int) kd = knossosdataset.KnossosDataset() kd.initialize_from_knossos_path(kd_path) sd = segmentation.SegmentationDataset(obj_type=obj_type, working_dir=working_dir, version=version) overlay_block = np.zeros(block_size, dtype=np.uint64) block_start = (block_loc * block_size).astype(np.int) for so_id in so_ids: so = sd.get_segmentation_object(so_id, False) vx = so.voxel_list - block_start vx = vx[~np.any(vx < 0, axis=1)] vx = vx[~np.any(vx >= block_edge_length, axis=1)] overlay_block[vx[:, 0], vx[:, 1], vx[:, 2]] = so_id kd.from_matrix_to_cubes(block_start, data=overlay_block, overwrite=True, nb_threads=1, verbose=True)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--cube', default='../full_stack_rot_knossos/mag1/knossos.conf') parser.add_argument('--anno', default='../trace/') parser.add_argument('--output', default='.') parser.add_argument('--features', type=str, choices=['both', 'cellbody', 'dendrite', 'mask'], default='both') args = parser.parse_args() f_knossos = args.cube f_overlay = max(glob.iglob(os.path.join(args.anno, '*.k.zip')), key=os.path.getmtime) f_output = args.output if not os.path.exists(f_output): os.makedirs(f_output) f_swc = os.path.join(f_output, 'output.swc') f_center = os.path.join(f_output, 'center.txt') kd = knossosdataset.KnossosDataset() kd.initialize_from_knossos_path(f_knossos) #raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0]) print(kd._experiment_name) print(kd.boundary) #overlay= kd.from_kzip_to_matrix(path=f_overlay,size=kd.boundary, offset=[0, 0, 0], mag=1, verbose=True, alt_exp_name_kzip_path_mode=False) sk = skeleton.Skeleton() sk.fromNml(f_overlay) if args.features == 'both': sk2swc_and_center(sk, f_swc, f_center) elif args.features == 'cellbody': sk2swc_cellbody_and_center(sk, f_swc, f_center) elif args.features == 'dendrite': sk2swc_dendrite_and_center(sk, f_swc, f_center) elif args.features == 'mask': sk.toSWC(f_swc)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--cube', default=None) parser.add_argument('--anno', default=None) parser.add_argument('--mag', type=int, default=1) parser.add_argument('--output', default=None) args = parser.parse_args() f_knossos = args.cube f_overlay = args.anno f_out = args.output # matches = re.match('^.*/mag(?P<mag>[0-9]+)/.*$', args.cube).groupdict() # mag = int(matches['mag']) mag = args.mag kd = knossosdataset.KnossosDataset() kd.initialize_from_knossos_path(f_knossos) #raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0]) print(kd._experiment_name) # sys.exit() print(kd.boundary) overlay = kd.from_kzip_to_matrix( path=f_overlay, # size=kd.boundary//16, size=[1000, 1000, 1000], offset=[4300, 4300, 4300], mag=mag, verbose=True) # alt_exp_name_kzip_path_mode=True) print(overlay.shape) print(np.max(overlay[:])) overlay = np.rollaxis(overlay, 2, 0) overlay = np.rollaxis(overlay, 2, 1) #dxchange.write_tiff(overlay.astype(np.uint32), f_out, overwrite=True) print(overlay.shape, overlay.dtype) print(np.mean(overlay[...]))
def extract_agg_contact_sites(cset, working_dir, filename='cs', hdf5name='cs', n_folders_fs=10000, suffix="", n_max_co_processes=None, n_chunk_jobs=2000, size=None, offset=None, log=None): """ Parameters ---------- cset : working_dir : filename : hdf5name : n_folders_fs : suffix : n_max_co_processes : n_chunk_jobs : size : offset : Returns ------- """ if log is None: log = log_extraction chunky.save_dataset(cset) # init CS segmentation KD kd = kd_factory(global_params.config.kd_seg_path) path = "{}/knossosdatasets/{}_seg/".format( global_params.config.working_dir, filename) if os.path.isdir(path): log.debug('Found existing KD at {}. Removing it now.'.format(path)) shutil.rmtree(path) target_kd = knossosdataset.KnossosDataset() scale = np.array(global_params.config.entries["Dataset"]["scaling"]) target_kd.initialize_without_conf(path, kd.boundary, scale, kd.experiment_name, mags=[ 1, ]) target_kd = knossosdataset.KnossosDataset() target_kd.initialize_from_knossos_path(path) # convert Chunkdataset to KD export_cset_to_kd_batchjob(cset, target_kd, '{}'.format(filename), [hdf5name], offset=offset, size=size, stride=[4 * 128, 4 * 128, 4 * 128], as_raw=False, orig_dtype=np.uint64, unified_labels=False, n_max_co_processes=n_max_co_processes) log.debug( 'Finished conversion of ChunkDataset ({}) into KnossosDataset ({})'. format(cset.path_head_folder, target_kd.knossos_path)) # get CS SD from_ids_to_objects(cset, filename, overlaydataset_path=target_kd.conf_path, n_chunk_jobs=n_chunk_jobs, hdf5names=[hdf5name], n_max_co_processes=n_max_co_processes, workfolder=working_dir, n_folders_fs=n_folders_fs, use_combined_extraction=True, suffix=suffix, size=size, offset=offset, log=log)
def extract_contact_sites(n_max_co_processes: Optional[int] = None, chunk_size: Optional[Tuple[int, int, int]] = None, log: Optional[Logger] = None, max_n_jobs: Optional[int] = None, cube_of_interest_bb: Optional[np.ndarray] = None, n_folders_fs: int = 1000): """ Extracts contact sites and their overlap with `sj` objects and stores them in a :class:`~syconn.reps.segmentation.SegmentationDataset` of type ``cs`` and ``syn`` respectively. If synapse type is available, this information will be stored as the voxel-ratio per class in the attribute dictionary of the ``syn`` objects (keys: ``sym_prop``, ``asym_prop``). Notes: Replaced ``find_contact_sites``, ``extract_agg_contact_sites``, ` `syn_gen_via_cset`` and ``extract_synapse_type``. Args: n_max_co_processes: Number of parallel workers. chunk_size: Sub-cube volume which is processed at a time. log: Logger. max_n_jobs: Maximum number of jobs. cube_of_interest_bb: Sub-volume of the data set which is processed. Default: Entire data set. n_folders_fs: Number of folders used for organizing supervoxel data. """ if extract_cs_syntype is None: msg = '`extract_contact_sites` requires the cythonized method ' \ '`extract_cs_syntype`. Use `find_contact_sites` and others ' \ 'for contact site processing.' log_extraction.error(msg) raise ImportError(msg) kd = kd_factory(global_params.config.kd_seg_path) if cube_of_interest_bb is None: cube_of_interest_bb = [np.zeros(3, dtype=np.int), kd.boundary] if chunk_size is None: chunk_size = (512, 512, 512) size = cube_of_interest_bb[1] - cube_of_interest_bb[0] + 1 offset = cube_of_interest_bb[0] # Initital contact site extraction cd_dir = global_params.config.temp_path + "/chunkdatasets/cs/" # Class that contains a dict of chunks (with coordinates) after initializing it cset = chunky.ChunkDataset() cset.initialize(kd, kd.boundary, chunk_size, cd_dir, box_coords=[0, 0, 0], fit_box_size=True) if max_n_jobs is None: max_n_jobs = global_params.NCORE_TOTAL * 2 if log is None: log = log_extraction if size is not None and offset is not None: chunk_list, _ = \ calculate_chunk_numbers_for_box(cset, offset, size) else: chunk_list = [ii for ii in range(len(cset.chunk_dict))] # shuffle chunklist to get a more balanced work-load rand_ixs = np.arange(len(chunk_list)) np.random.shuffle(rand_ixs) chunk_list = np.array(chunk_list)[rand_ixs] os.makedirs(cset.path_head_folder, exist_ok=True) multi_params = [] # TODO: currently pickles Chunk objects -> job submission might be slow for chunk_k in chunkify(chunk_list, max_n_jobs): multi_params.append([[cset.chunk_dict[k] for k in chunk_k], global_params.config.kd_seg_path]) if not qu.batchjob_enabled(): results = start_multiprocess_imap(_contact_site_extraction_thread, multi_params, debug=False, nb_cpus=n_max_co_processes) else: path_to_out = qu.QSUB_script(multi_params, "contact_site_extraction", n_max_co_processes=n_max_co_processes, log=log) out_files = glob.glob(path_to_out + "/*") results = [] for out_file in out_files: with open(out_file, 'rb') as f: results.append(pkl.load(f)) shutil.rmtree(os.path.abspath(path_to_out + "/../"), ignore_errors=True) # reduce step cs_props = [{}, defaultdict(list), {}] syn_props = [{}, defaultdict(list), {}] tot_sym_cnt = {} tot_asym_cnt = {} for curr_props, curr_syn_props, asym_cnt, sym_cnt in results: merge_prop_dicts([cs_props, curr_props]) merge_prop_dicts([syn_props, curr_syn_props]) merge_type_dicts([tot_asym_cnt, asym_cnt]) merge_type_dicts([tot_sym_cnt, sym_cnt]) log.info('Finished contact site (#objects: {}) and synapse (#objects: {})' ' extraction.'.format(len(cs_props[0]), len(syn_props[0]))) if len(syn_props[0]) == 0: log.critical( 'WARNING: Did not find any synapses during extraction step.') # TODO: extract syn objects! maybe replace sj_0 Segmentation dataset by the overlapping CS<-> # sj objects -> run syn. extraction and sd_generation in parallel and return mi_0, vc_0 and # syn_0 -> use syns as new sjs during rendering! # -> Run CS generation in parallel with mapping to at least get the syn objects before # rendering the neuron views (which need subcellular structures, there one can then use mi, # vc and syn (instead of sj)) dict_paths = [] # dump intermediate results # TODO: size filter here or during write-out? TODO: use config parameter dict_p = "{}/cs_prop_dict.pkl".format(global_params.config.temp_path) with open(dict_p, "wb") as f: pkl.dump(cs_props, f) del cs_props dict_paths.append(dict_p) dict_p = "{}/syn_prop_dict.pkl".format(global_params.config.temp_path) with open(dict_p, "wb") as f: pkl.dump(syn_props, f) del syn_props dict_paths.append(dict_p) # convert counting dicts to store ratio of syn. type voxels dict_p = "{}/cs_sym_cnt.pkl".format(global_params.config.temp_path) with open(dict_p, "wb") as f: pkl.dump(tot_sym_cnt, f) del tot_sym_cnt dict_paths.append(dict_p) dict_p = "{}/cs_asym_cnt.pkl".format(global_params.config.temp_path) with open(dict_p, "wb") as f: pkl.dump(tot_asym_cnt, f) del tot_asym_cnt dict_paths.append(dict_p) # write cs and syn segmentation to KD and SD chunky.save_dataset(cset) kd = kd_factory(global_params.config.kd_seg_path) # convert Chunkdataset to syn and cs KD # TODO: spawn in parallel for obj_type in ['cs', 'syn']: path = "{}/knossosdatasets/{}_seg/".format( global_params.config.working_dir, obj_type) if os.path.isdir(path): log.debug('Found existing KD at {}. Removing it now.'.format(path)) shutil.rmtree(path) target_kd = knossosdataset.KnossosDataset() scale = np.array(global_params.config.entries["Dataset"]["scaling"]) target_kd.initialize_without_conf(path, kd.boundary, scale, kd.experiment_name, mags=[ 1, ]) target_kd = knossosdataset.KnossosDataset() target_kd.initialize_from_knossos_path(path) export_cset_to_kd_batchjob(cset, target_kd, obj_type, [obj_type], offset=offset, size=size, stride=chunk_size, as_raw=False, orig_dtype=np.uint64, unified_labels=False, n_max_co_processes=n_max_co_processes, log=log) log.debug( 'Finished conversion of ChunkDataset ({}) into KnossosDataset' ' ({})'.format(cset.path_head_folder, target_kd.knossos_path)) # Write SD max_n_jobs = global_params.NNODES_TOTAL * 2 path = "{}/knossosdatasets/syn_seg/".format( global_params.config.working_dir) path_cs = "{}/knossosdatasets/cs_seg/".format( global_params.config.working_dir) storage_location_ids = rep_helper.get_unique_subfold_ixs(n_folders_fs) multi_params = [ (sv_id_block, n_folders_fs, path, path_cs) for sv_id_block in basics.chunkify(storage_location_ids, max_n_jobs) ] if not qu.batchjob_enabled(): start_multiprocess_imap(_write_props_to_syn_singlenode_thread, multi_params, nb_cpus=1, debug=False) else: qu.QSUB_script(multi_params, "write_props_to_syn_singlenode", log=log, n_cores=global_params.NCORES_PER_NODE, n_max_co_processes=global_params.NNODES_TOTAL, remove_jobfolder=True) sd = segmentation.SegmentationDataset( working_dir=global_params.config.working_dir, obj_type='syn', version=0) dataset_analysis(sd, recompute=True, compute_meshprops=False) sd = segmentation.SegmentationDataset( working_dir=global_params.config.working_dir, obj_type='cs', version=0) dataset_analysis(sd, recompute=True, compute_meshprops=False) for p in dict_paths: os.remove(p) shutil.rmtree(cd_dir, ignore_errors=True)
def node_to_sv_id_queue_worker(node_to_sv_mappings_todo_queue, node_to_sv_mappings_done_queue, use_brainmaps=False, kd_seg_path='/mnt/j0126_cubed/'): if use_brainmaps == True: bmi = bm.BrainmapsInteraction(json_key=bm.service_account) volume_id = 'j0126_13_v4b_cbs_ext0_fixed' project_id = '611024335609' dataset_id = 'j0126' # volume_id = 'nov2015_mask135_0945_1000_gala_0.95' # project_id = '611024335609' # dataset_id = 'j0126' else: kd = kds.KnossosDataset() kd.initialize_from_knossos_path(kd_seg_path, cache_size=10) while True: # this is blocking and therefore fine # skel_node = node_to_sv_mappings_todo_queue.get() corner_xyz = node_to_sv_mappings_todo_queue.get() # corner_xyz = skel_node.getCoordinate() # hack to reverse xy # corner_xyz = [corner_xyz[0], corner_xyz[1], corner_xyz[2]] corner_xyz[0], corner_xyz[1], corner_xyz[ 2] = corner_xyz[0] - 1, corner_xyz[1] - 1, corner_xyz[2] - 1 oob = False if corner_xyz[0] <= 0: oob = True # corner_xyz[0] = 1; oob = True if corner_xyz[1] <= 0: oob = True # corner_xyz[1] = 1; oob = True if corner_xyz[2] <= 0: oob = True # corner_xyz[2] = 1; oob = True if corner_xyz[0] > 10500: oob = True # corner_xyz[0] = 10500; oob = True if corner_xyz[1] > 10770: oob = True # corner_xyz[1] = 10770; oob = True if corner_xyz[2] > 5698: oob = True # corner_xyz[2] = 5698; oob = True # if corner_xyz[0] > 5000: corner_xyz[0] = 5000 # if corner_xyz[1] > 5000: corner_xyz[1] = 5000 # if corner_xyz[2] > 5000: corner_xyz[2] = 5000 attempt = 0 succes = False while not succes: try: if use_brainmaps: node_sample = bmi.get_subvolume_chunk( project_id=project_id, dataset_id=dataset_id, volume_id=volume_id, corner_z=corner_xyz[0], corner_y=corner_xyz[1], corner_x=corner_xyz[2], size_z=2, size_y=2, size_x=2, subvol_format='RAW_SNAPPY', gzip=0) else: if not oob: node_sample = kd.from_overlaycubes_to_matrix( (1, 1, 1), corner_xyz, verbose=False, show_progress=False, nb_threads=1) succes = True except Exception as e: print("".join(traceback.format_exception(*sys.exc_info()))) print(str(e)) attempt += 1 time.sleep(10) print( 'Retrying subvol download, attempt: {0}, subvol: {1}, reinitializing bmi' .format(attempt, (corner_xyz))) if use_brainmaps == True: bmi = bm.BrainmapsInteraction(json_key=bm.service_account) if not oob: # find most frequent id node_sample = flatten(node_sample.tolist()) #print(node_sample) # print('got node sample {0}'.format(node_sample)) from collections import Counter cnt = Counter(node_sample) sv_id = cnt.most_common()[0][0] else: sv_id = 0 # sv_id = np.argmax(np.bincount(node_sample.reshape([node_sample.size]).astype(np.int64))) corner_xyz[0], corner_xyz[1], corner_xyz[ 2] = corner_xyz[0] + 1, corner_xyz[1] + 1, corner_xyz[2] + 1 node_to_sv_mappings_done_queue.put((corner_xyz, sv_id)) node_to_sv_mappings_todo_queue.task_done() return
""" Script to convert Numpy npy files to Knossos datasets (.conf and .raw files). Command line arguments: * Path to input numpy file * Path to output dataset. Knossos creates top-level directories (mag1, mag2, etc.) automatically - provide the parent of this top-level directory! Do not provide uncreated directory paths. * Experiment Name WARNING: I don't think using npy_to_knossos.py and knossos_to_npy.py in conjunction will leave you with identical npy files - once touched by Knossos, forever changed (Knossos uses uint8 I think). That is to say, npy1 -> knossos (.raw) -> npy2 =/=> npy1 = npy2 if you use npy_to_knossos.py and knossos_to_npy.py """ import sys import numpy from knossos_utils import knossosdataset as knosdata if __name__ == '__main__': if len(sys.argv) < 4: raise RuntimeError( 'Usage: python npy_to_knossos.py <path_to_input_file> <path_to_output_dataset> <experiment_name>' ) in_file = sys.argv[1] out_path = sys.argv[2] exp_name = sys.argv[3] volume = numpy.load(in_file).T kd = knosdata.KnossosDataset() kd.initialize_from_matrix(out_path, [1.0, 1.0, 1.0], exp_name, data=volume)
if not (os.path.isfile(mpath) or os.path.isdir(mpath)): raise ValueError('Could not find model "{}". Make sure to copy the' ' "models" folder into the current working ' 'directory "{}".'.format(mpath, example_wd)) if not prior_glia_removal: shutil.copy(h5_dir + "/neuron_rag.bz2", global_params.config.init_rag_path) else: shutil.copy(h5_dir + "/rag.bz2", global_params.config.init_rag_path) bb = parse_movement_area_from_zip(kzip_p) offset = np.array([0, 0, 0]) bd = bb[1] - bb[0] # INITIALIZE DATA kd = knossosdataset.KnossosDataset() kd.initialize_from_matrix(global_params.config.kd_seg_path, scale, experiment_name, offset=offset, boundary=bd, fast_downsampling=True, data_path=h5_dir + 'raw.h5', mags=[1, 2, 4], hdf5_names=['raw']) seg_d = load_from_h5py(h5_dir + 'seg.h5', hdf5_names=['seg'])[0] kd.from_matrix_to_cubes(offset, mags=[1, 2, 4], data=seg_d, fast_downsampling=True, as_raw=False) kd_mi = knossosdataset.KnossosDataset() kd_mi.initialize_from_matrix(global_params.config.kd_mi_path, scale, experiment_name, offset=offset, boundary=bd, fast_downsampling=True, data_path=h5_dir + 'mi.h5', mags=[1, 2], hdf5_names=['mi']) kd_vc = knossosdataset.KnossosDataset() kd_vc.initialize_from_matrix(global_params.config.kd_vc_path, scale, experiment_name,
def run_create_sds(chunk_size=None, n_folders_fs=10000, max_n_jobs=None, generate_sv_meshes=False, load_from_kd_overlaycubes=False, cube_of_interest_bb=None): """ Parameters ---------- chunk_size : max_n_jobs : int n_folders_fs : generate_sv_meshes : load_from_kd_overlaycubes : bool Load prob/seg data from overlaycubes instead of raw cubes. cube_of_interest_bb : Tuple[np.ndarray] Defines the bounding box of the cube to process. By default this is set to (np.zoers(3); kd.boundary). Returns ------- """ if chunk_size is None: chunk_size = [512, 512, 512] if max_n_jobs is None: max_n_jobs = global_params.NCORE_TOTAL * 3 log = initialize_logging('create_sds', global_params.config.working_dir + '/logs/', overwrite=False) # Sets initial values of object kd = kd_factory(global_params.config.kd_seg_path) if cube_of_interest_bb is None: cube_of_interest_bb = [np.zeros(3, dtype=np.int), kd.boundary] size = cube_of_interest_bb[1] - cube_of_interest_bb[0] + 1 offset = cube_of_interest_bb[0] # TODO: get rid of explicit voxel extraction, all info necessary should be extracted # at the beginning, e.g. size, bounding box etc and then refactor to only use those cached attributes! # resulting ChunkDataset, required for SV extraction -- # Object extraction - 2h, the same has to be done for all cell organelles cd_dir = global_params.config.working_dir + "chunkdatasets/sv/" # Class that contains a dict of chunks (with coordinates) after initializing it cd = chunky.ChunkDataset() cd.initialize(kd, kd.boundary, chunk_size, cd_dir, box_coords=[0, 0, 0], fit_box_size=True) log.info('Generating SegmentationDatasets for cell and cell ' 'organelle supervoxels.') oew.from_ids_to_objects( cd, "sv", overlaydataset_path=global_params.config.kd_seg_path, n_chunk_jobs=max_n_jobs, hdf5names=["sv"], n_max_co_processes=None, n_folders_fs=n_folders_fs, use_combined_extraction=True, size=size, offset=offset) # Object Processing -- Perform after mapping to also cache mapping ratios sd = SegmentationDataset("sv", working_dir=global_params.config.working_dir) sd_proc.dataset_analysis(sd, recompute=True, compute_meshprops=False) log.info("Extracted {} cell SVs. Preparing rendering locations " "(and meshes if not provided).".format(len(sd.ids))) start = time.time() # chunk them multi_params = chunkify(sd.so_dir_paths, max_n_jobs) # all other kwargs like obj_type='sv' and version are the current SV SegmentationDataset by default so_kwargs = dict(working_dir=global_params.config.working_dir, obj_type='sv') multi_params = [[par, so_kwargs] for par in multi_params] if generate_sv_meshes: _ = qu.QSUB_script(multi_params, "mesh_caching", n_max_co_processes=global_params.NCORE_TOTAL) _ = qu.QSUB_script(multi_params, "sample_location_caching", n_max_co_processes=global_params.NCORE_TOTAL) # recompute=False: only collect new sample_location property sd_proc.dataset_analysis(sd, compute_meshprops=True, recompute=False) log.info( 'Finished preparation of cell SVs after {:.0f}s.'.format(time.time() - start)) # create SegmentationDataset for each cell organelle for co in global_params.existing_cell_organelles: start = time.time() cd_dir = global_params.config.working_dir + "chunkdatasets/{}/".format( co) cd.initialize(kd, kd.boundary, chunk_size, cd_dir, box_coords=[0, 0, 0], fit_box_size=True) log.info('Started object extraction of cellular organelles "{}" from ' '{} chunks.'.format(co, len(cd.chunk_dict))) prob_kd_path_dict = { co: getattr(global_params.config, 'kd_{}_path'.format(co)) } # This creates a SegmentationDataset of type 'co' prob_thresh = global_params.config.entries["Probathresholds"][ co] # get probability threshold path = "{}/knossosdatasets/{}_seg/".format( global_params.config.working_dir, co) target_kd = knossosdataset.KnossosDataset() target_kd.initialize_without_conf(path, kd.boundary, kd.scale, kd.experiment_name, mags=[ 1, ]) target_kd = knossosdataset.KnossosDataset() target_kd.initialize_from_knossos_path(path) oew.from_probabilities_to_objects( cd, co, # membrane_kd_path=global_params.config.kd_barrier_path, # TODO: currently does not exist prob_kd_path_dict=prob_kd_path_dict, thresholds=[prob_thresh], workfolder=global_params.config.working_dir, hdf5names=[co], n_max_co_processes=None, target_kd=target_kd, n_folders_fs=n_folders_fs, debug=False, size=size, offset=offset, load_from_kd_overlaycubes=load_from_kd_overlaycubes) sd_co = SegmentationDataset( obj_type=co, working_dir=global_params.config.working_dir) # TODO: check if this is faster then the alternative below sd_proc.dataset_analysis(sd_co, recompute=True, compute_meshprops=False) multi_params = chunkify(sd_co.so_dir_paths, max_n_jobs) so_kwargs = dict(working_dir=global_params.config.working_dir, obj_type=co) multi_params = [[par, so_kwargs] for par in multi_params] _ = qu.QSUB_script(multi_params, "mesh_caching", n_max_co_processes=global_params.NCORE_TOTAL) sd_proc.dataset_analysis(sd_co, recompute=False, compute_meshprops=True) # # Old alternative, requires much more reads/writes then above solution # sd_proc.dataset_analysis(sd_co, recompute=True, compute_meshprops=True) # About 0.2 h per object class log.info('Started mapping of {} cellular organelles of type "{}" to ' 'cell SVs.'.format(len(sd_co.ids), co)) sd_proc.map_objects_to_sv(sd, co, global_params.config.kd_seg_path, n_jobs=max_n_jobs) log.info('Finished preparation of {} "{}"-SVs after {:.0f}s.' ''.format(len(sd_co.ids), co, time.time() - start))
def _map_objects_thread(args): """Worker of map_objects_to_sv""" # TODO: this needs to be done densely by matching cell organelle segmentation (see corresponding ChunkDataset which is an intermediate result of 'from_probabilities_to_objects') to SV segmentation paths = args[0] obj_type = args[1] obj_version = args[2] working_dir = args[3] kd_path = args[4] readonly = args[5] global_params.wd = working_dir if len(args) > 6: datatype = args[6] else: datatype = np.uint64 kd = knossosdataset.KnossosDataset() kd.initialize_from_knossos_path(kd_path) seg_dataset = segmentation.SegmentationDataset(obj_type, version=obj_version, working_dir=working_dir) sv_id_dict = {} for p in paths: this_attr_dc = AttributeDict(p + "/attr_dict.pkl", read_only=readonly, disable_locking=True) this_vx_dc = VoxelStorage(p + "/voxel.pkl", read_only=True, disable_locking=True) for so_id in this_vx_dc.keys(): so = seg_dataset.get_segmentation_object(so_id) so.attr_dict = this_attr_dc[so_id] so.load_voxels(voxel_dc=this_vx_dc) if readonly: if "mapping_ids" in so.attr_dict: ids = so.attr_dict["mapping_ids"] id_ratios = so.attr_dict["mapping_ratios"] for i_id in range(len(ids)): if ids[i_id] in sv_id_dict: sv_id_dict[ids[i_id]][so_id] = id_ratios[i_id] else: sv_id_dict[ids[i_id]] = {so_id: id_ratios[i_id]} else: if np.product(so.shape) > 1e12: # TODO: Seems hacky continue vx_list = np.argwhere(so.voxels) + so.bounding_box[0] try: id_list = kd.from_overlaycubes_to_list(vx_list, datatype=datatype) except: log_proc.error('Could not load overlaycube ' 'during object mapping') continue ids, id_counts = np.unique(id_list, return_counts=True) id_ratios = id_counts / float(np.sum(id_counts)) for i_id in range(len(ids)): if ids[i_id] in sv_id_dict: sv_id_dict[ids[i_id]][so_id] = id_ratios[i_id] else: sv_id_dict[ids[i_id]] = {so_id: id_ratios[i_id]} so.attr_dict["mapping_ids"] = ids so.attr_dict["mapping_ratios"] = id_ratios this_attr_dc[so_id] = so.attr_dict if not readonly: this_attr_dc.push() return sv_id_dict