def generate_subcell_kd_from_proba(co, chunk_size=None, transf_func_kd_overlay=None,
load_cellorganelles_from_kd_overlaycubes=False,
cube_of_interest_bb=None, log=None, **kwargs):
"""
Generate KnossosDatasets for given subcellular structure key (e.g. 'mi').
The required initial data format is a chunkdataset located at
``"{}/chunkdatasets/{}/".format(global_params.config.working_dir, co)``.
Resulting KD will be stored at
``"{}/knossosdatasets/{}_seg/".format(global_params.config.working_dir, co)``.
See :func:`~syconn.extraction.object_extraction_wrapper.from_probabilities_to_kd` for details of
the conversion process from the initial probability map to the SV segmentation. Default:
thresholding and connected components, thresholds are set via the `config.ini` file, check
``syconn.global_params.config.entries["Probathresholds"]`` of an initialized
:calss:`~syconn.handler.config.DynConfig` object.
Parameters
----------
co : str
chunk_size : Tuple
transf_func_kd_overlay : callable
load_cellorganelles_from_kd_overlaycubes : bool
cube_of_interest_bb : Tuple[Tuple[int]] or np.ndarray
log : logger
"""
if chunk_size is None:
chunk_size = [512, 512, 512]
if log is None:
log = log_extraction
kd = basics.kd_factory(global_params.config.kd_seg_path)
if cube_of_interest_bb is None:
cube_of_interest_bb = [np.zeros(3, dtype=np.int), kd.boundary]
size = cube_of_interest_bb[1] - cube_of_interest_bb[0] + 1
offset = cube_of_interest_bb[0]
cd_dir = "{}/chunkdatasets/{}/".format(global_params.config.working_dir, co)
cd = chunky.ChunkDataset()
cd.initialize(kd, kd.boundary, chunk_size, cd_dir,
box_coords=[0, 0, 0], fit_box_size=True,
list_of_coords=[])
log.info('Started object extraction of cellular organelles "{}" from '
'{} chunks.'.format(co, len(cd.chunk_dict)))
prob_kd_path_dict = {co: getattr(global_params.config, 'kd_{}_path'.format(co))}
# This creates a SegmentationDataset of type 'co'
prob_thresh = global_params.config.entries["Probathresholds"][co] # get probability threshold
# `from_probabilities_to_objects` will export a KD at `path`, remove if already existing
path = global_params.config.kd_organelle_seg_paths[co]
if os.path.isdir(path):
log.debug('Found existing KD at {}. Removing it now.'.format(path))
shutil.rmtree(path)
target_kd = knossosdataset.KnossosDataset()
scale = np.array(global_params.config.entries["Dataset"]["scaling"], dtype=np.float32)
target_kd.initialize_without_conf(path, kd.boundary, scale, kd.experiment_name, mags=[1, ])
target_kd = knossosdataset.KnossosDataset()
target_kd.initialize_from_knossos_path(path)
from_probabilities_to_kd(cd, co, # membrane_kd_path=global_params.config.kd_barrier_path, # TODO: currently does not exist
prob_kd_path_dict=prob_kd_path_dict, thresholds=[prob_thresh],
hdf5names=[co], n_max_co_processes=None, target_kd=target_kd,
debug=False, size=size, offset=offset,
load_from_kd_overlaycubes=load_cellorganelles_from_kd_overlaycubes,
transf_func_kd_overlay=transf_func_kd_overlay[co], log=log, **kwargs)
def _extract_synapse_type_thread(args):
paths = args[0]
obj_version = args[1]
working_dir = args[2]
kd_asym_path = args[3]
kd_sym_path = args[4]
trafo_dict_path = args[5]
if trafo_dict_path is not None:
with open(trafo_dict_path, "rb") as f:
trafo_dict = pkl.load(f)
else:
trafo_dict = None
kd_asym = knossosdataset.KnossosDataset()
kd_asym.initialize_from_knossos_path(kd_asym_path)
kd_sym = knossosdataset.KnossosDataset()
kd_sym.initialize_from_knossos_path(kd_sym_path)
seg_dataset = segmentation.SegmentationDataset("syn_ssv",
version=obj_version,
working_dir=working_dir)
for p in paths:
this_attr_dc = AttributeDict(p + "/attr_dict.pkl",
read_only=False, disable_locking=True)
for so_id in this_attr_dc.keys():
so = seg_dataset.get_segmentation_object(so_id)
so.attr_dict = this_attr_dc[so_id]
so.load_voxel_list()
vxl = so.voxel_list
if trafo_dict is not None:
vxl -= trafo_dict[so_id]
vxl = vxl[:, [1, 0, 2]]
# TODO: remvoe try-except
try:
asym_prop = np.mean(kd_asym.from_raw_cubes_to_list(vxl))
sym_prop = np.mean(kd_sym.from_raw_cubes_to_list(vxl))
except:
log_proc.error("Failed to read raw cubes during synapse type "
"extraction.")
sym_prop = 0
asym_prop = 0
if sym_prop + asym_prop == 0:
sym_ratio = -1
else:
sym_ratio = sym_prop / float(asym_prop + sym_prop)
so.attr_dict["syn_type_sym_ratio"] = sym_ratio
syn_sign = -1 if sym_ratio > global_params.sym_thresh else 1
so.attr_dict["syn_sign"] = syn_sign
this_attr_dc[so_id] = so.attr_dict
this_attr_dc.push()
def swc_to_mask(f_cube, f_swc, f_output):
''' f_cube: ../mag1/knossos.conf
f_swc: ../*.swc
f_output: output dir
'''
if not os.path.exists(f_output):
os.makedirs(f_output)
f_cube = os.path.abspath(f_cube)
f_output = os.path.abspath(f_output)
f_swc = os.path.abspath(f_swc)
f_tiff_output = os.path.join(f_output, 'stack')
f_mask = os.path.join(f_output, 'output_mask')
f_v3d = os.path.join(f_output, 'output_mask.v3draw')
kd = knossosdataset.KnossosDataset()
kd.initialize_from_knossos_path(f_cube)
print(kd._experiment_name)
print(kd.boundary)
x, y, z = kd.boundary
#z,x,y = kd.boundary
command = r'vaa3d -x swc_to_maskimage_sphere -f swc_to_maskimage -i {} -p {} {} {} -o {}.tiff'.format(
f_swc, str(x), str(y), str(z), f_mask)
print(command)
#fiji_command = r'run("Convert...", "input={} output={} output_format=TIFF interpolation=Bilinear scale=1"); '.format(f_output, f_output)
# print(fiji_command)
#f_fiji = os.path.join(f_output, 'tmp.ijm')
f_log = os.path.join(f_output, 'log.txt')
# with open(f_fiji, 'w') as f:
# f.write(fiji_command+'\n')
#run_fiji_command = r'fiji -macro {}'.format(f_fiji)
subprocess.call(command, shell=True)
def labels_to_knossos(f_cube, f_anno, f_ext, mode):
'''converts segmentated mask back into knossos segmentation channel'''
kd = knossosdataset.KnossosDataset()
kd.initialize_from_knossos_path(f_cube)
#raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0])
print(kd._experiment_name)
print(kd.boundary)
if mode == 'to':
# print(overlay.shape)
# print(np.unique(overlay[:]))
new_overlay = omni_read(f_ext)
new_overlay = np.rollaxis(new_overlay, 2, 0)
new_overlay = np.rollaxis(new_overlay, 2, 1)
print(new_overlay.shape)
print(np.unique(new_overlay))
new_kzip = kd.from_matrix_to_cubes(offset=(
0, 0, 0), data=new_overlay, kzip_path=os.path.join(os.path.dirname(f_anno), 'new'))
elif mode == 'from':
overlay = kd.from_kzip_to_matrix(path=f_anno, size=kd.boundary, offset=[
0, 0, 0], mag=1, verbose=True, alt_exp_name_kzip_path_mode=True)
print(np.unique(overlay[:]))
rolled_overlay = np.rollaxis(overlay, 2, 0)
rolled_overlay = np.rollaxis(rolled_overlay, 2, 1)
f_out = os.path.dirname(f_ext)
if not os.path.exists(f_out):
os.makedirs(f_out)
tifffile.imsave(f_ext, np.uint32(rolled_overlay))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--cube',
default=
'/mnt/md0/XRay/Knossos_measurements/WT_colon_380/knossos_cube/mag1/knossos.conf'
)
parser.add_argument(
'--anno',
default=
'/mnt/md0/XRay/Knossos_measurements/WT_colon_380/annotation/muscle_skeleton_annotation-20180113T1805.139.k.zip'
)
args = parser.parse_args()
print(args.cube, args.anno)
root_dir = '.'
f_knossos = args.cube
f_overlay = args.anno
f_out = os.path.join(root_dir, 'output')
if not os.path.exists(f_out):
os.makedirs(f_out)
kd = knossosdataset.KnossosDataset()
kd.initialize_from_knossos_path(f_knossos)
#raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0])
print(kd._experiment_name)
print(kd.boundary)
#overlay= kd.from_kzip_to_matrix(path=f_overlay,size=kd.boundary, offset=[0, 0, 0], mag=1, verbose=True, alt_exp_name_kzip_path_mode=False)
sk = skeleton.Skeleton()
sk.fromNml(f_overlay)
result_dict = sweep_sk(sk)
result_dict = get_stat(sk, result_dict)
convert_to_csv(result_dict, f_out)
sys.exit()
def _export_sd_to_knossosdataset_thread(args):
block_loc = args[0]
so_ids = args[1]
obj_type = args[2]
version = args[3]
working_dir = args[4]
kd_path = args[5]
block_edge_length = args[6]
block_size = np.array([block_edge_length] * 3, dtype=np.int)
kd = knossosdataset.KnossosDataset()
kd.initialize_from_knossos_path(kd_path)
sd = segmentation.SegmentationDataset(obj_type=obj_type,
working_dir=working_dir,
version=version)
overlay_block = np.zeros(block_size, dtype=np.uint64)
block_start = (block_loc * block_size).astype(np.int)
for so_id in so_ids:
so = sd.get_segmentation_object(so_id, False)
vx = so.voxel_list - block_start
vx = vx[~np.any(vx < 0, axis=1)]
vx = vx[~np.any(vx >= block_edge_length, axis=1)]
overlay_block[vx[:, 0], vx[:, 1], vx[:, 2]] = so_id
kd.from_matrix_to_cubes(block_start,
data=overlay_block,
overwrite=True,
nb_threads=1,
verbose=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cube',
default='../full_stack_rot_knossos/mag1/knossos.conf')
parser.add_argument('--anno', default='../trace/')
parser.add_argument('--output', default='.')
parser.add_argument('--features',
type=str,
choices=['both', 'cellbody', 'dendrite', 'mask'],
default='both')
args = parser.parse_args()
f_knossos = args.cube
f_overlay = max(glob.iglob(os.path.join(args.anno, '*.k.zip')),
key=os.path.getmtime)
f_output = args.output
if not os.path.exists(f_output):
os.makedirs(f_output)
f_swc = os.path.join(f_output, 'output.swc')
f_center = os.path.join(f_output, 'center.txt')
kd = knossosdataset.KnossosDataset()
kd.initialize_from_knossos_path(f_knossos)
#raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0])
print(kd._experiment_name)
print(kd.boundary)
#overlay= kd.from_kzip_to_matrix(path=f_overlay,size=kd.boundary, offset=[0, 0, 0], mag=1, verbose=True, alt_exp_name_kzip_path_mode=False)
sk = skeleton.Skeleton()
sk.fromNml(f_overlay)
if args.features == 'both':
sk2swc_and_center(sk, f_swc, f_center)
elif args.features == 'cellbody':
sk2swc_cellbody_and_center(sk, f_swc, f_center)
elif args.features == 'dendrite':
sk2swc_dendrite_and_center(sk, f_swc, f_center)
elif args.features == 'mask':
sk.toSWC(f_swc)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cube', default=None)
parser.add_argument('--anno', default=None)
parser.add_argument('--mag', type=int, default=1)
parser.add_argument('--output', default=None)
args = parser.parse_args()
f_knossos = args.cube
f_overlay = args.anno
f_out = args.output
# matches = re.match('^.*/mag(?P<mag>[0-9]+)/.*$', args.cube).groupdict()
# mag = int(matches['mag'])
mag = args.mag
kd = knossosdataset.KnossosDataset()
kd.initialize_from_knossos_path(f_knossos)
#raw = kd.from_raw_cubes_to_matrix(size=kd.boundary, offset=[0, 0, 0])
print(kd._experiment_name)
# sys.exit()
print(kd.boundary)
overlay = kd.from_kzip_to_matrix(
path=f_overlay,
# size=kd.boundary//16,
size=[1000, 1000, 1000],
offset=[4300, 4300, 4300],
mag=mag,
verbose=True)
# alt_exp_name_kzip_path_mode=True)
print(overlay.shape)
print(np.max(overlay[:]))
overlay = np.rollaxis(overlay, 2, 0)
overlay = np.rollaxis(overlay, 2, 1)
#dxchange.write_tiff(overlay.astype(np.uint32), f_out, overwrite=True)
print(overlay.shape, overlay.dtype)
print(np.mean(overlay[...]))
def extract_agg_contact_sites(cset,
working_dir,
filename='cs',
hdf5name='cs',
n_folders_fs=10000,
suffix="",
n_max_co_processes=None,
n_chunk_jobs=2000,
size=None,
offset=None,
log=None):
"""
Parameters
----------
cset :
working_dir :
filename :
hdf5name :
n_folders_fs :
suffix :
n_max_co_processes :
n_chunk_jobs :
size :
offset :
Returns
-------
"""
if log is None:
log = log_extraction
chunky.save_dataset(cset)
# init CS segmentation KD
kd = kd_factory(global_params.config.kd_seg_path)
path = "{}/knossosdatasets/{}_seg/".format(
global_params.config.working_dir, filename)
if os.path.isdir(path):
log.debug('Found existing KD at {}. Removing it now.'.format(path))
shutil.rmtree(path)
target_kd = knossosdataset.KnossosDataset()
scale = np.array(global_params.config.entries["Dataset"]["scaling"])
target_kd.initialize_without_conf(path,
kd.boundary,
scale,
kd.experiment_name,
mags=[
1,
])
target_kd = knossosdataset.KnossosDataset()
target_kd.initialize_from_knossos_path(path)
# convert Chunkdataset to KD
export_cset_to_kd_batchjob(cset,
target_kd,
'{}'.format(filename), [hdf5name],
offset=offset,
size=size,
stride=[4 * 128, 4 * 128, 4 * 128],
as_raw=False,
orig_dtype=np.uint64,
unified_labels=False,
n_max_co_processes=n_max_co_processes)
log.debug(
'Finished conversion of ChunkDataset ({}) into KnossosDataset ({})'.
format(cset.path_head_folder, target_kd.knossos_path))
# get CS SD
from_ids_to_objects(cset,
filename,
overlaydataset_path=target_kd.conf_path,
n_chunk_jobs=n_chunk_jobs,
hdf5names=[hdf5name],
n_max_co_processes=n_max_co_processes,
workfolder=working_dir,
n_folders_fs=n_folders_fs,
use_combined_extraction=True,
suffix=suffix,
size=size,
offset=offset,
log=log)
def extract_contact_sites(n_max_co_processes: Optional[int] = None,
chunk_size: Optional[Tuple[int, int, int]] = None,
log: Optional[Logger] = None,
max_n_jobs: Optional[int] = None,
cube_of_interest_bb: Optional[np.ndarray] = None,
n_folders_fs: int = 1000):
"""
Extracts contact sites and their overlap with `sj` objects and stores them in a
:class:`~syconn.reps.segmentation.SegmentationDataset` of type ``cs`` and ``syn``
respectively. If synapse type is available, this information will be stored
as the voxel-ratio per class in the attribute dictionary of the ``syn``
objects (keys: ``sym_prop``, ``asym_prop``).
Notes:
Replaced ``find_contact_sites``, ``extract_agg_contact_sites``, `
`syn_gen_via_cset`` and ``extract_synapse_type``.
Args:
n_max_co_processes: Number of parallel workers.
chunk_size: Sub-cube volume which is processed at a time.
log: Logger.
max_n_jobs: Maximum number of jobs.
cube_of_interest_bb: Sub-volume of the data set which is processed.
Default: Entire data set.
n_folders_fs: Number of folders used for organizing supervoxel data.
"""
if extract_cs_syntype is None:
msg = '`extract_contact_sites` requires the cythonized method ' \
'`extract_cs_syntype`. Use `find_contact_sites` and others ' \
'for contact site processing.'
log_extraction.error(msg)
raise ImportError(msg)
kd = kd_factory(global_params.config.kd_seg_path)
if cube_of_interest_bb is None:
cube_of_interest_bb = [np.zeros(3, dtype=np.int), kd.boundary]
if chunk_size is None:
chunk_size = (512, 512, 512)
size = cube_of_interest_bb[1] - cube_of_interest_bb[0] + 1
offset = cube_of_interest_bb[0]
# Initital contact site extraction
cd_dir = global_params.config.temp_path + "/chunkdatasets/cs/"
# Class that contains a dict of chunks (with coordinates) after initializing it
cset = chunky.ChunkDataset()
cset.initialize(kd,
kd.boundary,
chunk_size,
cd_dir,
box_coords=[0, 0, 0],
fit_box_size=True)
if max_n_jobs is None:
max_n_jobs = global_params.NCORE_TOTAL * 2
if log is None:
log = log_extraction
if size is not None and offset is not None:
chunk_list, _ = \
calculate_chunk_numbers_for_box(cset, offset, size)
else:
chunk_list = [ii for ii in range(len(cset.chunk_dict))]
# shuffle chunklist to get a more balanced work-load
rand_ixs = np.arange(len(chunk_list))
np.random.shuffle(rand_ixs)
chunk_list = np.array(chunk_list)[rand_ixs]
os.makedirs(cset.path_head_folder, exist_ok=True)
multi_params = []
# TODO: currently pickles Chunk objects -> job submission might be slow
for chunk_k in chunkify(chunk_list, max_n_jobs):
multi_params.append([[cset.chunk_dict[k] for k in chunk_k],
global_params.config.kd_seg_path])
if not qu.batchjob_enabled():
results = start_multiprocess_imap(_contact_site_extraction_thread,
multi_params,
debug=False,
nb_cpus=n_max_co_processes)
else:
path_to_out = qu.QSUB_script(multi_params,
"contact_site_extraction",
n_max_co_processes=n_max_co_processes,
log=log)
out_files = glob.glob(path_to_out + "/*")
results = []
for out_file in out_files:
with open(out_file, 'rb') as f:
results.append(pkl.load(f))
shutil.rmtree(os.path.abspath(path_to_out + "/../"),
ignore_errors=True)
# reduce step
cs_props = [{}, defaultdict(list), {}]
syn_props = [{}, defaultdict(list), {}]
tot_sym_cnt = {}
tot_asym_cnt = {}
for curr_props, curr_syn_props, asym_cnt, sym_cnt in results:
merge_prop_dicts([cs_props, curr_props])
merge_prop_dicts([syn_props, curr_syn_props])
merge_type_dicts([tot_asym_cnt, asym_cnt])
merge_type_dicts([tot_sym_cnt, sym_cnt])
log.info('Finished contact site (#objects: {}) and synapse (#objects: {})'
' extraction.'.format(len(cs_props[0]), len(syn_props[0])))
if len(syn_props[0]) == 0:
log.critical(
'WARNING: Did not find any synapses during extraction step.')
# TODO: extract syn objects! maybe replace sj_0 Segmentation dataset by the overlapping CS<->
# sj objects -> run syn. extraction and sd_generation in parallel and return mi_0, vc_0 and
# syn_0 -> use syns as new sjs during rendering!
# -> Run CS generation in parallel with mapping to at least get the syn objects before
# rendering the neuron views (which need subcellular structures, there one can then use mi,
# vc and syn (instead of sj))
dict_paths = []
# dump intermediate results
# TODO: size filter here or during write-out? TODO: use config parameter
dict_p = "{}/cs_prop_dict.pkl".format(global_params.config.temp_path)
with open(dict_p, "wb") as f:
pkl.dump(cs_props, f)
del cs_props
dict_paths.append(dict_p)
dict_p = "{}/syn_prop_dict.pkl".format(global_params.config.temp_path)
with open(dict_p, "wb") as f:
pkl.dump(syn_props, f)
del syn_props
dict_paths.append(dict_p)
# convert counting dicts to store ratio of syn. type voxels
dict_p = "{}/cs_sym_cnt.pkl".format(global_params.config.temp_path)
with open(dict_p, "wb") as f:
pkl.dump(tot_sym_cnt, f)
del tot_sym_cnt
dict_paths.append(dict_p)
dict_p = "{}/cs_asym_cnt.pkl".format(global_params.config.temp_path)
with open(dict_p, "wb") as f:
pkl.dump(tot_asym_cnt, f)
del tot_asym_cnt
dict_paths.append(dict_p)
# write cs and syn segmentation to KD and SD
chunky.save_dataset(cset)
kd = kd_factory(global_params.config.kd_seg_path)
# convert Chunkdataset to syn and cs KD
# TODO: spawn in parallel
for obj_type in ['cs', 'syn']:
path = "{}/knossosdatasets/{}_seg/".format(
global_params.config.working_dir, obj_type)
if os.path.isdir(path):
log.debug('Found existing KD at {}. Removing it now.'.format(path))
shutil.rmtree(path)
target_kd = knossosdataset.KnossosDataset()
scale = np.array(global_params.config.entries["Dataset"]["scaling"])
target_kd.initialize_without_conf(path,
kd.boundary,
scale,
kd.experiment_name,
mags=[
1,
])
target_kd = knossosdataset.KnossosDataset()
target_kd.initialize_from_knossos_path(path)
export_cset_to_kd_batchjob(cset,
target_kd,
obj_type, [obj_type],
offset=offset,
size=size,
stride=chunk_size,
as_raw=False,
orig_dtype=np.uint64,
unified_labels=False,
n_max_co_processes=n_max_co_processes,
log=log)
log.debug(
'Finished conversion of ChunkDataset ({}) into KnossosDataset'
' ({})'.format(cset.path_head_folder, target_kd.knossos_path))
# Write SD
max_n_jobs = global_params.NNODES_TOTAL * 2
path = "{}/knossosdatasets/syn_seg/".format(
global_params.config.working_dir)
path_cs = "{}/knossosdatasets/cs_seg/".format(
global_params.config.working_dir)
storage_location_ids = rep_helper.get_unique_subfold_ixs(n_folders_fs)
multi_params = [
(sv_id_block, n_folders_fs, path, path_cs)
for sv_id_block in basics.chunkify(storage_location_ids, max_n_jobs)
]
if not qu.batchjob_enabled():
start_multiprocess_imap(_write_props_to_syn_singlenode_thread,
multi_params,
nb_cpus=1,
debug=False)
else:
qu.QSUB_script(multi_params,
"write_props_to_syn_singlenode",
log=log,
n_cores=global_params.NCORES_PER_NODE,
n_max_co_processes=global_params.NNODES_TOTAL,
remove_jobfolder=True)
sd = segmentation.SegmentationDataset(
working_dir=global_params.config.working_dir,
obj_type='syn',
version=0)
dataset_analysis(sd, recompute=True, compute_meshprops=False)
sd = segmentation.SegmentationDataset(
working_dir=global_params.config.working_dir, obj_type='cs', version=0)
dataset_analysis(sd, recompute=True, compute_meshprops=False)
for p in dict_paths:
os.remove(p)
shutil.rmtree(cd_dir, ignore_errors=True)
def node_to_sv_id_queue_worker(node_to_sv_mappings_todo_queue,
node_to_sv_mappings_done_queue,
use_brainmaps=False,
kd_seg_path='/mnt/j0126_cubed/'):
if use_brainmaps == True:
bmi = bm.BrainmapsInteraction(json_key=bm.service_account)
volume_id = 'j0126_13_v4b_cbs_ext0_fixed'
project_id = '611024335609'
dataset_id = 'j0126'
# volume_id = 'nov2015_mask135_0945_1000_gala_0.95'
# project_id = '611024335609'
# dataset_id = 'j0126'
else:
kd = kds.KnossosDataset()
kd.initialize_from_knossos_path(kd_seg_path, cache_size=10)
while True:
# this is blocking and therefore fine
# skel_node = node_to_sv_mappings_todo_queue.get()
corner_xyz = node_to_sv_mappings_todo_queue.get()
# corner_xyz = skel_node.getCoordinate()
# hack to reverse xy
# corner_xyz = [corner_xyz[0], corner_xyz[1], corner_xyz[2]]
corner_xyz[0], corner_xyz[1], corner_xyz[
2] = corner_xyz[0] - 1, corner_xyz[1] - 1, corner_xyz[2] - 1
oob = False
if corner_xyz[0] <= 0: oob = True # corner_xyz[0] = 1; oob = True
if corner_xyz[1] <= 0: oob = True # corner_xyz[1] = 1; oob = True
if corner_xyz[2] <= 0: oob = True # corner_xyz[2] = 1; oob = True
if corner_xyz[0] > 10500:
oob = True # corner_xyz[0] = 10500; oob = True
if corner_xyz[1] > 10770:
oob = True # corner_xyz[1] = 10770; oob = True
if corner_xyz[2] > 5698: oob = True # corner_xyz[2] = 5698; oob = True
# if corner_xyz[0] > 5000: corner_xyz[0] = 5000
# if corner_xyz[1] > 5000: corner_xyz[1] = 5000
# if corner_xyz[2] > 5000: corner_xyz[2] = 5000
attempt = 0
succes = False
while not succes:
try:
if use_brainmaps:
node_sample = bmi.get_subvolume_chunk(
project_id=project_id,
dataset_id=dataset_id,
volume_id=volume_id,
corner_z=corner_xyz[0],
corner_y=corner_xyz[1],
corner_x=corner_xyz[2],
size_z=2,
size_y=2,
size_x=2,
subvol_format='RAW_SNAPPY',
gzip=0)
else:
if not oob:
node_sample = kd.from_overlaycubes_to_matrix(
(1, 1, 1),
corner_xyz,
verbose=False,
show_progress=False,
nb_threads=1)
succes = True
except Exception as e:
print("".join(traceback.format_exception(*sys.exc_info())))
print(str(e))
attempt += 1
time.sleep(10)
print(
'Retrying subvol download, attempt: {0}, subvol: {1}, reinitializing bmi'
.format(attempt, (corner_xyz)))
if use_brainmaps == True:
bmi = bm.BrainmapsInteraction(json_key=bm.service_account)
if not oob:
# find most frequent id
node_sample = flatten(node_sample.tolist())
#print(node_sample)
# print('got node sample {0}'.format(node_sample))
from collections import Counter
cnt = Counter(node_sample)
sv_id = cnt.most_common()[0][0]
else:
sv_id = 0
# sv_id = np.argmax(np.bincount(node_sample.reshape([node_sample.size]).astype(np.int64)))
corner_xyz[0], corner_xyz[1], corner_xyz[
2] = corner_xyz[0] + 1, corner_xyz[1] + 1, corner_xyz[2] + 1
node_to_sv_mappings_done_queue.put((corner_xyz, sv_id))
node_to_sv_mappings_todo_queue.task_done()
return
"""
Script to convert Numpy npy files to Knossos datasets (.conf and .raw files).
Command line arguments:
* Path to input numpy file
* Path to output dataset. Knossos creates top-level directories (mag1, mag2, etc.) automatically - provide the parent of this top-level directory! Do not provide uncreated directory paths.
* Experiment Name
WARNING: I don't think using npy_to_knossos.py and knossos_to_npy.py in conjunction will leave you with identical npy files -
once touched by Knossos, forever changed (Knossos uses uint8 I think). That is to say, npy1 -> knossos (.raw) -> npy2 =/=> npy1 = npy2
if you use npy_to_knossos.py and knossos_to_npy.py
"""
import sys
import numpy
from knossos_utils import knossosdataset as knosdata
if __name__ == '__main__':
if len(sys.argv) < 4:
raise RuntimeError(
'Usage: python npy_to_knossos.py <path_to_input_file> <path_to_output_dataset> <experiment_name>'
)
in_file = sys.argv[1]
out_path = sys.argv[2]
exp_name = sys.argv[3]
volume = numpy.load(in_file).T
kd = knosdata.KnossosDataset()
kd.initialize_from_matrix(out_path, [1.0, 1.0, 1.0], exp_name, data=volume)
if not (os.path.isfile(mpath) or os.path.isdir(mpath)):
raise ValueError('Could not find model "{}". Make sure to copy the'
' "models" folder into the current working '
'directory "{}".'.format(mpath, example_wd))
if not prior_glia_removal:
shutil.copy(h5_dir + "/neuron_rag.bz2", global_params.config.init_rag_path)
else:
shutil.copy(h5_dir + "/rag.bz2", global_params.config.init_rag_path)
bb = parse_movement_area_from_zip(kzip_p)
offset = np.array([0, 0, 0])
bd = bb[1] - bb[0]
# INITIALIZE DATA
kd = knossosdataset.KnossosDataset()
kd.initialize_from_matrix(global_params.config.kd_seg_path, scale, experiment_name,
offset=offset, boundary=bd, fast_downsampling=True,
data_path=h5_dir + 'raw.h5', mags=[1, 2, 4], hdf5_names=['raw'])
seg_d = load_from_h5py(h5_dir + 'seg.h5', hdf5_names=['seg'])[0]
kd.from_matrix_to_cubes(offset, mags=[1, 2, 4], data=seg_d,
fast_downsampling=True, as_raw=False)
kd_mi = knossosdataset.KnossosDataset()
kd_mi.initialize_from_matrix(global_params.config.kd_mi_path, scale, experiment_name,
offset=offset, boundary=bd, fast_downsampling=True,
data_path=h5_dir + 'mi.h5', mags=[1, 2], hdf5_names=['mi'])
kd_vc = knossosdataset.KnossosDataset()
kd_vc.initialize_from_matrix(global_params.config.kd_vc_path, scale, experiment_name,
def run_create_sds(chunk_size=None,
n_folders_fs=10000,
max_n_jobs=None,
generate_sv_meshes=False,
load_from_kd_overlaycubes=False,
cube_of_interest_bb=None):
"""
Parameters
----------
chunk_size :
max_n_jobs : int
n_folders_fs :
generate_sv_meshes :
load_from_kd_overlaycubes : bool
Load prob/seg data from overlaycubes instead of raw cubes.
cube_of_interest_bb : Tuple[np.ndarray]
Defines the bounding box of the cube to process. By default this is
set to (np.zoers(3); kd.boundary).
Returns
-------
"""
if chunk_size is None:
chunk_size = [512, 512, 512]
if max_n_jobs is None:
max_n_jobs = global_params.NCORE_TOTAL * 3
log = initialize_logging('create_sds',
global_params.config.working_dir + '/logs/',
overwrite=False)
# Sets initial values of object
kd = kd_factory(global_params.config.kd_seg_path)
if cube_of_interest_bb is None:
cube_of_interest_bb = [np.zeros(3, dtype=np.int), kd.boundary]
size = cube_of_interest_bb[1] - cube_of_interest_bb[0] + 1
offset = cube_of_interest_bb[0]
# TODO: get rid of explicit voxel extraction, all info necessary should be extracted
# at the beginning, e.g. size, bounding box etc and then refactor to only use those cached attributes!
# resulting ChunkDataset, required for SV extraction --
# Object extraction - 2h, the same has to be done for all cell organelles
cd_dir = global_params.config.working_dir + "chunkdatasets/sv/"
# Class that contains a dict of chunks (with coordinates) after initializing it
cd = chunky.ChunkDataset()
cd.initialize(kd,
kd.boundary,
chunk_size,
cd_dir,
box_coords=[0, 0, 0],
fit_box_size=True)
log.info('Generating SegmentationDatasets for cell and cell '
'organelle supervoxels.')
oew.from_ids_to_objects(
cd,
"sv",
overlaydataset_path=global_params.config.kd_seg_path,
n_chunk_jobs=max_n_jobs,
hdf5names=["sv"],
n_max_co_processes=None,
n_folders_fs=n_folders_fs,
use_combined_extraction=True,
size=size,
offset=offset)
# Object Processing -- Perform after mapping to also cache mapping ratios
sd = SegmentationDataset("sv",
working_dir=global_params.config.working_dir)
sd_proc.dataset_analysis(sd, recompute=True, compute_meshprops=False)
log.info("Extracted {} cell SVs. Preparing rendering locations "
"(and meshes if not provided).".format(len(sd.ids)))
start = time.time()
# chunk them
multi_params = chunkify(sd.so_dir_paths, max_n_jobs)
# all other kwargs like obj_type='sv' and version are the current SV SegmentationDataset by default
so_kwargs = dict(working_dir=global_params.config.working_dir,
obj_type='sv')
multi_params = [[par, so_kwargs] for par in multi_params]
if generate_sv_meshes:
_ = qu.QSUB_script(multi_params,
"mesh_caching",
n_max_co_processes=global_params.NCORE_TOTAL)
_ = qu.QSUB_script(multi_params,
"sample_location_caching",
n_max_co_processes=global_params.NCORE_TOTAL)
# recompute=False: only collect new sample_location property
sd_proc.dataset_analysis(sd, compute_meshprops=True, recompute=False)
log.info(
'Finished preparation of cell SVs after {:.0f}s.'.format(time.time() -
start))
# create SegmentationDataset for each cell organelle
for co in global_params.existing_cell_organelles:
start = time.time()
cd_dir = global_params.config.working_dir + "chunkdatasets/{}/".format(
co)
cd.initialize(kd,
kd.boundary,
chunk_size,
cd_dir,
box_coords=[0, 0, 0],
fit_box_size=True)
log.info('Started object extraction of cellular organelles "{}" from '
'{} chunks.'.format(co, len(cd.chunk_dict)))
prob_kd_path_dict = {
co: getattr(global_params.config, 'kd_{}_path'.format(co))
}
# This creates a SegmentationDataset of type 'co'
prob_thresh = global_params.config.entries["Probathresholds"][
co] # get probability threshold
path = "{}/knossosdatasets/{}_seg/".format(
global_params.config.working_dir, co)
target_kd = knossosdataset.KnossosDataset()
target_kd.initialize_without_conf(path,
kd.boundary,
kd.scale,
kd.experiment_name,
mags=[
1,
])
target_kd = knossosdataset.KnossosDataset()
target_kd.initialize_from_knossos_path(path)
oew.from_probabilities_to_objects(
cd,
co, # membrane_kd_path=global_params.config.kd_barrier_path, # TODO: currently does not exist
prob_kd_path_dict=prob_kd_path_dict,
thresholds=[prob_thresh],
workfolder=global_params.config.working_dir,
hdf5names=[co],
n_max_co_processes=None,
target_kd=target_kd,
n_folders_fs=n_folders_fs,
debug=False,
size=size,
offset=offset,
load_from_kd_overlaycubes=load_from_kd_overlaycubes)
sd_co = SegmentationDataset(
obj_type=co, working_dir=global_params.config.working_dir)
# TODO: check if this is faster then the alternative below
sd_proc.dataset_analysis(sd_co,
recompute=True,
compute_meshprops=False)
multi_params = chunkify(sd_co.so_dir_paths, max_n_jobs)
so_kwargs = dict(working_dir=global_params.config.working_dir,
obj_type=co)
multi_params = [[par, so_kwargs] for par in multi_params]
_ = qu.QSUB_script(multi_params,
"mesh_caching",
n_max_co_processes=global_params.NCORE_TOTAL)
sd_proc.dataset_analysis(sd_co,
recompute=False,
compute_meshprops=True)
# # Old alternative, requires much more reads/writes then above solution
# sd_proc.dataset_analysis(sd_co, recompute=True, compute_meshprops=True)
# About 0.2 h per object class
log.info('Started mapping of {} cellular organelles of type "{}" to '
'cell SVs.'.format(len(sd_co.ids), co))
sd_proc.map_objects_to_sv(sd,
co,
global_params.config.kd_seg_path,
n_jobs=max_n_jobs)
log.info('Finished preparation of {} "{}"-SVs after {:.0f}s.'
''.format(len(sd_co.ids), co,
time.time() - start))
def _map_objects_thread(args):
"""Worker of map_objects_to_sv"""
# TODO: this needs to be done densely by matching cell organelle segmentation (see corresponding ChunkDataset which is an intermediate result of 'from_probabilities_to_objects') to SV segmentation
paths = args[0]
obj_type = args[1]
obj_version = args[2]
working_dir = args[3]
kd_path = args[4]
readonly = args[5]
global_params.wd = working_dir
if len(args) > 6:
datatype = args[6]
else:
datatype = np.uint64
kd = knossosdataset.KnossosDataset()
kd.initialize_from_knossos_path(kd_path)
seg_dataset = segmentation.SegmentationDataset(obj_type, version=obj_version,
working_dir=working_dir)
sv_id_dict = {}
for p in paths:
this_attr_dc = AttributeDict(p + "/attr_dict.pkl", read_only=readonly,
disable_locking=True)
this_vx_dc = VoxelStorage(p + "/voxel.pkl", read_only=True,
disable_locking=True)
for so_id in this_vx_dc.keys():
so = seg_dataset.get_segmentation_object(so_id)
so.attr_dict = this_attr_dc[so_id]
so.load_voxels(voxel_dc=this_vx_dc)
if readonly:
if "mapping_ids" in so.attr_dict:
ids = so.attr_dict["mapping_ids"]
id_ratios = so.attr_dict["mapping_ratios"]
for i_id in range(len(ids)):
if ids[i_id] in sv_id_dict:
sv_id_dict[ids[i_id]][so_id] = id_ratios[i_id]
else:
sv_id_dict[ids[i_id]] = {so_id: id_ratios[i_id]}
else:
if np.product(so.shape) > 1e12: # TODO: Seems hacky
continue
vx_list = np.argwhere(so.voxels) + so.bounding_box[0]
try:
id_list = kd.from_overlaycubes_to_list(vx_list, datatype=datatype)
except:
log_proc.error('Could not load overlaycube '
'during object mapping')
continue
ids, id_counts = np.unique(id_list, return_counts=True)
id_ratios = id_counts / float(np.sum(id_counts))
for i_id in range(len(ids)):
if ids[i_id] in sv_id_dict:
sv_id_dict[ids[i_id]][so_id] = id_ratios[i_id]
else:
sv_id_dict[ids[i_id]] = {so_id: id_ratios[i_id]}
so.attr_dict["mapping_ids"] = ids
so.attr_dict["mapping_ratios"] = id_ratios
this_attr_dc[so_id] = so.attr_dict
if not readonly:
this_attr_dc.push()
return sv_id_dict
