def run_matrix_export():
# cache cell attributes
ssd = SuperSegmentationDataset(
working_dir=global_params.config.working_dir)
ssd.save_dataset_deep()
log = initialize_logging('synapse_analysis',
global_params.config.working_dir + '/logs/',
overwrite=True)
sd_syn_ssv = SegmentationDataset(
working_dir=global_params.config.working_dir, obj_type='syn_ssv')
# as an alternative to the skeletons, use vertex predictions or
# sample_locations, ~3.5h @ 300 cpus
# TODO: requires speed-up; one could collect properties only for synapses >
# probability threshold
# synssv_ids = synssv_ids[syn_prob > .5]
# ssv_partners = ssv_partners[syn_prob > .5]
# One could also re-use the cached synssv IDs (computed during mapping of
# synssv to SSVs) -> saves finding SSV ID indices in synapse arrays (->
# slow for many synapses)
cps.collect_properties_from_ssv_partners(global_params.config.working_dir,
debug=True)
#
# collect new object attributes collected above partner axoness, celltypes,
# synapse probabilities etc, no need to compute size/rep_coord etc. ->
# recompute=False
dataset_analysis(sd_syn_ssv, compute_meshprops=False, recompute=False)
log.info('Synapse property collection from SSVs finished.')
# export_matrix
log.info('Exporting connectivity matrix now.')
dest_folder = global_params.config.working_dir + '/connectivity_matrix/'
cps.export_matrix(dest_folder=dest_folder)
log.info('Connectivity matrix was epxorted to "{}".'.format(dest_folder))
def run_matrix_export():
"""
Export the matrix as a ``.csv`` file at the ``connectivity_matrix`` folder
of the currently active working directory.
Also collects the following synapse properties from prior analysis
steps:
* 'partner_axoness': Cell compartment type (axon: 1, dendrite: 0, soma: 2,
en-passant bouton: 3, terminal bouton: 4) of the partner neurons.
* 'partner_spiness': Spine compartment predictions of both neurons.
* 'partner_celltypes': Celltype of the both neurons.
* 'latent_morph': Local morphology embeddings of the pre- and post-
synaptic partners.
Examples:
See :class:`~syconn.reps.segmentation.SegmentationDataset` for examples.
"""
# cache cell attributes
ssd = SuperSegmentationDataset(
working_dir=global_params.config.working_dir)
ssd.save_dataset_deep()
log = initialize_logging('synapse_analysis',
global_params.config.working_dir + '/logs/',
overwrite=True)
sd_syn_ssv = SegmentationDataset(
working_dir=global_params.config.working_dir, obj_type='syn_ssv')
# as an alternative to the skeletons, use vertex predictions or
# sample_locations, ~3.5h @ 300 cpus
# TODO: requires speed-up; one could collect properties only for synapses >
# probability threshold
# synssv_ids = synssv_ids[syn_prob > .5]
# ssv_partners = ssv_partners[syn_prob > .5]
# One could also re-use the cached synssv IDs (computed during mapping of
# synssv to SSVs) -> saves finding SSV ID indices in synapse arrays (->
# slow for many synapses)
cps.collect_properties_from_ssv_partners(global_params.config.working_dir,
debug=True)
#
# collect new object attributes collected above partner axoness, celltypes,
# synapse probabilities etc, no need to compute size/rep_coord etc. ->
# recompute=False
dataset_analysis(sd_syn_ssv, compute_meshprops=False, recompute=False)
log.info('Synapse property collection from SSVs finished.')
# export_matrix
log.info('Exporting connectivity matrix now.')
dest_folder = global_params.config.working_dir + '/connectivity_matrix/'
cps.export_matrix(dest_folder=dest_folder)
log.info('Connectivity matrix was exported to "{}".'.format(dest_folder))
def run_create_neuron_ssd():
"""
Creates SuperSegmentationDataset with `version=0`.
"""
log = initialize_logging('create_neuron_ssd',
global_params.config.working_dir + '/logs/',
overwrite=False)
suffix = global_params.rag_suffix
g_p = "{}/glia/neuron_rag{}.bz2".format(global_params.config.working_dir,
suffix)
rag_g = nx.read_edgelist(g_p, nodetype=np.uint)
# e.g. if rag was not created by glia splitting procedure this filtering is required
ccs = nx.connected_components(rag_g)
cc_dict = {}
for cc in ccs:
cc_arr = np.array(list(cc))
cc_dict[np.min(cc_arr)] = cc_arr
cc_dict_inv = {}
for ssv_id, cc in cc_dict.items():
for sv_id in cc:
cc_dict_inv[sv_id] = ssv_id
log.info('Parsed RAG from {} with {} SSVs and {} SVs.'.format(
g_p, len(cc_dict), len(cc_dict_inv)))
ssd = SuperSegmentationDataset(
working_dir=global_params.config.working_dir,
version='0',
ssd_type="ssv",
sv_mapping=cc_dict_inv)
# create cache-arrays for frequently used attributes
ssd.save_dataset_deep(n_max_co_processes=global_params.NCORE_TOTAL
) # also executes 'ssd.save_dataset_shallow()'
exec_skeleton.run_skeleton_generation()
log.info('Finished SSD initialization. Starting cellular '
'organelle mapping.')
# map cellular organelles to SSVs
# TODO: sort by SSV size (descending)
ssd_proc.aggregate_segmentation_object_mappings(
ssd, global_params.existing_cell_organelles)
ssd_proc.apply_mapping_decisions(ssd,
global_params.existing_cell_organelles)
log.info('Finished mapping of cellular organelles to SSVs. '
'Writing individual SSV graphs.')
# Write SSV RAGs
pbar = tqdm.tqdm(total=len(ssd.ssv_ids), mininterval=0.5)
for ssv in ssd.ssvs:
# get all nodes in CC of this SSV
if len(cc_dict[
ssv.id]) > 1: # CCs with 1 node do not exist in the global RAG
n_list = nx.node_connected_component(rag_g, ssv.id)
# get SSV RAG as subgraph
ssv_rag = nx.subgraph(rag_g, n_list)
else:
ssv_rag = nx.Graph()
# ssv.id is the minimal SV ID, and therefore the only SV in this case
ssv_rag.add_edge(ssv.id, ssv.id)
nx.write_edgelist(ssv_rag, ssv.edgelist_path)
pbar.update(1)
pbar.close()
log.info('Finished saving individual SSV RAGs.')
def run_create_neuron_ssd(prior_glia_removal=True):
"""
Creates SuperSegmentationDataset with version 0.
Parameters
----------
prior_glia_removal : bool
If False, will apply filtering to create SSO objects above minimum size, see global_params.min_cc_size_ssv
and cache SV sample locations.
Returns
-------
"""
log = initialize_logging('create_neuron_ssd',
global_params.config.working_dir + '/logs/',
overwrite=False)
suffix = global_params.rag_suffix
# TODO: the following paths currently require prior glia-splitting
g_p = "{}/glia/neuron_rag{}.bz2".format(global_params.config.working_dir,
suffix)
rag_g = nx.read_edgelist(g_p, nodetype=np.uint)
# e.g. if rag was not created by glia splitting procedure this filtering is required
if not prior_glia_removal:
sd = SegmentationDataset("sv",
working_dir=global_params.config.working_dir)
sv_size_dict = {}
bbs = sd.load_cached_data('bounding_box') * sd.scaling
for ii in range(len(sd.ids)):
sv_size_dict[sd.ids[ii]] = bbs[ii]
ccsize_dict = create_ccsize_dict(rag_g, sv_size_dict)
log.debug("Finished preparation of SSV size dictionary based "
"on bounding box diagional of corresponding SVs.")
before_cnt = len(rag_g.nodes())
for ix in list(rag_g.nodes()):
if ccsize_dict[ix] < global_params.min_cc_size_ssv:
rag_g.remove_node(ix)
log.debug("Removed %d neuron CCs because of size." %
(before_cnt - len(rag_g.nodes())))
ccs = nx.connected_components(rag_g)
cc_dict = {}
for cc in ccs:
cc_arr = np.array(list(cc))
cc_dict[np.min(cc_arr)] = cc_arr
cc_dict_inv = {}
for ssv_id, cc in cc_dict.items():
for sv_id in cc:
cc_dict_inv[sv_id] = ssv_id
log.info('Parsed RAG from {} with {} SSVs and {} SVs.'.format(
g_p, len(cc_dict), len(cc_dict_inv)))
ssd = SuperSegmentationDataset(
working_dir=global_params.config.working_dir,
version='0',
ssd_type="ssv",
sv_mapping=cc_dict_inv)
# create cache-arrays for frequently used attributes
ssd.save_dataset_deep(n_max_co_processes=global_params.NCORE_TOTAL
) # also executes 'ssd.save_dataset_shallow()'
exec_skeleton.run_skeleton_generation()
log.info('Finished SSD initialization. Starting cellular '
'organelle mapping.')
# map cellular organelles to SSVs
# TODO: increase number of jobs in the next two QSUB submissions and sort by SSV size (descending)
ssd_proc.aggregate_segmentation_object_mappings(
ssd, global_params.existing_cell_organelles, qsub_pe="openmp")
ssd_proc.apply_mapping_decisions(ssd,
global_params.existing_cell_organelles,
qsub_pe="openmp")
log.info('Finished mapping of cellular organelles to SSVs. '
'Writing individual SSV graphs.')
# Write SSV RAGs
pbar = tqdm.tqdm(total=len(ssd.ssv_ids), mininterval=0.5)
for ssv in ssd.ssvs:
# get all nodes in CC of this SSV
if len(cc_dict[
ssv.id]) > 1: # CCs with 1 node do not exist in the global RAG
n_list = nx.node_connected_component(rag_g, ssv.id)
# get SSV RAG as subgraph
ssv_rag = nx.subgraph(rag_g, n_list)
else:
ssv_rag = nx.Graph()
# ssv.id is the minimal SV ID, and therefore the only SV in this case
ssv_rag.add_edge(ssv.id, ssv.id)
nx.write_edgelist(ssv_rag, ssv.edgelist_path)
pbar.update(1)
pbar.close()
log.info('Finished saving individual SSV RAGs.')