def test_multiprocessed_vs_serial_rendering():
# TODO: use example data and improve logging, see test_backend.py
working_dir = "/wholebrain/scratch/areaxfs3/"
render_indexview = True
ssc = SuperSegmentationDataset(working_dir)
ssv = ssc.get_super_segmentation_object(29753344)
ssv.nb_cpus = cpu_count()
exlocs = np.concatenate(ssv.sample_locations())
exlocs = exlocs[:1000]
views = render_sso_coords_multiprocessing(
ssv,
working_dir,
rendering_locations=exlocs,
render_indexviews=render_indexview,
n_jobs=10,
verbose=True)
# overwrite any precomputed caches by re-initialization of SSV
ssv = ssc.get_super_segmentation_object(29753344)
ssv.nb_cpus = cpu_count()
exlocs = np.concatenate(ssv.sample_locations())
exlocs = exlocs[:1000]
if render_indexview:
views2 = render_sso_coords_index_views(ssv, exlocs, verbose=True)
else:
views2 = render_sso_coords(ssv, exlocs, verbose=True)
print('Fraction of different index values in index-views: {:.4f}'
''.format(np.sum(views != views2) / np.prod(views.shape)))
assert np.all(views == views2)
def preds2kzip(pred_folder: str, out_path: str, ssd_path: str, col_lookup: dict,
label_mappings: Optional[List[Tuple[int, int]]] = None):
pred_folder = os.path.expanduser(pred_folder)
out_path = os.path.expanduser(out_path)
if not os.path.exists(out_path):
os.makedirs(out_path)
files = glob.glob(pred_folder + '*_preds.pkl')
ssd = SuperSegmentationDataset(ssd_path)
for file in tqdm(files):
hc_voxeled = preds2hc(file)
sso_id = int(re.findall(r"/sso_(\d+).", file)[0])
sso = ssd.get_super_segmentation_object(sso_id)
verts = sso.mesh[1].reshape(-1, 3)
hc = HybridCloud(nodes=hc_voxeled.nodes, edges=hc_voxeled.edges, node_labels=hc_voxeled.node_labels,
pred_node_labels=hc_voxeled.pred_node_labels, vertices=verts)
hc.nodel2vertl()
hc.prednodel2predvertl()
if label_mappings is not None:
hc.map_labels(label_mappings)
cols = np.array([col_lookup[el] for el in hc.pred_labels.squeeze()], dtype=np.uint8)
sso.mesh2kzip(out_path + f'p_{sso_id}.k.zip', ext_color=cols)
cols = np.array([col_lookup[el] for el in hc.labels.squeeze()], dtype=np.uint8)
sso.mesh2kzip(out_path + f't_{sso_id}.k.zip', ext_color=cols)
comments = list(hc.pred_node_labels.reshape(-1))
for node in range(len(hc.nodes)):
if hc.pred_node_labels[node] != hc.node_labels[node] and hc.pred_node_labels[node] != -1:
comments[node] = 'e' + str(comments[node])
sso.save_skeleton_to_kzip(out_path + f'p_{sso_id}.k.zip', comments=comments)
comments = hc.node_labels.reshape(-1)
sso.save_skeleton_to_kzip(out_path + f't_{sso_id}.k.zip', comments=comments)
def get_sso_specs(set_path: str, out_path: str, ssd: SuperSegmentationDataset):
set_path = os.path.expanduser(set_path)
out_path = os.path.expanduser(out_path)
files = glob.glob(set_path + '*.pkl')
total_edge_length = 0
total_voxel_size = 0
for file in tqdm(files):
sso_id = int(re.findall(r"/sso_(\d+).", file)[0])
sso = ssd.get_super_segmentation_object(sso_id)
total_edge_length += sso.total_edge_length()
total_voxel_size += sso.size
info = f'{sso_id}:\nskeleton path length:\t{sso.total_edge_length()}\nvoxel size:\t{sso.size}\n\n'
with open(out_path, 'a') as f:
f.write(info)
f.close()
with open(out_path, 'a') as f:
f.write(
f'total edge length: {total_edge_length}\ntotal voxel size: {total_voxel_size}'
)
f.close()
def GT_generation(kzip_paths,
ssd_version,
gt_type,
nb_views,
dest_dir=None,
n_voting=40,
ws=(256, 128),
comp_window=8e3):
"""
Generates a .npy GT file from all kzip paths.
Parameters
----------
kzip_paths : List[str]
gt_type : str
n_voting : int
Number of collected nodes during BFS for majority vote (label smoothing)
Returns
-------
"""
sso_ids = [
int(re.findall("/(\d+).", kzip_path)[0]) for kzip_path in kzip_paths
]
ssd = SuperSegmentationDataset()
if not np.all([
ssv.lookup_in_attribute_dict("size") is not None
for ssv in ssd.get_super_segmentation_object(sso_ids)
]):
print("Not all SSV IDs are part of " \
"the current SSD. IDs: {}".format([sso_id for sso_id in sso_ids if sso_id not in
ssd.ssv_ids]))
kzip_paths = np.array(kzip_paths)[np.array([
ssv.lookup_in_attribute_dict("size") is not None
for ssv in ssd.get_super_segmentation_object(sso_ids)
])]
print("Ignoring missing IDs. Using {} k.zip files for GT "
"generation,".format(len(kzip_paths)))
if dest_dir is None:
dest_dir = os.path.expanduser("~/{}_semseg/".format(gt_type))
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
dest_p_cache = "{}/cache_{}votes/".format(dest_dir, n_voting)
params = [(p, ssd_version, gt_type, n_voting, nb_views, ws, comp_window,
dest_p_cache) for p in kzip_paths]
if not os.path.isdir(dest_p_cache):
os.makedirs(dest_p_cache)
start_multiprocess_imap(gt_generation_helper,
params,
nb_cpus=cpu_count(),
debug=False)
# TODO: in case GT is too big to hold all views in memory
# if gt_type == 'axgt':
# return
# Create Dataset splits for training, validation and test
all_raw_views = []
all_label_views = []
# all_index_views = [] # Removed index views
print("Writing views.")
for ii in range(len(kzip_paths)):
sso_id = int(re.findall("/(\d+).", kzip_paths[ii])[0])
dest_p = "{}/{}/".format(dest_p_cache, sso_id)
raw_v = np.load(dest_p + "raw.npy")
label_v = np.load(dest_p + "label.npy")
# index_v = np.load(dest_p + "index.npy") # Removed index views
all_raw_views.append(raw_v)
all_label_views.append(label_v)
# all_index_views.append(index_v) # Removed index views
all_raw_views = np.concatenate(all_raw_views)
all_label_views = np.concatenate(all_label_views)
# all_index_views = np.concatenate(all_index_views) # Removed index views
print("{} view locations collected. Shuffling views.".format(
len(all_label_views)))
np.random.seed(0)
ixs = np.arange(len(all_raw_views))
np.random.shuffle(ixs)
all_raw_views = all_raw_views[ixs]
all_label_views = all_label_views[ixs]
# all_index_views = all_index_views[ixs] # Removed index views
print("Swapping axes.")
all_raw_views = all_raw_views.swapaxes(2, 1)
all_label_views = all_label_views.swapaxes(2, 1)
# all_index_views = all_index_views.swapaxes(2, 1) # Removed index views
print("Reshaping arrays.")
all_raw_views = all_raw_views.reshape((-1, 4, ws[1], ws[0]))
all_label_views = all_label_views.reshape((-1, 1, ws[1], ws[0]))
# # all_index_views = all_index_views.reshape((-1, 1, 128, 256)) # Removed index views
# # all_raw_views = np.concatenate([all_raw_views, all_index_views], axis=1) # Removed index views
raw_train, raw_valid, label_train, label_valid = train_test_split(
all_raw_views, all_label_views, train_size=0.9, shuffle=False)
# # raw_valid, raw_test, label_valid, label_test = train_test_split(raw_other, label_other, train_size=0.5, shuffle=False) # Removed index views
print("Writing h5 files.")
os.makedirs(dest_dir, exist_ok=True)
# chunk output data
for ii in range(5):
save_to_h5py([raw_train[ii::5]],
dest_dir + "/raw_train_{}.h5".format(ii), ["raw"])
save_to_h5py([raw_valid[ii::5]],
dest_dir + "/raw_valid_{}.h5".format(ii), ["raw"])
# save_to_h5py([raw_test], dest_dir + "/raw_test.h5",
# ["raw"]) # Removed index views
save_to_h5py([label_train[ii::5]],
dest_dir + "/label_train_{}.h5".format(ii), ["label"])
save_to_h5py([label_valid[ii::5]],
dest_dir + "/label_valid_{}.h5".format(ii), ["label"])
def worker_split(id_queue: Queue,
chunk_queue: Queue,
ssd: SuperSegmentationDataset,
ctx: int,
base_node_dst: int,
parts: Dict[str, List[int]],
labels_itf: str,
label_mappings: List[Tuple[int, int]],
split_jitter: int = 0):
"""
Args:
id_queue: Input queue with cell ids.
chunk_queue: Output queue with cell chunks.
ssd: SuperSegmentationDataset which contains the cells to which the chunkhandler should get applied.
ctx: Context size for splitting.
base_node_dst: Distance between base nodes. Corresponds to redundancy / number of chunks per cell.
parts: Information about cell surface and organelles, Tuples like (voxel_param, feature) keyed by identifier
compatible with syconn (e.g. 'sv' or 'mi').
labels_itf: Label identifier for existing label predictions within the sso objects of the ssd dataset.
label_mappings: Tuples where label at index 0 should get mapped to label at index 1.
split_jitter: Derivation from context size during splitting.
"""
while True:
if not id_queue.empty():
ssv_id = id_queue.get()
sso = ssd.get_super_segmentation_object(ssv_id)
vert_dc = {}
label_dc = {}
encoding = {}
offset = 0
obj_bounds = {}
for ix, k in enumerate(parts):
pcd = o3d.geometry.PointCloud()
verts = sso.load_mesh(k)[1].reshape(-1, 3)
pcd.points = o3d.utility.Vector3dVector(verts)
pcd, idcs = pcd.voxel_down_sample_and_trace(
parts[k][0], pcd.get_min_bound(), pcd.get_max_bound())
idcs = np.max(idcs, axis=1)
vert_dc[k] = np.asarray(pcd.points)
obj_bounds[k] = [offset, offset + len(pcd.points)]
offset += len(pcd.points)
if k == 'sv':
# prepare mask for filtering background / unpredicted points
mask = None
if labels_itf == 'axoness':
# 0: dendrite, 1: axon, 2: soma, 3: bouton, 4: terminal, 5/6: background/unpredicted
labels_total = sso.label_dict()[labels_itf][idcs]
mask = labels_total < 5
labels_total = labels_total[mask]
elif labels_itf == 'spiness':
# 1: head, 0: neck, 2: shaft, 3: other, 4/5: background/unpredicted
labels_total = sso.label_dict()['axoness'][idcs]
spiness = sso.label_dict()['spiness'][idcs]
mask = np.logical_not(
np.logical_or(
labels_total > 4,
np.logical_and(labels_total == 0,
spiness > 3)))
labels_total = labels_total[mask]
spiness = spiness[mask]
labels_total[labels_total != 0] = 3
labels_total[labels_total == 0] = spiness[labels_total
== 0]
else:
labels_total = sso.label_dict()[labels_itf][idcs]
mask = np.ones(len(labels_total)).astype(bool)
labels = labels_total
vert_dc[k] = vert_dc[k][mask]
else:
labels = np.ones(len(vert_dc[k])) + ix + labels_total.max()
encoding[k] = ix + 1 + labels_total.max()
label_dc[k] = labels
sample_feats = np.concatenate([[parts[k][1]] * len(vert_dc[k])
for k in parts]).reshape(-1, 1)
sample_feats = label_binarize(sample_feats,
classes=np.arange(len(parts)))
sample_pts = np.concatenate([vert_dc[k] for k in parts])
sample_labels = np.concatenate([label_dc[k] for k in parts])
# mark cellular organelles to be excluded from loss calculation - see torchhandler for use of no_pred
no_pred = list(encoding.keys())
if not sso.load_skeleton():
raise ValueError(f'Couldnt find skeleton of {sso}')
nodes, edges = sso.skeleton['nodes'] * sso.scaling, sso.skeleton[
'edges']
hc = HybridCloud(nodes,
edges,
vertices=sample_pts,
features=sample_feats,
obj_bounds=obj_bounds,
no_pred=no_pred,
labels=sample_labels,
encoding=encoding)
if label_mappings is not None:
hc.map_labels(label_mappings)
_ = hc.verts2node
jitter = random.randint(0, split_jitter)
node_arrs, source_nodes = splitting.split_single(
hc, ctx + jitter, base_node_dst)
for ix, node_arr in enumerate(node_arrs):
sample, _ = objects.extract_cloud_subset(hc, node_arr)
chunk_queue.put(sample)
else:
time.sleep(0.5)
def __init__(self):
ssc = SuperSegmentationDataset('/wholebrain/scratch/areaxfs3/')
ssv = ssc.get_super_segmentation_object(29753344)
#self.ssc = ssc.get_super_segmentation_object(29753344)
exloc = np.array([5602, 4173, 4474]) * ssv.scaling
self.exlocs = np.concatenate(ssv.sample_locations())
def __init__(self):
ssc = SuperSegmentationDataset('/wholebrain/scratch/areaxfs3/')
ssv = ssc.get_super_segmentation_object(29753344)
#self.ssc = ssc.get_super_segmentation_object(29753344)
exloc = np.array([5602, 4173, 4474]) * ssv.scaling
self.exlocs = np.concatenate(ssv.sample_locations())
if __name__ == '__main__':
# TODO: use toy data and improve logging, see test_backend.py
working_dir = "/wholebrain/scratch/areaxfs3/"
render_indexview = True
now = time.time()
ssc = SuperSegmentationDataset(working_dir)
ssv = ssc.get_super_segmentation_object(29753344)
exlocs = np.concatenate(ssv.sample_locations())
exlocs = exlocs[::30]
print("Example location array:", exlocs.shape)
print(working_dir)
now2 = time.time()
print("time for reading data")
print(now2 - now)
"""
i = 0
exlocs = chunkify_successive(exlocs, 10)
params = []
for ex in exlocs:
params.append([exlocs, i])
i=i+1
print(i)
from syconn import global_params
path_storage_file = sys.argv[1]
path_out_file = sys.argv[2]
with open(path_storage_file, 'rb') as f:
args = []
while True:
try:
args.append(pkl.load(f))
except EOFError:
break
ssv_ids = args[0]
version = args[1]
version_dict = args[2]
working_dir = args[3]
ssd = SuperSegmentationDataset(working_dir=working_dir,
version=version,
version_dict=version_dict)
for ssv in ssd.get_super_segmentation_object(ssv_ids):
ssv.load_skeleton()
ssv.skeleton["myelin"] = map_myelin2coords(ssv.skeleton["nodes"], mag=4)
majorityvote_skeleton_property(
ssv, prop_key='myelin', max_dist=global_params.DIST_AXONESS_AVERAGING)
ssv.save_skeleton()
with open(path_out_file, "wb") as f:
pkl.dump("0", f)
# test prediction, copy trained models, see paths to 'models' folder defined in global_params.config
from syconn.handler.prediction import get_celltype_model_large_e3, \
get_tripletnet_model_large_e3, get_celltype_model_e3
from syconn.reps.super_segmentation import SuperSegmentationDataset, SuperSegmentationObject
import tqdm
pred_key_appendix1 = "test_pred_v1_large"
pred_key_appendix2 = ""
ssd = SuperSegmentationDataset(working_dir=WD, version="ctgt_v2")
# #
# # prediction
pbar = tqdm.tqdm(total=len(ssv_ids))
m = get_celltype_model_e3()
m_large = get_celltype_model_large_e3()
m_tnet = get_tripletnet_model_large_e3()
for ssv_id in ssv_ids:
ssv = ssd.get_super_segmentation_object(ssv_id)
ssv._view_caching = True
# ssv.predict_celltype_cnn(model=m_large, pred_key_appendix=pred_key_appendix1,
# model_tnet=m_tnet)
ssv.predict_celltype_cnn(model=m,
pred_key_appendix=pred_key_appendix1,
largeFoV=False,
view_props={"overwrite": False})
pbar.update()
pbar.close()
# analysis
from syconn.proc.stats import cluster_summary, projection_tSNE, model_performance
gt_l = []
pred_l = []
pred_proba = []
pred_l_large = []
def run_neuron_rendering(max_n_jobs=2000):
log = initialize_logging('neuron_view_rendering',
global_params.config.working_dir + '/logs/')
# TODO: currently working directory has to be set globally in global_params
# and is not adjustable here because all qsub jobs will start a script
# referring to 'global_params.config.working_dir'
# view rendering prior to glia removal, choose SSD accordingly
ssd = SuperSegmentationDataset(
working_dir=global_params.config.working_dir)
# TODO: use actual size criteria, e.g. number of sampling locations
nb_svs_per_ssv = np.array(
[len(ssd.mapping_dict[ssv_id]) for ssv_id in ssd.ssv_ids])
# render normal size SSVs
size_mask = nb_svs_per_ssv <= global_params.RENDERING_MAX_NB_SV
multi_params = ssd.ssv_ids[size_mask]
# TODO: move from osmesa to egl, egl rendering worker (10 cpus, 1 gpu) then should utilize more threads for bigger
# SSVs, and run more SSVs in parallel if they are small
# sort ssv ids according to their number of SVs (descending)
ordering = np.argsort(nb_svs_per_ssv[size_mask])
multi_params = multi_params[ordering[::-1]]
multi_params = chunkify(multi_params, max_n_jobs)
# list of SSV IDs and SSD parameters need to be given to a single QSUB job
multi_params = [(ixs, global_params.config.working_dir)
for ixs in multi_params]
log.info('Started rendering of {} SSVs. '.format(np.sum(size_mask)))
if np.sum(~size_mask) > 0:
log.info('{} huge SSVs will be rendered afterwards using the whole'
' cluster.'.format(np.sum(~size_mask)))
# generic
# TODO: switch n_cores to `global_params.NGPUS_TOTAL` as soon as EGL ressource allocation works!
if global_params.PYOPENGL_PLATFORM == 'osmesa': # utilize all CPUs
path_to_out = qu.QSUB_script(
multi_params,
"render_views",
n_max_co_processes=global_params.NCORE_TOTAL)
elif global_params.PYOPENGL_PLATFORM == 'egl': # utilize 1 GPU per task
# TODO: use render_views_egl script
path_to_out = qu.QSUB_script(
multi_params,
"render_views",
n_max_co_processes=global_params.NNODES_TOTAL,
additional_flags="--gres=gpu:2",
n_cores=global_params.NCORES_PER_NODE)
else:
raise RuntimeError('Specified OpenGL platform "{}" not supported.'
''.format(global_params.PYOPENGL_PLATFORM))
if np.sum(~size_mask) > 0:
log.info('Finished rendering of {}/{} SSVs.'.format(
len(ordering), len(nb_svs_per_ssv)))
# identify huge SSVs and process them individually on whole cluster
big_ssv = ssd.ssv_ids[~size_mask]
for kk, ssv_id in enumerate(big_ssv):
ssv = ssd.get_super_segmentation_object(ssv_id)
log.info(
"Processing SSV [{}/{}] with {} SVs on whole cluster.".format(
kk + 1, len(big_ssv), len(ssv.sv_ids)))
ssv.render_views(add_cellobjects=True,
cellobjects_only=False,
woglia=True,
qsub_pe="openmp",
overwrite=True,
qsub_co_jobs=global_params.NCORE_TOTAL,
skip_indexviews=False,
resume_job=False)
log.info('Finished rendering of all SSVs. Checking completeness.')
res = find_incomplete_ssv_views(ssd,
woglia=True,
n_cores=global_params.NCORES_PER_NODE)
if len(res) != 0:
msg = "Not all SSVs were rendered completely! Missing:\n{}".format(res)
log.error(msg)
raise RuntimeError(msg)
else:
log.info('Success.')
if __name__ == '__main__':
# TODO: use toy data and improve logging, see test_backend.py
working_dir = "/wholebrain/scratch/areaxfs3/"
location_array = [
29753344, 29833344, 30393344, 30493344, 32393344, 34493344, 39553344,
42173344
]
ssv_array = []
length_array = []
render_indexview = True
now = time.time()
ssc = SuperSegmentationDataset(working_dir)
ix = 0
for loc in location_array:
ssv_array.append(ssc.get_super_segmentation_object(loc))
exlocs = np.concatenate(ssv_array[ix].sample_locations())
length_array.append(len(exlocs))
ix = ix + 1
#exlocs = exlocs[::30]
print("Example location array:", exlocs.shape)
print(working_dir)
now2 = time.time()
print("time for reading data")
print(now2 - now)
array = [19, 2, 31, 45, 6, 11, 121, 27]
index_array = sorting_in_descending_order(array)
print(array)
print(index_array)
index_array = sort_ssv_descending(ssv_array)
except ImportError:
import pickle as pkl
from syconn.reps.super_segmentation_helper import create_sso_skeletons_wrapper
from syconn.reps.super_segmentation import SuperSegmentationDataset
path_storage_file = sys.argv[1]
path_out_file = sys.argv[2]
with open(path_storage_file, 'rb') as f:
args = []
while True:
try:
args.append(pkl.load(f))
except EOFError:
break
ssv_ids = args[0]
version = args[1]
version_dict = args[2]
working_dir = args[3]
map_myelin = args[4]
ssd = SuperSegmentationDataset(working_dir=working_dir,
version=version,
version_dict=version_dict)
ssvs = ssd.get_super_segmentation_object(ssv_ids)
create_sso_skeletons_wrapper(ssvs, map_myelin=map_myelin)
with open(path_out_file, "wb") as f:
pkl.dump("0", f)