Ejemplo n.º 1
0
def test_multiprocessed_vs_serial_rendering():
    # TODO: use example data and improve logging, see test_backend.py
    working_dir = "/wholebrain/scratch/areaxfs3/"
    render_indexview = True

    ssc = SuperSegmentationDataset(working_dir)
    ssv = ssc.get_super_segmentation_object(29753344)
    ssv.nb_cpus = cpu_count()
    exlocs = np.concatenate(ssv.sample_locations())
    exlocs = exlocs[:1000]
    views = render_sso_coords_multiprocessing(
        ssv,
        working_dir,
        rendering_locations=exlocs,
        render_indexviews=render_indexview,
        n_jobs=10,
        verbose=True)

    # overwrite any precomputed caches by re-initialization of SSV
    ssv = ssc.get_super_segmentation_object(29753344)
    ssv.nb_cpus = cpu_count()
    exlocs = np.concatenate(ssv.sample_locations())
    exlocs = exlocs[:1000]
    if render_indexview:
        views2 = render_sso_coords_index_views(ssv, exlocs, verbose=True)
    else:
        views2 = render_sso_coords(ssv, exlocs, verbose=True)

    print('Fraction of different index values in index-views: {:.4f}'
          ''.format(np.sum(views != views2) / np.prod(views.shape)))
    assert np.all(views == views2)
Ejemplo n.º 2
0
def preds2kzip(pred_folder: str, out_path: str, ssd_path: str, col_lookup: dict,
               label_mappings: Optional[List[Tuple[int, int]]] = None):
    pred_folder = os.path.expanduser(pred_folder)
    out_path = os.path.expanduser(out_path)
    if not os.path.exists(out_path):
        os.makedirs(out_path)
    files = glob.glob(pred_folder + '*_preds.pkl')
    ssd = SuperSegmentationDataset(ssd_path)
    for file in tqdm(files):
        hc_voxeled = preds2hc(file)
        sso_id = int(re.findall(r"/sso_(\d+).", file)[0])
        sso = ssd.get_super_segmentation_object(sso_id)

        verts = sso.mesh[1].reshape(-1, 3)
        hc = HybridCloud(nodes=hc_voxeled.nodes, edges=hc_voxeled.edges, node_labels=hc_voxeled.node_labels,
                         pred_node_labels=hc_voxeled.pred_node_labels, vertices=verts)
        hc.nodel2vertl()
        hc.prednodel2predvertl()
        if label_mappings is not None:
            hc.map_labels(label_mappings)

        cols = np.array([col_lookup[el] for el in hc.pred_labels.squeeze()], dtype=np.uint8)
        sso.mesh2kzip(out_path + f'p_{sso_id}.k.zip', ext_color=cols)
        cols = np.array([col_lookup[el] for el in hc.labels.squeeze()], dtype=np.uint8)
        sso.mesh2kzip(out_path + f't_{sso_id}.k.zip', ext_color=cols)

        comments = list(hc.pred_node_labels.reshape(-1))
        for node in range(len(hc.nodes)):
            if hc.pred_node_labels[node] != hc.node_labels[node] and hc.pred_node_labels[node] != -1:
                comments[node] = 'e' + str(comments[node])
        sso.save_skeleton_to_kzip(out_path + f'p_{sso_id}.k.zip', comments=comments)
        comments = hc.node_labels.reshape(-1)
        sso.save_skeleton_to_kzip(out_path + f't_{sso_id}.k.zip', comments=comments)
Ejemplo n.º 3
0
def get_sso_specs(set_path: str, out_path: str, ssd: SuperSegmentationDataset):
    set_path = os.path.expanduser(set_path)
    out_path = os.path.expanduser(out_path)
    files = glob.glob(set_path + '*.pkl')
    total_edge_length = 0
    total_voxel_size = 0
    for file in tqdm(files):
        sso_id = int(re.findall(r"/sso_(\d+).", file)[0])
        sso = ssd.get_super_segmentation_object(sso_id)
        total_edge_length += sso.total_edge_length()
        total_voxel_size += sso.size
        info = f'{sso_id}:\nskeleton path length:\t{sso.total_edge_length()}\nvoxel size:\t{sso.size}\n\n'
        with open(out_path, 'a') as f:
            f.write(info)
        f.close()
    with open(out_path, 'a') as f:
        f.write(
            f'total edge length: {total_edge_length}\ntotal voxel size: {total_voxel_size}'
        )
    f.close()
Ejemplo n.º 4
0
def GT_generation(kzip_paths,
                  ssd_version,
                  gt_type,
                  nb_views,
                  dest_dir=None,
                  n_voting=40,
                  ws=(256, 128),
                  comp_window=8e3):
    """
    Generates a .npy GT file from all kzip paths.

    Parameters
    ----------
    kzip_paths : List[str]
    gt_type : str
    n_voting : int
        Number of collected nodes during BFS for majority vote (label smoothing)
    Returns
    -------

    """
    sso_ids = [
        int(re.findall("/(\d+).", kzip_path)[0]) for kzip_path in kzip_paths
    ]
    ssd = SuperSegmentationDataset()
    if not np.all([
            ssv.lookup_in_attribute_dict("size") is not None
            for ssv in ssd.get_super_segmentation_object(sso_ids)
    ]):
        print("Not all SSV IDs are part of " \
            "the current SSD. IDs: {}".format([sso_id for sso_id in sso_ids if sso_id not in
                                               ssd.ssv_ids]))
        kzip_paths = np.array(kzip_paths)[np.array([
            ssv.lookup_in_attribute_dict("size") is not None
            for ssv in ssd.get_super_segmentation_object(sso_ids)
        ])]
        print("Ignoring missing IDs. Using {} k.zip files for GT "
              "generation,".format(len(kzip_paths)))
    if dest_dir is None:
        dest_dir = os.path.expanduser("~/{}_semseg/".format(gt_type))
    if not os.path.isdir(dest_dir):
        os.makedirs(dest_dir)
    dest_p_cache = "{}/cache_{}votes/".format(dest_dir, n_voting)
    params = [(p, ssd_version, gt_type, n_voting, nb_views, ws, comp_window,
               dest_p_cache) for p in kzip_paths]
    if not os.path.isdir(dest_p_cache):
        os.makedirs(dest_p_cache)
    start_multiprocess_imap(gt_generation_helper,
                            params,
                            nb_cpus=cpu_count(),
                            debug=False)
    # TODO: in case GT is too big to hold all views in memory
    # if gt_type == 'axgt':
    #     return
    # Create Dataset splits for training, validation and test
    all_raw_views = []
    all_label_views = []
    # all_index_views = []  # Removed index views
    print("Writing views.")
    for ii in range(len(kzip_paths)):
        sso_id = int(re.findall("/(\d+).", kzip_paths[ii])[0])
        dest_p = "{}/{}/".format(dest_p_cache, sso_id)
        raw_v = np.load(dest_p + "raw.npy")
        label_v = np.load(dest_p + "label.npy")
        # index_v = np.load(dest_p + "index.npy")  # Removed index views
        all_raw_views.append(raw_v)
        all_label_views.append(label_v)
        # all_index_views.append(index_v)  # Removed index views
    all_raw_views = np.concatenate(all_raw_views)
    all_label_views = np.concatenate(all_label_views)
    # all_index_views = np.concatenate(all_index_views)  # Removed index views
    print("{} view locations collected. Shuffling views.".format(
        len(all_label_views)))
    np.random.seed(0)
    ixs = np.arange(len(all_raw_views))
    np.random.shuffle(ixs)
    all_raw_views = all_raw_views[ixs]
    all_label_views = all_label_views[ixs]
    # all_index_views = all_index_views[ixs]  # Removed index views
    print("Swapping axes.")
    all_raw_views = all_raw_views.swapaxes(2, 1)
    all_label_views = all_label_views.swapaxes(2, 1)
    # all_index_views = all_index_views.swapaxes(2, 1)  # Removed index views
    print("Reshaping arrays.")
    all_raw_views = all_raw_views.reshape((-1, 4, ws[1], ws[0]))
    all_label_views = all_label_views.reshape((-1, 1, ws[1], ws[0]))
    # # all_index_views = all_index_views.reshape((-1, 1, 128, 256))  # Removed index views
    # # all_raw_views = np.concatenate([all_raw_views, all_index_views], axis=1)  # Removed index views
    raw_train, raw_valid, label_train, label_valid = train_test_split(
        all_raw_views, all_label_views, train_size=0.9, shuffle=False)
    # # raw_valid, raw_test, label_valid, label_test = train_test_split(raw_other, label_other, train_size=0.5, shuffle=False)  # Removed index views
    print("Writing h5 files.")
    os.makedirs(dest_dir, exist_ok=True)
    # chunk output data
    for ii in range(5):
        save_to_h5py([raw_train[ii::5]],
                     dest_dir + "/raw_train_{}.h5".format(ii), ["raw"])
        save_to_h5py([raw_valid[ii::5]],
                     dest_dir + "/raw_valid_{}.h5".format(ii), ["raw"])
        # save_to_h5py([raw_test], dest_dir + "/raw_test.h5",
        # ["raw"])  # Removed index views
        save_to_h5py([label_train[ii::5]],
                     dest_dir + "/label_train_{}.h5".format(ii), ["label"])
        save_to_h5py([label_valid[ii::5]],
                     dest_dir + "/label_valid_{}.h5".format(ii), ["label"])
Ejemplo n.º 5
0
def worker_split(id_queue: Queue,
                 chunk_queue: Queue,
                 ssd: SuperSegmentationDataset,
                 ctx: int,
                 base_node_dst: int,
                 parts: Dict[str, List[int]],
                 labels_itf: str,
                 label_mappings: List[Tuple[int, int]],
                 split_jitter: int = 0):
    """
    Args:
        id_queue: Input queue with cell ids.
        chunk_queue: Output queue with cell chunks.
        ssd: SuperSegmentationDataset which contains the cells to which the chunkhandler should get applied.
        ctx: Context size for splitting.
        base_node_dst: Distance between base nodes. Corresponds to redundancy / number of chunks per cell.
        parts: Information about cell surface and organelles, Tuples like (voxel_param, feature) keyed by identifier
            compatible with syconn (e.g. 'sv' or 'mi').
        labels_itf: Label identifier for existing label predictions within the sso objects of the ssd dataset.
        label_mappings: Tuples where label at index 0 should get mapped to label at index 1.
        split_jitter: Derivation from context size during splitting.
    """
    while True:
        if not id_queue.empty():
            ssv_id = id_queue.get()
            sso = ssd.get_super_segmentation_object(ssv_id)
            vert_dc = {}
            label_dc = {}
            encoding = {}
            offset = 0
            obj_bounds = {}
            for ix, k in enumerate(parts):
                pcd = o3d.geometry.PointCloud()
                verts = sso.load_mesh(k)[1].reshape(-1, 3)
                pcd.points = o3d.utility.Vector3dVector(verts)
                pcd, idcs = pcd.voxel_down_sample_and_trace(
                    parts[k][0], pcd.get_min_bound(), pcd.get_max_bound())
                idcs = np.max(idcs, axis=1)
                vert_dc[k] = np.asarray(pcd.points)
                obj_bounds[k] = [offset, offset + len(pcd.points)]
                offset += len(pcd.points)
                if k == 'sv':
                    # prepare mask for filtering background / unpredicted points
                    mask = None
                    if labels_itf == 'axoness':
                        # 0: dendrite, 1: axon, 2: soma, 3: bouton, 4: terminal, 5/6: background/unpredicted
                        labels_total = sso.label_dict()[labels_itf][idcs]
                        mask = labels_total < 5
                        labels_total = labels_total[mask]
                    elif labels_itf == 'spiness':
                        # 1: head, 0: neck, 2: shaft, 3: other, 4/5: background/unpredicted
                        labels_total = sso.label_dict()['axoness'][idcs]
                        spiness = sso.label_dict()['spiness'][idcs]
                        mask = np.logical_not(
                            np.logical_or(
                                labels_total > 4,
                                np.logical_and(labels_total == 0,
                                               spiness > 3)))
                        labels_total = labels_total[mask]
                        spiness = spiness[mask]
                        labels_total[labels_total != 0] = 3
                        labels_total[labels_total == 0] = spiness[labels_total
                                                                  == 0]
                    else:
                        labels_total = sso.label_dict()[labels_itf][idcs]
                        mask = np.ones(len(labels_total)).astype(bool)
                    labels = labels_total
                    vert_dc[k] = vert_dc[k][mask]
                else:
                    labels = np.ones(len(vert_dc[k])) + ix + labels_total.max()
                    encoding[k] = ix + 1 + labels_total.max()
                label_dc[k] = labels
            sample_feats = np.concatenate([[parts[k][1]] * len(vert_dc[k])
                                           for k in parts]).reshape(-1, 1)
            sample_feats = label_binarize(sample_feats,
                                          classes=np.arange(len(parts)))
            sample_pts = np.concatenate([vert_dc[k] for k in parts])
            sample_labels = np.concatenate([label_dc[k] for k in parts])
            # mark cellular organelles to be excluded from loss calculation - see torchhandler for use of no_pred
            no_pred = list(encoding.keys())
            if not sso.load_skeleton():
                raise ValueError(f'Couldnt find skeleton of {sso}')
            nodes, edges = sso.skeleton['nodes'] * sso.scaling, sso.skeleton[
                'edges']
            hc = HybridCloud(nodes,
                             edges,
                             vertices=sample_pts,
                             features=sample_feats,
                             obj_bounds=obj_bounds,
                             no_pred=no_pred,
                             labels=sample_labels,
                             encoding=encoding)
            if label_mappings is not None:
                hc.map_labels(label_mappings)
            _ = hc.verts2node
            jitter = random.randint(0, split_jitter)
            node_arrs, source_nodes = splitting.split_single(
                hc, ctx + jitter, base_node_dst)
            for ix, node_arr in enumerate(node_arrs):
                sample, _ = objects.extract_cloud_subset(hc, node_arr)
                chunk_queue.put(sample)
        else:
            time.sleep(0.5)
Ejemplo n.º 6
0
 def __init__(self):
     ssc = SuperSegmentationDataset('/wholebrain/scratch/areaxfs3/')
     ssv = ssc.get_super_segmentation_object(29753344)
     #self.ssc = ssc.get_super_segmentation_object(29753344)
     exloc = np.array([5602, 4173, 4474]) * ssv.scaling
     self.exlocs = np.concatenate(ssv.sample_locations())
Ejemplo n.º 7
0
    def __init__(self):
        ssc = SuperSegmentationDataset('/wholebrain/scratch/areaxfs3/')
        ssv = ssc.get_super_segmentation_object(29753344)
        #self.ssc = ssc.get_super_segmentation_object(29753344)
        exloc = np.array([5602, 4173, 4474]) * ssv.scaling
        self.exlocs = np.concatenate(ssv.sample_locations())


if __name__ == '__main__':
    # TODO: use toy data and improve logging, see test_backend.py
    working_dir = "/wholebrain/scratch/areaxfs3/"
    render_indexview = True
    now = time.time()
    ssc = SuperSegmentationDataset(working_dir)
    ssv = ssc.get_super_segmentation_object(29753344)
    exlocs = np.concatenate(ssv.sample_locations())
    exlocs = exlocs[::30]
    print("Example location array:", exlocs.shape)
    print(working_dir)
    now2 = time.time()
    print("time for reading data")
    print(now2 - now)
    """
    i = 0
    exlocs = chunkify_successive(exlocs, 10)
    params = []
    for ex in exlocs:
        params.append([exlocs, i])
        i=i+1
        print(i)
Ejemplo n.º 8
0
from syconn import global_params

path_storage_file = sys.argv[1]
path_out_file = sys.argv[2]

with open(path_storage_file, 'rb') as f:
    args = []
    while True:
        try:
            args.append(pkl.load(f))
        except EOFError:
            break

ssv_ids = args[0]
version = args[1]
version_dict = args[2]
working_dir = args[3]

ssd = SuperSegmentationDataset(working_dir=working_dir,
                               version=version,
                               version_dict=version_dict)
for ssv in ssd.get_super_segmentation_object(ssv_ids):
    ssv.load_skeleton()
    ssv.skeleton["myelin"] = map_myelin2coords(ssv.skeleton["nodes"], mag=4)
    majorityvote_skeleton_property(
        ssv, prop_key='myelin', max_dist=global_params.DIST_AXONESS_AVERAGING)
    ssv.save_skeleton()

with open(path_out_file, "wb") as f:
    pkl.dump("0", f)
Ejemplo n.º 9
0
 # test prediction, copy trained models, see paths to 'models' folder defined in global_params.config
 from syconn.handler.prediction import get_celltype_model_large_e3, \
     get_tripletnet_model_large_e3, get_celltype_model_e3
 from syconn.reps.super_segmentation import SuperSegmentationDataset, SuperSegmentationObject
 import tqdm
 pred_key_appendix1 = "test_pred_v1_large"
 pred_key_appendix2 = ""
 ssd = SuperSegmentationDataset(working_dir=WD, version="ctgt_v2")
 # #
 # # prediction
 pbar = tqdm.tqdm(total=len(ssv_ids))
 m = get_celltype_model_e3()
 m_large = get_celltype_model_large_e3()
 m_tnet = get_tripletnet_model_large_e3()
 for ssv_id in ssv_ids:
     ssv = ssd.get_super_segmentation_object(ssv_id)
     ssv._view_caching = True
     # ssv.predict_celltype_cnn(model=m_large, pred_key_appendix=pred_key_appendix1,
     #                          model_tnet=m_tnet)
     ssv.predict_celltype_cnn(model=m,
                              pred_key_appendix=pred_key_appendix1,
                              largeFoV=False,
                              view_props={"overwrite": False})
     pbar.update()
 pbar.close()
 # analysis
 from syconn.proc.stats import cluster_summary, projection_tSNE, model_performance
 gt_l = []
 pred_l = []
 pred_proba = []
 pred_l_large = []
Ejemplo n.º 10
0
def run_neuron_rendering(max_n_jobs=2000):
    log = initialize_logging('neuron_view_rendering',
                             global_params.config.working_dir + '/logs/')
    # TODO: currently working directory has to be set globally in global_params
    #  and is not adjustable here because all qsub jobs will start a script
    #  referring to 'global_params.config.working_dir'
    # view rendering prior to glia removal, choose SSD accordingly
    ssd = SuperSegmentationDataset(
        working_dir=global_params.config.working_dir)

    #  TODO: use actual size criteria, e.g. number of sampling locations
    nb_svs_per_ssv = np.array(
        [len(ssd.mapping_dict[ssv_id]) for ssv_id in ssd.ssv_ids])

    # render normal size SSVs
    size_mask = nb_svs_per_ssv <= global_params.RENDERING_MAX_NB_SV
    multi_params = ssd.ssv_ids[size_mask]
    # TODO: move from osmesa to egl, egl rendering worker (10 cpus, 1 gpu) then should utilize more threads for bigger
    #  SSVs, and run more SSVs in parallel if they are small
    # sort ssv ids according to their number of SVs (descending)
    ordering = np.argsort(nb_svs_per_ssv[size_mask])
    multi_params = multi_params[ordering[::-1]]
    multi_params = chunkify(multi_params, max_n_jobs)
    # list of SSV IDs and SSD parameters need to be given to a single QSUB job
    multi_params = [(ixs, global_params.config.working_dir)
                    for ixs in multi_params]
    log.info('Started rendering of {} SSVs. '.format(np.sum(size_mask)))
    if np.sum(~size_mask) > 0:
        log.info('{} huge SSVs will be rendered afterwards using the whole'
                 ' cluster.'.format(np.sum(~size_mask)))
    # generic
    # TODO: switch n_cores to `global_params.NGPUS_TOTAL` as soon as EGL ressource allocation works!
    if global_params.PYOPENGL_PLATFORM == 'osmesa':  # utilize all CPUs
        path_to_out = qu.QSUB_script(
            multi_params,
            "render_views",
            n_max_co_processes=global_params.NCORE_TOTAL)
    elif global_params.PYOPENGL_PLATFORM == 'egl':  # utilize 1 GPU per task
        # TODO: use render_views_egl script
        path_to_out = qu.QSUB_script(
            multi_params,
            "render_views",
            n_max_co_processes=global_params.NNODES_TOTAL,
            additional_flags="--gres=gpu:2",
            n_cores=global_params.NCORES_PER_NODE)
    else:
        raise RuntimeError('Specified OpenGL platform "{}" not supported.'
                           ''.format(global_params.PYOPENGL_PLATFORM))
    if np.sum(~size_mask) > 0:
        log.info('Finished rendering of {}/{} SSVs.'.format(
            len(ordering), len(nb_svs_per_ssv)))
        # identify huge SSVs and process them individually on whole cluster
        big_ssv = ssd.ssv_ids[~size_mask]
        for kk, ssv_id in enumerate(big_ssv):
            ssv = ssd.get_super_segmentation_object(ssv_id)
            log.info(
                "Processing SSV [{}/{}] with {} SVs on whole cluster.".format(
                    kk + 1, len(big_ssv), len(ssv.sv_ids)))
            ssv.render_views(add_cellobjects=True,
                             cellobjects_only=False,
                             woglia=True,
                             qsub_pe="openmp",
                             overwrite=True,
                             qsub_co_jobs=global_params.NCORE_TOTAL,
                             skip_indexviews=False,
                             resume_job=False)
    log.info('Finished rendering of all SSVs. Checking completeness.')
    res = find_incomplete_ssv_views(ssd,
                                    woglia=True,
                                    n_cores=global_params.NCORES_PER_NODE)
    if len(res) != 0:
        msg = "Not all SSVs were rendered completely! Missing:\n{}".format(res)
        log.error(msg)
        raise RuntimeError(msg)
    else:
        log.info('Success.')
Ejemplo n.º 11
0
if __name__ == '__main__':
    # TODO: use toy data and improve logging, see test_backend.py
    working_dir = "/wholebrain/scratch/areaxfs3/"
    location_array = [
        29753344, 29833344, 30393344, 30493344, 32393344, 34493344, 39553344,
        42173344
    ]
    ssv_array = []
    length_array = []
    render_indexview = True
    now = time.time()
    ssc = SuperSegmentationDataset(working_dir)
    ix = 0
    for loc in location_array:
        ssv_array.append(ssc.get_super_segmentation_object(loc))
        exlocs = np.concatenate(ssv_array[ix].sample_locations())
        length_array.append(len(exlocs))
        ix = ix + 1
    #exlocs = exlocs[::30]
    print("Example location array:", exlocs.shape)
    print(working_dir)
    now2 = time.time()
    print("time for reading data")
    print(now2 - now)

    array = [19, 2, 31, 45, 6, 11, 121, 27]
    index_array = sorting_in_descending_order(array)
    print(array)
    print(index_array)
    index_array = sort_ssv_descending(ssv_array)
Ejemplo n.º 12
0
except ImportError:
    import pickle as pkl
from syconn.reps.super_segmentation_helper import create_sso_skeletons_wrapper
from syconn.reps.super_segmentation import SuperSegmentationDataset

path_storage_file = sys.argv[1]
path_out_file = sys.argv[2]

with open(path_storage_file, 'rb') as f:
    args = []
    while True:
        try:
            args.append(pkl.load(f))
        except EOFError:
            break

ssv_ids = args[0]
version = args[1]
version_dict = args[2]
working_dir = args[3]
map_myelin = args[4]

ssd = SuperSegmentationDataset(working_dir=working_dir,
                               version=version,
                               version_dict=version_dict)
ssvs = ssd.get_super_segmentation_object(ssv_ids)
create_sso_skeletons_wrapper(ssvs, map_myelin=map_myelin)

with open(path_out_file, "wb") as f:
    pkl.dump("0", f)