def nx_skel_to_knossos_skel(nxg): this_anno = skeleton.SkeletonAnnotation() this_anno.scaling = [10., 10., 20.] nodes_to_remove = [] for n in nxg.nodes(): #print(nxg.node[n]) try: coord = map( int, nxg.node[n]['coord'].replace('(', '').replace(')', '').replace( 'L', '').split(',')) except KeyError: #print('nx node has no coord defined, skipping.') nodes_to_remove.append(n) continue node = skeleton.SkeletonNode() node.from_scratch(this_anno, *coord, ID=n) this_anno.addNode(node) [nxg.remove_node(n) for n in nodes_to_remove] #if len(nodes_to_remove): # print('Had to remove {0} nodes without coord'.format(len(nodes_to_remove))) # add edges to skeleton file for e in nxg.edges(): this_anno.addEdge(this_anno.node_ID_to_node[e[0]], this_anno.node_ID_to_node[e[1]]) return this_anno
def update_RAG_with_reconnects( reconnect_folder='/mnt/j0126/areaxfs_v10/reconnect_tasks/final_tasks/', path_to_skeletons='/mnt/j0126/areaxfs_v10/reconnect_tasks/traced_skeletons/', path_to_reconn_rags='/mnt/j0126/areaxfs_v10/reconnect_tasks/resulting_ssv_rags/' ): """ Applies the reconnect skeleton tasks to an existing RAG by adding edges. Requires a Knossos segmentation dataset, from which the segmentation IDs are collected. :param reconnect_folder: :param path_to_skeletons: :param path_to_reconn_rags: :return: """ # load rag rag = load_best_bet_rag() # load all reconnects kzips = glob.glob(reconnect_folder + '*.k.zip') parsing_errors = [] task_dicts = [] for kzip in kzips: try: task_dicts.append(analyze_j0126_reconnector_task(kzip)) print('Successfully parsed task {0}'.format(kzip)) except: parsing_errors.append(kzip) print('Error parsing task {0}'.format(kzip)) #all_recon_tasks = flatten([t['k_annos'] for t in task_dicts]) all_recon_tasks = [] for task_dict in task_dicts: all_recon_tasks.extend(task_dict['k_annos']) all_src_ids = [] for task_dict in task_dicts: all_src_ids.extend(task_dict['src_ids']) all_src_coords = [] for task_dict in task_dicts: all_src_coords.extend(task_dict['src_coords']) print('Got in total {0} tasks'.format(len(all_recon_tasks))) # filter the skeletons that do not reconnect anything, defined by less than 5 reconnect nodes positive_reconnects = [a for a in all_recon_tasks if len(a.getNodes()) > 5] print('Got in total {0} reconnects > 5 nodes'.format( len(positive_reconnects))) print('Total parsing errors: {0}'.format(len(parsing_errors))) #return positive_reconnects # contains additional edges for RAG from manual reconnects additional_edges = [] workers = init_node_to_sv_id_workers() total_reconnects = len(positive_reconnects) recon_cnt = 0. rag_extension = [] start = time.time() #print(all_recon_tasks) #print(all_src_coords) #print(all_src_ids) unmapped_nodes_cnt = 0 for reconnect_anno, src_coords, src_id in zip(all_recon_tasks, all_src_coords, all_src_ids): #print(src_coords) #print(src_id) if len(reconnect_anno.getNodes()) < 5: continue recon_cnt += 1. print('Reconnects done: {0}%'.format(recon_cnt / total_reconnects * 100.)) mapped_nodes = [] node1 = skeleton.SkeletonNode() node1.from_scratch(reconnect_anno, *src_coords[0]) node1.setPureComment('source 1') reconnect_anno.addNode(node1) node2 = skeleton.SkeletonNode() node2.from_scratch(reconnect_anno, *src_coords[1]) node2.setPureComment('source 2') reconnect_anno.addNode(node2) reconnect_anno.addEdge(node1, node2) # connect source 1 with the closest node an annotator made if there is # one kd_tree = su.KDtree( reconnect_anno.getNodes(), [n.getCoordinate() for n in reconnect_anno.getNodes()]) nodes, dists = kd_tree.query_k_nearest([src_coords[0]], k=3, return_dists=True) for node, dist in zip(nodes, dists): # link the source 1 node with the first annotator placed node in the # skeleton; query k = 3 is necessary to ensure that one of the hits # is an annotator created node! if not 'source' in node.getPureComment(): reconnect_anno.addEdge(node1, node) break orig_reconnect_anno = copy.deepcopy(reconnect_anno) #reconnect_anno.interpolate_nodes(max_node_dist_scaled=200) # push nodes onto queue anno_nodes = reconnect_anno.getNodes() all_node_cnt = len(anno_nodes) + 10 for skel_node in reconnect_anno.getNodes(): node_to_sv_mappings_todo_queue.put(skel_node.getCoordinate()) # push the seed coordinates onto the queue - this is a hack, make sure # that they are mapped > 5 times, see below [node_to_sv_mappings_todo_queue.put(src_coords[0]) for i in range(5)] [node_to_sv_mappings_todo_queue.put(src_coords[1]) for i in range(5)] # wait for all nodes to be mapped done_nodes = 0 while done_nodes < all_node_cnt: node_coord, sv_id = node_to_sv_mappings_done_queue.get() mapped_nodes.append((node_coord, sv_id)) #done_node.setPureComment(str(sv_id)) done_nodes += 1 #print('\r\x1b[K Nodes done: {0} from {1} total'.format(done_nodes, all_node_cnt), end='') #time.sleep(0.01) all_mapped_sv_ids = [el[1] for el in mapped_nodes] # after interpolation, a new kd tree is needed kd_tree = su.KDtree( reconnect_anno.getNodes(), [n.getCoordinate() for n in reconnect_anno.getNodes()]) #print('Len reconnect anno nodes: {0}'.format(len(reconnect_anno.getNodes()))) #print('Len mapped nodes: {0}'.format( # len(mapped_nodes))) for mapped_node in mapped_nodes: #print('mapped_nodes[1]: {0}'.format(mapped_node[0])) #anno_node = kd_tree.query_ball_point(mapped_node[0], radius=10.) anno_node, dist = kd_tree.query_k_nearest([mapped_node[0]], return_dists=True) #anno_node = anno_node[0] #dist = dist[0] #if dist > 0.: #print('anno_node: {0}'.format(anno_node)) #print('dist: {0}'.format(dist)) # temp storage for mapped sv_id anno_node.sv_id = mapped_node[1] # count sv_ID occurences and replace infrequent ones with 0 very_likely = [] keep_ids = dict() #keep_ids[0] = False #print('Type sv_id 0 {0}'.format(type(0))) unique_ids, counts = np.unique(all_mapped_sv_ids, return_counts=True) for sv_id, cnt in zip(unique_ids, counts): #print('Type sv_id {0}'.format(type(sv_id))) if sv_id != 0: if cnt > 4: keep_ids[sv_id] = True very_likely.append(sv_id) else: keep_ids[sv_id] = False else: keep_ids[sv_id] = False # prune skeleton using keep_ids to remove nodes over the background # and also nodes with sv_ids that were mapped to sv_ids that were too # infrequent #all_nodes = list(reconnect_anno.getNodes()) nx_g = su.annotation_to_nx_graph(reconnect_anno) n_o_i = list({k for k, v in nx_g.degree() if v > 1}) # delete in-between nodes that should not be included #print('noi {0}'.format(n_o_i)) for node in n_o_i: #print('node.sv_id: {0}'.format(node.sv_id)) try: if keep_ids[node.sv_id] == False: # remove this node, relinking it to neighbors neighbors = list(nx_g[node].keys()) #print('Found neighbors {0}'.format(neighbors)) # connect all neighbors, take first (arbitrary) if len(neighbors) > 1: src_neighbor = neighbors[0] for neighbor in neighbors[1:]: reconnect_anno.addEdge(src_neighbor, neighbor) nx_g.add_edge(src_neighbor, neighbor) reconnect_anno.removeNode(node) nx_g.remove_node(node) except AttributeError: unmapped_nodes_cnt += 1 print('Node {0} of src_id {1} without sv_id.'.format( node, src_id)) n_o_i = list({k for k, v in nx_g.degree() if v == 1}) # delete end nodes that should not be included for node in n_o_i: try: if keep_ids[node.sv_id] == False: reconnect_anno.removeNode(node) nx_g.remove_node(node) except AttributeError: unmapped_nodes_cnt += 1 print('Node {0} of src_id {1} without sv_id.'.format( node, src_id)) for n in reconnect_anno.getNodes(): try: n.setPureComment('{0} sv id: {1}'.format( n.getPureComment(), n.sv_id)) except AttributeError: n.setPureComment('{0} sv id: {1}'.format( n.getPureComment(), 'not mapped')) # convert the skeleton to nx graph by iterating over the edges of the # skeleton annotation; the result is a nx graph representing the # topology of the skeleton, but consisting of sv_ids as nodes; topo_nx_g = nx.Graph() edges = reconnect_anno.getEdges() for src_node in list(edges.keys()): for trg_node in edges[src_node]: try: if src_node.sv_id != trg_node.sv_id: topo_nx_g.add_edge(src_node.sv_id, trg_node.sv_id) except AttributeError: pass #rag_extension.append(very_likely) #very_likely = [] #unique_likely, counts = np.unique(all_mapped_sv_ids, return_counts=True) #for sv_id, cnt in zip(unique_likely, counts): # if sv_id != 0: # if cnt > 3: # very_likely.append(sv_id) # list of list of svs that belong together according to the reconnect tracing rag_extension.append(topo_nx_g) # write topo_nx_g to file for later bidirectionality analysis # write annotation with mergelist as kzip to folder #for src_id, anno, src_coords in zip(all_recon_tasks['src_ids'], # all_recon_tasks['k_annos'], # all_recon_tasks['src_coords']): skel_obj = skeleton.Skeleton() #this_anno = skeleton.SkeletonAnnotation() #this_anno.scaling = [10.,10., 20.] skel_obj.add_annotation(orig_reconnect_anno) orig_reconnect_anno.setComment('tracing') skel_obj.add_annotation(reconnect_anno) reconnect_anno.setComment('sv topo') outfile = path_to_skeletons + 'reconnect_{0}.k.zip'.format(src_id) #print('Writing {0}'.format(outfile)) #skel_paths.append(outfile) skel_obj.to_kzip(outfile) # add mergelist to the kzip buff = '' buff += '{0} 0 0 '.format('1') for sv_id in very_likely: buff += '{0} '.format(sv_id) buff += '\n0 0 0\n\n\n' with zipfile.ZipFile(outfile, "a", zipfile.ZIP_DEFLATED) as zf: zf.writestr('mergelist.txt', buff) outfile = path_to_reconn_rags + 'ssv_rag_{0}.csv'.format(src_id) nx.write_edgelist(topo_nx_g, outfile, delimiter=',', data=False) for worker in workers: worker.terminate() added_rag_edges = 0 print('Extending global rag') for this_ssv_rag in rag_extension: new_edges = this_ssv_rag.edges() rag.add_edges_from(new_edges) added_rag_edges += len(new_edges) print('Done extending global rag') print('Added in total {0} edges to the rag'.format(added_rag_edges)) # create merge list with reconnects only for testing #for this_ext in rag_extension: # if len(this_ext) > 1: # last_id = this_ext[0] # for this_id in this_ext[1:]: # if not rag.has_edge(last_id, this_id): # rag.add_edge(last_id, this_id) # added_rag_edges.append((last_id, this_id)) nx_rag_to_knossos_mergelist( rag, path_to_mergelist= '/mnt/j0126/areaxfs_v10/RAGs/v4b_20180214_nocb_merges_reconnected_knossos_mergelist.txt' ) nx.write_edgelist( rag, '/mnt/j0126/areaxfs_v10/RAGs/v4b_20180214_nocb_merges_reconnected.txt', delimiter=',', data=False) print('Total number unmapped nodes: {0}'.format(unmapped_nodes_cnt)) print('Mapping {0} took {1}'.format(total_reconnects, (time.time() - start))) return
def add_node(x, y, z, r=5): new_node = knossos_skeleton.SkeletonNode() new_node.from_scratch(anno, x, y, z // 2, radius=r) anno.addNode(new_node) return new_node