def test_fix_source_sink_river(known_river): """Test fix_sources_and_sinks() with river example.""" links = known_river.links # flip links to create sources/sinks and check that it happened links = lnu.flip_link(links, 2635) links = lnu.flip_link(links, 2634) bad_nodes = di.check_continuity(links, known_river.nodes) assert bad_nodes == [1897, 1899] # now try to fix them newlinks, nodes = di.fix_sources_and_sinks(links, known_river.nodes) # re-check for sources and sinks and verify that there are less bad nodes fixed_nodes = di.check_continuity(newlinks, known_river.nodes) assert len(fixed_nodes) < len(bad_nodes)
def test_fix_source_sink(known_net): """Actually test fix_sources_and_sinks().""" links = known_net.links # verify that old problem node existed old_problem_nodes = di.check_continuity(links, known_net.nodes) assert old_problem_nodes == [177] # try to fix the continuity issue newlinks, nodes = di.fix_sources_and_sinks(links, known_net.nodes) # test fails at di.fix_sources_and_sinks(links, known_net.nodes) # we know an issue exists because we verified the problem node # error message: KeyError: 1081 problem_nodes = di.check_continuity(links, known_net.nodes) # make assertion that no problem node exists - aka 'fix' worked assert problem_nodes == []
def test_bad_continuity(known_net): """ Test check_continuity(). This test flips links so that continuity is disturbed. """ links = known_net.links links = lnu.flip_link(links, 199) links = lnu.flip_link(links, 198) problem_nodes = di.check_continuity(links, known_net.nodes) # make assertion that a problem node has been created assert problem_nodes == [177]
def test_fix_cycles(known_net): """Test fix_cycles().""" links = known_net.links nodes = known_net.nodes # check that cycle exists c_nodes, c_links = di.find_a_cycle(links, nodes) assert c_nodes == [761, 784] assert c_links == [964, 964] # check that there are no continuity issues problem_nodes = di.check_continuity(links, nodes) assert problem_nodes == [] # now try to fix the cycles links, nodes, n_cycles = di.fix_cycles(links, nodes)
def test_set_link_directions(known_net): """Test set_link_directions().""" links = known_net.links nodes = known_net.nodes imshape = known_net.Imask.shape # verify that old problem node exists or create it old_problem_nodes = di.check_continuity(links, nodes) if 177 in old_problem_nodes: assert old_problem_nodes == [177] else: links = lnu.flip_link(links, 198) old_problem_nodes = di.check_continuity(links, nodes) assert old_problem_nodes == [177] # set up capture string capturedOutput = io.StringIO() sys.stdout = capturedOutput # apply the function newlinks, newnodes = dd.set_link_directions(links, nodes, imshape) # grab output sys.stdout = sys.__stdout__ # assert output assert capturedOutput.getvalue()[:-1] == 'Nodes 177 violate continuity. Check connected links and fix manually.'
def test_fix_delta_cycles(known_net): """Test fix_delta_cycles().""" links = known_net.links nodes = known_net.nodes imshape = known_net.Imask.shape # check if a cycle exists c_nodes, c_links = di.find_a_cycle(links, nodes) assert c_nodes == [761, 784] assert c_links == [964, 964] # verify that no continuity issues exist problem_nodes = di.check_continuity(links, nodes) assert problem_nodes == [] # try to fix the cycle links, nodes, allfixed = dd.fix_delta_cycles(links, nodes, imshape)
def test_fix_cycles_river(known_river): """Test fix_cycles() with river example.""" links = known_river.links nodes = known_river.nodes # flip link to create a cycle links = lnu.flip_link(links, 2494) # check that cycle exists c_nodes, c_links = di.find_a_cycle(links, nodes) assert c_nodes == [1794, 1805] assert c_links == [2494, 2494] # check that there are no continuity issues problem_nodes = di.check_continuity(links, nodes) assert problem_nodes == [] # now try to fix the cycles links, nodes, n_cycles = di.fix_cycles(links, nodes)
def set_link_directions(links, nodes, imshape, manual_set_csv=None): """ Set each link direction in a network. This function sets the direction of each link within the network. It calls a number of helping functions and uses a somewhat-complicated logic to achieve this. The algorithms and logic is described in this open-access paper: https://esurf.copernicus.org/articles/8/87/2020/esurf-8-87-2020.pdf Every time this is run, all directionality information is reset and recomputed. This includes checking for manually set links via the provided csv. Parameters ---------- links : dict Network links and associated properties. nodes : dict Network nodes and associated properties. imshape : tuple Shape of binary mask as (nrows, ncols). manual_set_csv : str, optional Path to a user-provided csv file of known link directions. The default is None. Returns ------- links : dict Network links and associated properties with all directions set. nodes : dict Network nodes and associated properties with all directions set. """ # Add fields to links dict for tracking and setting directionality links, nodes = dy.add_directionality_trackers(links, nodes, 'delta') # If a manual fix csv has been provided, set those links first links, nodes = dy.dir_set_manually(links, nodes, manual_set_csv) # Initial attempt to set directions links, nodes = set_initial_directionality(links, nodes, imshape) # At this point, all links have been set. Check for nodes that violate # continuity cont_violators = dy.check_continuity(links, nodes) # Attempt to fix any sources or sinks within the network if len(cont_violators) > 0: links, nodes = dy.fix_sources_and_sinks(links, nodes) # Check that continuity problems are resolved cont_violators = dy.check_continuity(links, nodes) if len(cont_violators) > 0: print( 'Nodes {} violate continuity. Check connected links and fix manually.' .format(cont_violators)) # Attempt to fix any cycles in the network (reports unfixable within function) links, nodes, allcyclesfixed = fix_delta_cycles(links, nodes, imshape) # The following is done automatically now, regardless of if cycles or sinks exist # # Create a csv to store manual edits to directionality if does not exist # if os.path.isfile(manual_set_csv) is False: # if len(cont_violators) > 0 or allcyclesfixed == 0: # io.create_manual_dir_csv(manual_set_csv) # print('A .csv file for manual fixes to link directions at {}.'.format(manual_set_csv)) if allcyclesfixed == 2: print('No cycles were found in network.') return links, nodes
def ensure_single_inlet(links, nodes): """ Ensure only a single apex node exists. This dumbly just prunes all inlet nodes+links except the widest one. Recommended to use the super_apex() approach instead if you want to preserve all inlets. All the delta metrics here require a single apex node, and that that node be connected to at least two downstream links. This function ensures these conditions are met; where there are multiple inlets, the widest is chosen. This function also ensures that the inlet node is attached to at least two links--this is important for computing un-biased delta metrics. The links and nodes dicts are copied so they remain unaltered; the altered copies are returned. """ # Copy links and nodes so we preserve the originals links_edit = dict() links_edit.update(links) nodes_edit = dict() nodes_edit.update(nodes) # Find the widest inlet in_wids = [] for i in nodes_edit['inlets']: linkid = nodes_edit['conn'][nodes_edit['id'].index(i)][0] linkidx = links_edit['id'].index(linkid) in_wids.append(links_edit['wid_adj'][linkidx]) widest_inlet_idx = in_wids.index(max(in_wids)) inlets_to_remove = nodes_edit['inlets'][:] # Remove inlet nodes and links until continuity is no longer broken badnodes = dy.check_continuity(links_edit, nodes_edit) if len(badnodes) > 0: raise RuntimeError( 'Provided (links, nodes) has source or sink at nodes: {}.'.format( badnodes)) # Keep the widest inlet - delete all others (and remove their subnetworks) main_inlet = inlets_to_remove.pop(widest_inlet_idx) for i in inlets_to_remove: nodes_edit['inlets'].remove(i) badnodes = dy.check_continuity(links_edit, nodes_edit) while len(badnodes) > 0: badnode = badnodes.pop() # Remove the links connected to the bad node: # the hanging node will also be removed connlinks = nodes_edit['conn'][nodes_edit['id'].index(badnode)] for cl in connlinks: links_edit, nodes_edit = lnu.delete_link( links_edit, nodes_edit, cl) badnodes = dy.check_continuity(links_edit, nodes_edit) # Ensure there are at least two links emanating from the inlet node conn = nodes_edit['conn'][nodes_edit['id'].index(main_inlet)] while len(conn) == 1: main_inlet_new = links_edit['conn'][links_edit['id'].index(conn[0])][:] main_inlet_new.remove(main_inlet) links_edit, nodes_edit = lnu.delete_link(links_edit, nodes_edit, conn[0]) # Update new inlet node nodes_edit['inlets'].remove(main_inlet) main_inlet = main_inlet_new[0] nodes_edit['inlets'] = nodes_edit['inlets'] + [main_inlet] conn = nodes_edit['conn'][nodes_edit['id'].index(main_inlet)] return links_edit, nodes_edit
def fix_river_cycle(links, nodes, cyclelinks, cyclenodes, imshape): """ Attempt to fix a single cycle. Attempts to resolve a single cycle within a river network. The general logic is that all link directions of the cycle are un-set except for those set by highly-reliable algorithms, and a modified direction-setting recipe is implemented to re-set these algorithms. This was developed according to the most commonly-encountered cases for real braided rivers, but could certainly be improved. Parameters ---------- links : dict Network links and associated properties. nodes : dict Network nodes and associated properties. cyclelinks : list List of link ids that comprise a cycle. cyclenodes : list List of node ids taht comprise a cycle. imshape : tuple Shape of binary mask as (nrows, ncols). Returns ------- links : dict Network links and associated properties with the cycle fixed if possible. nodes : dict Network nodes and associated properties with the cycle fixed if possible. fixed : int 1 if the cycle was resolved, else 0. """ # dont_reset_algs = [20, 21, 22, 23, 0, 5] dont_reset_algs = [ dy.algmap(key) for key in [ 'manual_set', 'cl_dist_guess', 'cl_ang_guess', 'cl_dist_set', 'cl_ang_set', 'inletoutlet', 'bridges' ] ] fixed = 1 # One if fix was successful, else zero reset = 0 # One if original orientation need to be reset # If an artifical node triad is present, flip its direction and see if the # cycle is resolved. # See if any links are part of an artificial triad clset = set(cyclelinks) all_pars = [] for i, pl in enumerate(links['parallels']): if len(clset.intersection(set(pl))) > 0: all_pars.append(pl) # Get continuity violators before flipping pre_sourcesink = dy.check_continuity(links, nodes) if len( all_pars ) == 1: # There is one parallel link set, flip its direction and re-set other cycle links and see if cycle is resolved certzero = list(set(all_pars[0] + cyclelinks)) orig_links = dy.cycle_get_original_orientation( links, certzero ) # Save the original orientations in case the cycle can't be fixed for cz in certzero: links['certain'][links['id'].index(cz)] = 0 # Flip the links of the triad for l in all_pars[0]: links = lnu.flip_link(links, l) if len( all_pars ) > 1: # If there are multiple parallel pairs, more code needs to be written for these cases print( 'Multiple parallel pairs in the same cycle. Not implemented yet.') return links, nodes, 0 elif len( all_pars ) == 0: # No aritifical node triads; just re-set all the cycle links and see if cycle is resolved certzero = cyclelinks orig_links = dy.cycle_get_original_orientation(links, certzero) for cz in certzero: lidx = links['id'].index(cz) if links['certain_alg'][lidx] not in dont_reset_algs: links['certain'][lidx] = 0 # Resolve the unknown cycle links links, nodes = re_set_linkdirs(links, nodes, imshape) # See if the fix violated continuity - if not, reset to original post_sourcesink = dy.check_continuity(links, nodes) if len(set(post_sourcesink) - set(pre_sourcesink)) > 0: reset = 1 # See if the fix resolved the cycle - if not, reset to original cyc_n, cyc_l = dy.get_cycles(links, nodes, checknode=cyclenodes[0]) if cyc_n is not None and cyclenodes[0] in cyc_n[0]: reset = 1 # Return the links to their original orientations if cycle could not # be resolved if reset == 1: links = dy.cycle_return_to_original_orientation(links, orig_links) # Try a second method to fix the cycle: unset all the links of the # cycle AND the links connected to those links set_to_zero = set() for cn in cyclenodes: conn = nodes['conn'][nodes['id'].index(cn)] set_to_zero.update(conn) set_to_zero = list(set_to_zero) # Save original orientation in case cycle cannot be fixed orig_links = dy.cycle_get_original_orientation(links, set_to_zero) for s in set_to_zero: lidx = links['id'].index(s) if links['certain_alg'][lidx] not in dont_reset_algs: links['certain'][lidx] = 0 links, nodes = re_set_linkdirs(links, nodes, imshape) # See if the fix violated continuity - if not, reset to original post_sourcesink = dy.check_continuity(links, nodes) if len(set(post_sourcesink) - set(pre_sourcesink)) > 0: reset = 1 # See if the fix resolved the cycle - if not, reset to original cyc_n, cyc_l = dy.get_cycles(links, nodes, checknode=cyclenodes[0]) if cyc_n is not None and cyclenodes[0] in cyc_n[0]: reset = 1 if reset == 1: links = dy.cycle_return_to_original_orientation(links, orig_links) fixed = 0 return links, nodes, fixed
def set_directionality(links, nodes, Imask, exit_sides, gt, meshlines, meshpolys, Idt, pixlen, manual_set_csv): """ Set direction of each link. This function sets the direction of each link within the network. It calls a number of helping functions and uses a somewhat-complicated logic to achieve this. The algorithms and logic is described in this open-access paper: https://esurf.copernicus.org/articles/8/87/2020/esurf-8-87-2020.pdf Every time this is run, all directionality information is reset and recomputed. This includes checking for manually set links via the provided csv. Parameters ---------- links : dict Network links and associated properties. nodes : dict Network nodes and associated properties. Imask : np.array Binary mask of the network. exit_sides : str Two-character string of cardinal directions denoting the upstream and downsteram sides of the image that the network intersects (e.g. 'SW'). gt : tuple gdal-type GeoTransform of the original binary mask. meshlines : list List of shapely.geometry.LineStrings that define the valleyline mesh. meshpolys : list List of shapely.geometry.Polygons that define the valleyline mesh. Idt : np.array() Distance transform of Imask. pixlen : float Length resolution of each pixel. manual_set_csv : str, optional Path to a user-provided csv file of known link directions. The default is None. Returns ------- links : dict Network links and associated properties with all directions set. nodes : dict Network nodes and associated properties with all directions set. """ imshape = Imask.shape # Add fields to links dict for tracking and setting directionality links, nodes = dy.add_directionality_trackers(links, nodes, 'river') # If a manual fix csv has been provided, set those links first links, nodes = dy.dir_set_manually(links, nodes, manual_set_csv) # Append morphological information used to set directionality to links dict links, nodes = directional_info(links, nodes, Imask, pixlen, exit_sides, gt, meshlines, meshpolys, Idt) # Begin setting link directionality # First, set inlet/outlet directions as they are always 100% accurate links, nodes = dy.set_inletoutlet(links, nodes) # Set the directions of the links that are more certain via centerline # distance method # alg = 22 alg = dy.algmap('cl_dist_set') cl_distthresh = np.percentile(links['cldists'], 85) for lid, cld, lg, lga, cert in zip(links['id'], links['cldists'], links['guess'], links['guess_alg'], links['certain']): if cert == 1: continue if cld >= cl_distthresh: linkidx = links['id'].index(lid) if dy.algmap('cl_dist_guess') in lga: usnode = lg[lga.index(dy.algmap('cl_dist_guess'))] links, nodes = dy.set_link(links, nodes, linkidx, usnode, alg) # Set the directions of the links that are more certain via centerline # angle method # alg = 23 alg = dy.algmap('cl_ang_set') cl_angthresh = np.percentile( links['clangs'][np.isnan(links['clangs']) == 0], 25) for lid, cla, lg, lga, cert in zip(links['id'], links['clangs'], links['guess'], links['guess_alg'], links['certain']): if cert == 1: continue if np.isnan(cla) == True: continue if cla <= cl_angthresh: linkidx = links['id'].index(lid) if dy.algmap('cl_ang_guess') in lga: usnode = lg[lga.index(dy.algmap('cl_ang_guess'))] links, nodes = dy.set_link(links, nodes, linkidx, usnode, alg) # Set the directions of the links that are more certain via centerline # distance AND centerline angle methods # alg = 24 alg = dy.algmap('cl_dist_and_ang') cl_distthresh = np.percentile(links['cldists'], 70) ang_thresh = np.percentile(links['clangs'][np.isnan(links['clangs']) == 0], 35) for lid, cld, cla, lg, lga, cert in zip(links['id'], links['cldists'], links['clangs'], links['guess'], links['guess_alg'], links['certain']): if cert == 1: continue if cld >= cl_distthresh and cla < ang_thresh: linkidx = links['id'].index(lid) if dy.algmap('cl_dist_guess') in lga and dy.algmap( 'cl_ang_guess') in lga: if lg[lga.index(dy.algmap('cl_dist_guess'))] == lg[lga.index( dy.algmap('cl_ang_guess'))]: usnode = lg[lga.index(dy.algmap('cl_dist_guess'))] links, nodes = dy.set_link(links, nodes, linkidx, usnode, alg) # Set directions by most-certain angles angthreshs = np.linspace(0, 0.4, 10) for a in angthreshs: links, nodes = dy.set_by_known_flow_directions(links, nodes, imshape, angthresh=a, lenthresh=3) # Set using direction of nearest main channel links, nodes = dy.set_by_nearest_main_channel(links, nodes, imshape, nodethresh=1) angthreshs = np.linspace(0, 1.5, 20) for a in angthreshs: links, nodes = dy.set_by_known_flow_directions(links, nodes, imshape, angthresh=a) # At this point, if any links remain unset, they are just set randomly if np.sum(links['certain']) != len(links['id']): print('{} links were randomly set.'.format( len(links['id']) - np.sum(links['certain']))) links['certain'] = np.ones((len(links['id']), 1)) # Check for and try to fix cycles in the graph links, nodes, cantfix_cyclelinks, cantfix_cyclenodes = fix_river_cycles( links, nodes, imshape) # Check for sources or sinks within the graph cont_violators = dy.check_continuity(links, nodes) # Summary of problems: manual_fix = 0 if len(cantfix_cyclelinks) > 0: print('Could not fix cycle links: {}.'.format(cantfix_cyclelinks)) manual_fix = 1 else: print('All cycles were resolved.') if len(cont_violators) > 0: print('Continuity violated at nodes {}.'.format(cont_violators)) manual_fix = 1 # Create a csv to store manual edits to directionality if does not exist if manual_fix == 1: if os.path.isfile(manual_set_csv) is False: io.create_manual_dir_csv(manual_set_csv) print('A .csv file for manual fixes to link directions at {}.'. format(manual_set_csv)) else: print('Use the csv file at {} to manually fix link directions.'. format(manual_set_csv)) return links, nodes