def test_fix_source_sink_river(known_river):
"""Test fix_sources_and_sinks() with river example."""
links = known_river.links
# flip links to create sources/sinks and check that it happened
links = lnu.flip_link(links, 2635)
links = lnu.flip_link(links, 2634)
bad_nodes = di.check_continuity(links, known_river.nodes)
assert bad_nodes == [1897, 1899]
# now try to fix them
newlinks, nodes = di.fix_sources_and_sinks(links, known_river.nodes)
# re-check for sources and sinks and verify that there are less bad nodes
fixed_nodes = di.check_continuity(newlinks, known_river.nodes)
assert len(fixed_nodes) < len(bad_nodes)
def test_fix_source_sink(known_net):
"""Actually test fix_sources_and_sinks()."""
links = known_net.links
# verify that old problem node existed
old_problem_nodes = di.check_continuity(links, known_net.nodes)
assert old_problem_nodes == [177]
# try to fix the continuity issue
newlinks, nodes = di.fix_sources_and_sinks(links, known_net.nodes)
# test fails at di.fix_sources_and_sinks(links, known_net.nodes)
# we know an issue exists because we verified the problem node
# error message: KeyError: 1081
problem_nodes = di.check_continuity(links, known_net.nodes)
# make assertion that no problem node exists - aka 'fix' worked
assert problem_nodes == []
def test_bad_continuity(known_net):
"""
Test check_continuity().
This test flips links so that continuity is disturbed.
"""
links = known_net.links
links = lnu.flip_link(links, 199)
links = lnu.flip_link(links, 198)
problem_nodes = di.check_continuity(links, known_net.nodes)
# make assertion that a problem node has been created
assert problem_nodes == [177]
def test_fix_cycles(known_net):
"""Test fix_cycles()."""
links = known_net.links
nodes = known_net.nodes
# check that cycle exists
c_nodes, c_links = di.find_a_cycle(links, nodes)
assert c_nodes == [761, 784]
assert c_links == [964, 964]
# check that there are no continuity issues
problem_nodes = di.check_continuity(links, nodes)
assert problem_nodes == []
# now try to fix the cycles
links, nodes, n_cycles = di.fix_cycles(links, nodes)
def test_set_link_directions(known_net):
"""Test set_link_directions()."""
links = known_net.links
nodes = known_net.nodes
imshape = known_net.Imask.shape
# verify that old problem node exists or create it
old_problem_nodes = di.check_continuity(links, nodes)
if 177 in old_problem_nodes:
assert old_problem_nodes == [177]
else:
links = lnu.flip_link(links, 198)
old_problem_nodes = di.check_continuity(links, nodes)
assert old_problem_nodes == [177]
# set up capture string
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
# apply the function
newlinks, newnodes = dd.set_link_directions(links, nodes, imshape)
# grab output
sys.stdout = sys.__stdout__
# assert output
assert capturedOutput.getvalue()[:-1] == 'Nodes 177 violate continuity. Check connected links and fix manually.'
def test_fix_delta_cycles(known_net):
"""Test fix_delta_cycles()."""
links = known_net.links
nodes = known_net.nodes
imshape = known_net.Imask.shape
# check if a cycle exists
c_nodes, c_links = di.find_a_cycle(links, nodes)
assert c_nodes == [761, 784]
assert c_links == [964, 964]
# verify that no continuity issues exist
problem_nodes = di.check_continuity(links, nodes)
assert problem_nodes == []
# try to fix the cycle
links, nodes, allfixed = dd.fix_delta_cycles(links, nodes, imshape)
def test_fix_cycles_river(known_river):
"""Test fix_cycles() with river example."""
links = known_river.links
nodes = known_river.nodes
# flip link to create a cycle
links = lnu.flip_link(links, 2494)
# check that cycle exists
c_nodes, c_links = di.find_a_cycle(links, nodes)
assert c_nodes == [1794, 1805]
assert c_links == [2494, 2494]
# check that there are no continuity issues
problem_nodes = di.check_continuity(links, nodes)
assert problem_nodes == []
# now try to fix the cycles
links, nodes, n_cycles = di.fix_cycles(links, nodes)
def set_link_directions(links, nodes, imshape, manual_set_csv=None):
"""
Set each link direction in a network.
This function sets the direction of each link within the network. It
calls a number of helping functions and uses a somewhat-complicated logic
to achieve this. The algorithms and logic is described in this open-access
paper: https://esurf.copernicus.org/articles/8/87/2020/esurf-8-87-2020.pdf
Every time this is run, all directionality information is reset and
recomputed. This includes checking for manually set links via the provided
csv.
Parameters
----------
links : dict
Network links and associated properties.
nodes : dict
Network nodes and associated properties.
imshape : tuple
Shape of binary mask as (nrows, ncols).
manual_set_csv : str, optional
Path to a user-provided csv file of known link directions. The default
is None.
Returns
-------
links : dict
Network links and associated properties with all directions set.
nodes : dict
Network nodes and associated properties with all directions set.
"""
# Add fields to links dict for tracking and setting directionality
links, nodes = dy.add_directionality_trackers(links, nodes, 'delta')
# If a manual fix csv has been provided, set those links first
links, nodes = dy.dir_set_manually(links, nodes, manual_set_csv)
# Initial attempt to set directions
links, nodes = set_initial_directionality(links, nodes, imshape)
# At this point, all links have been set. Check for nodes that violate
# continuity
cont_violators = dy.check_continuity(links, nodes)
# Attempt to fix any sources or sinks within the network
if len(cont_violators) > 0:
links, nodes = dy.fix_sources_and_sinks(links, nodes)
# Check that continuity problems are resolved
cont_violators = dy.check_continuity(links, nodes)
if len(cont_violators) > 0:
print(
'Nodes {} violate continuity. Check connected links and fix manually.'
.format(cont_violators))
# Attempt to fix any cycles in the network (reports unfixable within function)
links, nodes, allcyclesfixed = fix_delta_cycles(links, nodes, imshape)
# The following is done automatically now, regardless of if cycles or sinks exist
# # Create a csv to store manual edits to directionality if does not exist
# if os.path.isfile(manual_set_csv) is False:
# if len(cont_violators) > 0 or allcyclesfixed == 0:
# io.create_manual_dir_csv(manual_set_csv)
# print('A .csv file for manual fixes to link directions at {}.'.format(manual_set_csv))
if allcyclesfixed == 2:
print('No cycles were found in network.')
return links, nodes
def ensure_single_inlet(links, nodes):
"""
Ensure only a single apex node exists. This dumbly just prunes all inlet
nodes+links except the widest one. Recommended to use the super_apex()
approach instead if you want to preserve all inlets.
All the delta metrics here require a single apex node, and that that node
be connected to at least two downstream links. This function ensures these
conditions are met; where there are multiple inlets, the widest is chosen.
This function also ensures that the inlet node is attached to at least two
links--this is important for computing un-biased delta metrics.
The links and nodes dicts are copied so they remain unaltered; the altered
copies are returned.
"""
# Copy links and nodes so we preserve the originals
links_edit = dict()
links_edit.update(links)
nodes_edit = dict()
nodes_edit.update(nodes)
# Find the widest inlet
in_wids = []
for i in nodes_edit['inlets']:
linkid = nodes_edit['conn'][nodes_edit['id'].index(i)][0]
linkidx = links_edit['id'].index(linkid)
in_wids.append(links_edit['wid_adj'][linkidx])
widest_inlet_idx = in_wids.index(max(in_wids))
inlets_to_remove = nodes_edit['inlets'][:]
# Remove inlet nodes and links until continuity is no longer broken
badnodes = dy.check_continuity(links_edit, nodes_edit)
if len(badnodes) > 0:
raise RuntimeError(
'Provided (links, nodes) has source or sink at nodes: {}.'.format(
badnodes))
# Keep the widest inlet - delete all others (and remove their subnetworks)
main_inlet = inlets_to_remove.pop(widest_inlet_idx)
for i in inlets_to_remove:
nodes_edit['inlets'].remove(i)
badnodes = dy.check_continuity(links_edit, nodes_edit)
while len(badnodes) > 0:
badnode = badnodes.pop()
# Remove the links connected to the bad node:
# the hanging node will also be removed
connlinks = nodes_edit['conn'][nodes_edit['id'].index(badnode)]
for cl in connlinks:
links_edit, nodes_edit = lnu.delete_link(
links_edit, nodes_edit, cl)
badnodes = dy.check_continuity(links_edit, nodes_edit)
# Ensure there are at least two links emanating from the inlet node
conn = nodes_edit['conn'][nodes_edit['id'].index(main_inlet)]
while len(conn) == 1:
main_inlet_new = links_edit['conn'][links_edit['id'].index(conn[0])][:]
main_inlet_new.remove(main_inlet)
links_edit, nodes_edit = lnu.delete_link(links_edit, nodes_edit,
conn[0])
# Update new inlet node
nodes_edit['inlets'].remove(main_inlet)
main_inlet = main_inlet_new[0]
nodes_edit['inlets'] = nodes_edit['inlets'] + [main_inlet]
conn = nodes_edit['conn'][nodes_edit['id'].index(main_inlet)]
return links_edit, nodes_edit
def fix_river_cycle(links, nodes, cyclelinks, cyclenodes, imshape):
"""
Attempt to fix a single cycle.
Attempts to resolve a single cycle within a river network. The general
logic is that all link directions of the cycle are un-set except for those
set by highly-reliable algorithms, and a modified direction-setting
recipe is implemented to re-set these algorithms. This was developed
according to the most commonly-encountered cases for real braided rivers,
but could certainly be improved.
Parameters
----------
links : dict
Network links and associated properties.
nodes : dict
Network nodes and associated properties.
cyclelinks : list
List of link ids that comprise a cycle.
cyclenodes : list
List of node ids taht comprise a cycle.
imshape : tuple
Shape of binary mask as (nrows, ncols).
Returns
-------
links : dict
Network links and associated properties with the cycle fixed if
possible.
nodes : dict
Network nodes and associated properties with the cycle fixed if
possible.
fixed : int
1 if the cycle was resolved, else 0.
"""
# dont_reset_algs = [20, 21, 22, 23, 0, 5]
dont_reset_algs = [
dy.algmap(key) for key in [
'manual_set', 'cl_dist_guess', 'cl_ang_guess', 'cl_dist_set',
'cl_ang_set', 'inletoutlet', 'bridges'
]
]
fixed = 1 # One if fix was successful, else zero
reset = 0 # One if original orientation need to be reset
# If an artifical node triad is present, flip its direction and see if the
# cycle is resolved.
# See if any links are part of an artificial triad
clset = set(cyclelinks)
all_pars = []
for i, pl in enumerate(links['parallels']):
if len(clset.intersection(set(pl))) > 0:
all_pars.append(pl)
# Get continuity violators before flipping
pre_sourcesink = dy.check_continuity(links, nodes)
if len(
all_pars
) == 1: # There is one parallel link set, flip its direction and re-set other cycle links and see if cycle is resolved
certzero = list(set(all_pars[0] + cyclelinks))
orig_links = dy.cycle_get_original_orientation(
links, certzero
) # Save the original orientations in case the cycle can't be fixed
for cz in certzero:
links['certain'][links['id'].index(cz)] = 0
# Flip the links of the triad
for l in all_pars[0]:
links = lnu.flip_link(links, l)
if len(
all_pars
) > 1: # If there are multiple parallel pairs, more code needs to be written for these cases
print(
'Multiple parallel pairs in the same cycle. Not implemented yet.')
return links, nodes, 0
elif len(
all_pars
) == 0: # No aritifical node triads; just re-set all the cycle links and see if cycle is resolved
certzero = cyclelinks
orig_links = dy.cycle_get_original_orientation(links, certzero)
for cz in certzero:
lidx = links['id'].index(cz)
if links['certain_alg'][lidx] not in dont_reset_algs:
links['certain'][lidx] = 0
# Resolve the unknown cycle links
links, nodes = re_set_linkdirs(links, nodes, imshape)
# See if the fix violated continuity - if not, reset to original
post_sourcesink = dy.check_continuity(links, nodes)
if len(set(post_sourcesink) - set(pre_sourcesink)) > 0:
reset = 1
# See if the fix resolved the cycle - if not, reset to original
cyc_n, cyc_l = dy.get_cycles(links, nodes, checknode=cyclenodes[0])
if cyc_n is not None and cyclenodes[0] in cyc_n[0]:
reset = 1
# Return the links to their original orientations if cycle could not
# be resolved
if reset == 1:
links = dy.cycle_return_to_original_orientation(links, orig_links)
# Try a second method to fix the cycle: unset all the links of the
# cycle AND the links connected to those links
set_to_zero = set()
for cn in cyclenodes:
conn = nodes['conn'][nodes['id'].index(cn)]
set_to_zero.update(conn)
set_to_zero = list(set_to_zero)
# Save original orientation in case cycle cannot be fixed
orig_links = dy.cycle_get_original_orientation(links, set_to_zero)
for s in set_to_zero:
lidx = links['id'].index(s)
if links['certain_alg'][lidx] not in dont_reset_algs:
links['certain'][lidx] = 0
links, nodes = re_set_linkdirs(links, nodes, imshape)
# See if the fix violated continuity - if not, reset to original
post_sourcesink = dy.check_continuity(links, nodes)
if len(set(post_sourcesink) - set(pre_sourcesink)) > 0:
reset = 1
# See if the fix resolved the cycle - if not, reset to original
cyc_n, cyc_l = dy.get_cycles(links, nodes, checknode=cyclenodes[0])
if cyc_n is not None and cyclenodes[0] in cyc_n[0]:
reset = 1
if reset == 1:
links = dy.cycle_return_to_original_orientation(links, orig_links)
fixed = 0
return links, nodes, fixed
def set_directionality(links, nodes, Imask, exit_sides, gt, meshlines,
meshpolys, Idt, pixlen, manual_set_csv):
"""
Set direction of each link.
This function sets the direction of each link within the network. It
calls a number of helping functions and uses a somewhat-complicated logic
to achieve this. The algorithms and logic is described in this open-access
paper: https://esurf.copernicus.org/articles/8/87/2020/esurf-8-87-2020.pdf
Every time this is run, all directionality information is reset and
recomputed. This includes checking for manually set links via the provided
csv.
Parameters
----------
links : dict
Network links and associated properties.
nodes : dict
Network nodes and associated properties.
Imask : np.array
Binary mask of the network.
exit_sides : str
Two-character string of cardinal directions denoting the upstream and
downsteram sides of the image that the network intersects (e.g. 'SW').
gt : tuple
gdal-type GeoTransform of the original binary mask.
meshlines : list
List of shapely.geometry.LineStrings that define the valleyline mesh.
meshpolys : list
List of shapely.geometry.Polygons that define the valleyline mesh.
Idt : np.array()
Distance transform of Imask.
pixlen : float
Length resolution of each pixel.
manual_set_csv : str, optional
Path to a user-provided csv file of known link directions. The default
is None.
Returns
-------
links : dict
Network links and associated properties with all directions set.
nodes : dict
Network nodes and associated properties with all directions set.
"""
imshape = Imask.shape
# Add fields to links dict for tracking and setting directionality
links, nodes = dy.add_directionality_trackers(links, nodes, 'river')
# If a manual fix csv has been provided, set those links first
links, nodes = dy.dir_set_manually(links, nodes, manual_set_csv)
# Append morphological information used to set directionality to links dict
links, nodes = directional_info(links, nodes, Imask, pixlen, exit_sides,
gt, meshlines, meshpolys, Idt)
# Begin setting link directionality
# First, set inlet/outlet directions as they are always 100% accurate
links, nodes = dy.set_inletoutlet(links, nodes)
# Set the directions of the links that are more certain via centerline
# distance method
# alg = 22
alg = dy.algmap('cl_dist_set')
cl_distthresh = np.percentile(links['cldists'], 85)
for lid, cld, lg, lga, cert in zip(links['id'], links['cldists'],
links['guess'], links['guess_alg'],
links['certain']):
if cert == 1:
continue
if cld >= cl_distthresh:
linkidx = links['id'].index(lid)
if dy.algmap('cl_dist_guess') in lga:
usnode = lg[lga.index(dy.algmap('cl_dist_guess'))]
links, nodes = dy.set_link(links, nodes, linkidx, usnode, alg)
# Set the directions of the links that are more certain via centerline
# angle method
# alg = 23
alg = dy.algmap('cl_ang_set')
cl_angthresh = np.percentile(
links['clangs'][np.isnan(links['clangs']) == 0], 25)
for lid, cla, lg, lga, cert in zip(links['id'], links['clangs'],
links['guess'], links['guess_alg'],
links['certain']):
if cert == 1:
continue
if np.isnan(cla) == True:
continue
if cla <= cl_angthresh:
linkidx = links['id'].index(lid)
if dy.algmap('cl_ang_guess') in lga:
usnode = lg[lga.index(dy.algmap('cl_ang_guess'))]
links, nodes = dy.set_link(links, nodes, linkidx, usnode, alg)
# Set the directions of the links that are more certain via centerline
# distance AND centerline angle methods
# alg = 24
alg = dy.algmap('cl_dist_and_ang')
cl_distthresh = np.percentile(links['cldists'], 70)
ang_thresh = np.percentile(links['clangs'][np.isnan(links['clangs']) == 0],
35)
for lid, cld, cla, lg, lga, cert in zip(links['id'], links['cldists'],
links['clangs'], links['guess'],
links['guess_alg'],
links['certain']):
if cert == 1:
continue
if cld >= cl_distthresh and cla < ang_thresh:
linkidx = links['id'].index(lid)
if dy.algmap('cl_dist_guess') in lga and dy.algmap(
'cl_ang_guess') in lga:
if lg[lga.index(dy.algmap('cl_dist_guess'))] == lg[lga.index(
dy.algmap('cl_ang_guess'))]:
usnode = lg[lga.index(dy.algmap('cl_dist_guess'))]
links, nodes = dy.set_link(links, nodes, linkidx, usnode,
alg)
# Set directions by most-certain angles
angthreshs = np.linspace(0, 0.4, 10)
for a in angthreshs:
links, nodes = dy.set_by_known_flow_directions(links,
nodes,
imshape,
angthresh=a,
lenthresh=3)
# Set using direction of nearest main channel
links, nodes = dy.set_by_nearest_main_channel(links,
nodes,
imshape,
nodethresh=1)
angthreshs = np.linspace(0, 1.5, 20)
for a in angthreshs:
links, nodes = dy.set_by_known_flow_directions(links,
nodes,
imshape,
angthresh=a)
# At this point, if any links remain unset, they are just set randomly
if np.sum(links['certain']) != len(links['id']):
print('{} links were randomly set.'.format(
len(links['id']) - np.sum(links['certain'])))
links['certain'] = np.ones((len(links['id']), 1))
# Check for and try to fix cycles in the graph
links, nodes, cantfix_cyclelinks, cantfix_cyclenodes = fix_river_cycles(
links, nodes, imshape)
# Check for sources or sinks within the graph
cont_violators = dy.check_continuity(links, nodes)
# Summary of problems:
manual_fix = 0
if len(cantfix_cyclelinks) > 0:
print('Could not fix cycle links: {}.'.format(cantfix_cyclelinks))
manual_fix = 1
else:
print('All cycles were resolved.')
if len(cont_violators) > 0:
print('Continuity violated at nodes {}.'.format(cont_violators))
manual_fix = 1
# Create a csv to store manual edits to directionality if does not exist
if manual_fix == 1:
if os.path.isfile(manual_set_csv) is False:
io.create_manual_dir_csv(manual_set_csv)
print('A .csv file for manual fixes to link directions at {}.'.
format(manual_set_csv))
else:
print('Use the csv file at {} to manually fix link directions.'.
format(manual_set_csv))
return links, nodes
