def write_v_map(self, name, remove=False, voltages=None, node_map=None):
"""Write voltage map identified by name. Remove the map after that if remove is True.
If voltages and node_map are provided, create space for voltage map disregarding prior allocation.
"""
if self.report_status==True:
ComputeBase.logger.info('writing voltage map ' + name)
if self.is_network:
if voltages is None:
vm = self.voltage_maps[name]
nn = self.node_names
else:
vm = voltages
nn = node_map
CSIO.write_voltages(self.options.output_file, vm, nn, name)
else:
fileadd = name if (name=='') else ('_'+name)
if voltages is None:
vm = self.voltage_maps[name]
else:
vm = self._create_voltage_map(node_map, voltages)
CSIO.write_aaigrid('voltmap', fileadd, vm, self.options, self.state)
if remove:
self.rm_v_map(name)
elif voltages is not None:
if self.is_network:
self.alloc_v_map(name)
self.accumulate_v_map(name, voltages, node_map)
else:
self.voltage_maps[name] = vm
def write_v_map(self, name, remove=False, voltages=None, node_map=None):
"""Write voltage map identified by name. Remove the map after that if remove is True.
If voltages and node_map are provided, create space for voltage map disregarding prior allocation.
"""
if self.report_status == True:
ComputeBase.logger.info('writing voltage map ' + name)
if self.is_network:
if voltages is None:
vm = self.voltage_maps[name]
nn = self.node_names
else:
vm = voltages
nn = node_map
CSIO.write_voltages(self.options.output_file, vm, nn, name)
else:
fileadd = name if (name == '') else ('_' + name)
if voltages is None:
vm = self.voltage_maps[name]
else:
vm = self._create_voltage_map(node_map, voltages)
CSIO.write_aaigrid('voltmap', fileadd, vm, self.options,
self.state)
if remove:
self.rm_v_map(name)
elif voltages is not None:
if self.is_network:
self.alloc_v_map(name)
self.accumulate_v_map(name, voltages, node_map)
else:
self.voltage_maps[name] = vm
def _write_store_c_map(self, name, remove, write, accumulate, voltages, G, node_map, finitegrounds, local_src, local_dst):
if self.is_network:
if voltages is not None:
(node_currents, branch_currents) = self._create_current_maps(voltages, G, finitegrounds)
branch_currents_array = Output._convert_graph_to_3_col(branch_currents, node_map)
else:
branch_currents, node_currents, branch_currents_array, _node_map = self.current_maps[name]
if write:
# Append node names and convert to array format
node_currents_array = Output._append_names_to_node_currents(node_currents, node_map)
# node_currents_array = Output._append_names_to_node_currents(node_currents, self.nn_sorted)# Fixme: Need something like this for multipe components in network mode?
CSIO.write_currents(self.options.output_file, branch_currents_array, node_currents_array, name, self.options)
if remove:
self.rm_c_map(name)
elif accumulate:
full_branch_currents, full_node_currents, _bca, _np = self.current_maps[name]
pos = np.searchsorted(self.nn_sorted, node_map)
full_node_currents[pos] += node_currents
bc_x = np.searchsorted(self.nn_sorted, branch_currents_array[:,0])
bc_y = np.searchsorted(self.nn_sorted, branch_currents_array[:,1])
full_branch_currents = full_branch_currents + sparse.csr_matrix((branch_currents_array[:,2], (bc_x, bc_y)), shape=full_branch_currents.shape)
self.current_maps[name] = (full_branch_currents, full_node_currents, branch_currents_array, node_map)
else:
self.current_maps[name] = (branch_currents, node_currents, branch_currents_array, node_map)
else:
if voltages is not None:
while LowMemRetry.retry():
with LowMemRetry():
current_map = self._create_current_maps(voltages, G, finitegrounds, node_map)
else:
current_map = self.current_maps[name]
if (self.options.set_focal_node_currents_to_zero==True) and (local_src is not None) and (local_dst is not None):
# set source and target node currents to zero
focal_node_pair_map = np.where(node_map == local_src+1, 0, 1)
focal_node_pair_map = np.where(node_map == local_dst+1, 0, focal_node_pair_map)
current_map = np.multiply(focal_node_pair_map, current_map)
del focal_node_pair_map
if write:
if name=='' and self.scenario != 'advanced':
curMapName = 'cum_curmap' #For backward compatibility- ArcGIS
else:
curMapName = 'curmap'
fileadd = name if (name=='') else ('_'+name)
CSIO.write_aaigrid(curMapName, fileadd, self._log_transform(current_map), self.options, self.state)
if remove:
self.rm_c_map(name)
elif accumulate:
self.current_maps[name] += current_map
else:
self.current_maps[name] = current_map
def compute_network(self):
"""Solves arbitrary graphs instead of raster grids."""
(g_graph, node_names) = self.read_graph(self.options.habitat_file)
fp = None
if self.options.scenario == 'pairwise':
if self.options.use_included_pairs==True:
self.state.included_pairs = IncludeExcludePairs(self.options.included_pairs_file)
focal_nodes = self.read_focal_nodes(self.options.point_file)
fp = FocalPoints(focal_nodes, self.state.included_pairs, True)
elif self.options.scenario == 'advanced':
self.state.source_map = CSIO.read_point_strengths(self.options.source_file)
self.state.ground_map = CSIO.read_point_strengths(self.options.ground_file)
g_habitat = HabitatGraph(g_graph=g_graph, node_names=node_names)
out = Output(self.options, self.state, False, node_names)
if self.options.write_cur_maps:
out.alloc_c_map('')
Compute.logger.info('Calling solver module.')
Compute.logger.info('Graph has ' + str(g_habitat.num_nodes) + ' nodes and '+ str(g_habitat.num_components)+ ' components.')
if self.options.scenario == 'pairwise':
(resistances, solver_failed) = self.single_ground_all_pair_resistances(g_habitat, fp, out, True)
if self.options.write_cur_maps:
full_branch_currents, full_node_currents, _bca, _np = out.get_c_map('')
_resistances, resistances_3col = self.write_resistances(fp.point_ids, resistances)
result1 = resistances_3col
elif self.options.scenario == 'advanced':
self.options.write_max_cur_maps = False
voltages, current_map, solver_failed = self.advanced_module(g_habitat, out, self.state.source_map, self.state.ground_map)
if self.options.write_cur_maps:
full_branch_currents, full_node_currents, _bca, _np = current_map
result1 = voltages
if solver_failed == True:
Compute.logger.error('Solver failed')
if self.options.write_cur_maps:
full_branch_currents = Output._convert_graph_to_3_col(full_branch_currents, node_names)
full_node_currents = Output._append_names_to_node_currents(full_node_currents, node_names)
ind = np.lexsort((full_branch_currents[:, 1], full_branch_currents[:, 0]))
full_branch_currents = full_branch_currents[ind]
ind = np.lexsort((full_node_currents[:, 1], full_node_currents[:, 0]))
full_node_currents = full_node_currents[ind]
CSIO.write_currents(self.options.output_file, full_branch_currents, full_node_currents, '',self.options)
return result1, solver_failed
def read_graph(self, filename):
"""Reads arbitrary graph from disk. Returns sparse adjacency matrix and node names ."""
graph_list = CSIO.load_graph(filename)
try:
zeros_in_resistance_graph = False
nodes = ComputeBase.deletecol(graph_list,2)
node_names = np.unique(np.asarray(nodes, dtype='int32'))
nodes[np.where(nodes>= 0)] = ComputeBase.relabel(nodes[np.where(nodes>= 0)], 0)
node1 = nodes[:,0]
node2 = nodes[:,1]
data = graph_list[:,2]
######################## Reclassification code
if self.options.use_reclass_table == True:
try:
reclass_table = CSIO.read_point_strengths(self.options.reclass_file)
except:
raise RuntimeError('Error reading reclass table. Please check file format.')
for i in range (0,reclass_table.shape[0]):
data = np.where(data==reclass_table[i,0], reclass_table[i,1],data)
Compute.logger.info('Reclassified landscape graph using %s'%(self.options.reclass_file,))
########################
if self.options.habitat_map_is_resistances == True:
zeros_in_resistance_graph = (np.where(data==0, 1, 0)).sum() > 0
conductances = 1/data
else:
conductances = data
numnodes = node_names.shape[0]
G = sparse.csr_matrix((conductances, (node1, node2)), shape = (numnodes, numnodes))
Gdense=G.todense()
g_graph = np.maximum(Gdense, Gdense.T) # To handle single or double entries for elements BHM 06/28/11
g_graph = sparse.csr_matrix(g_graph)
except:
raise RuntimeError('Error processing graph/network file. Please check file format')
if zeros_in_resistance_graph == True:
raise RuntimeError('Error: zero resistance values are not currently allowed in network/graph input file.')
return g_graph, node_names
def read_focal_nodes(self, filename):
"""Loads list of focal nodes for arbitrary graph."""
focal_nodes = CSIO.load_graph(filename)
try:
if filename == self.options.habitat_file:#If graph was used as focal node file, then just want first two columns for focal_nodes.
focal_nodes = ComputeBase.deletecol(focal_nodes, 2)
focal_nodes = np.unique(np.asarray(focal_nodes))
except:
raise RuntimeError('Error processing focal node file. Please check file format')
return focal_nodes
def load_maps(self):
"""Loads all raster maps into self.state."""
Compute.logger.info('Reading maps')
reclass_file = self.options.reclass_file if self.options.use_reclass_table else None
CSIO.read_cell_map(self.options.habitat_file, self.options.habitat_map_is_resistances, reclass_file, self.state)
if self.options.use_polygons:
self.state.poly_map = CSIO.read_poly_map(self.options.polygon_file, False, 0, self.state, True, "Short-circuit region", 'int32')
else:
self.state.poly_map = []
if self.options.use_mask==True:
mask = CSIO.read_poly_map(self.options.mask_file, True, 0, self.state, True, "Mask", 'int32')
mask = np.where(mask != 0, 1, 0)
self.state.g_map = np.multiply(self.state.g_map, mask)
del mask
sum_gmap = (self.state.g_map).sum()
if sum_gmap==0:
raise RuntimeError('All entries in landscape map have been dropped after masking with the mask file. There is nothing to solve.')
else:
self.state.mask = []
if self.options.scenario=='advanced':
self.state.points_rc = []
(self.state.source_map, self.state.ground_map) = CSIO.read_source_and_ground_maps(self.options.source_file, self.options.ground_file, self.state, self.options)
else:
self.state.points_rc = CSIO.read_point_map(self.options.point_file, "Focal node", self.state)
self.state.source_map = []
self.state.ground_map = []
if self.options.use_included_pairs==True:
self.state.included_pairs = IncludeExcludePairs(self.options.included_pairs_file)
self.state.point_strengths = None
if self.options.use_variable_source_strengths==True:
self.state.point_strengths = CSIO.read_point_strengths(self.options.variable_source_file)
Compute.logger.info('Processing maps')
return
def __init__(self, filename):
mode, point_ids, mat = CSIO.read_included_pairs(filename)
self.is_include = (mode == "include")
self.point_ids = point_ids
self.mat = mat.tocoo()
self.max_id = int(np.max(point_ids)) + 1
def write_resistances(self, point_ids, resistances):
"""Writes resistance file to disk."""
outputResistances = self.append_names_to_resistances(point_ids, resistances)
resistances3Columns = self.convertResistances3cols(outputResistances)
CSIO.write_resistances(self.options.output_file, outputResistances, resistances3Columns)
return outputResistances, resistances3Columns
def single_ground_all_pair_resistances(self, g_habitat, fp, cs, report_status):
"""Handles pairwise resistance/current/voltage calculations.
Called once when focal points are used, called multiple times when focal regions are used.
"""
options = self.options
last_write_time = time.time()
numpoints = fp.num_points()
parallelize = options.parallelize
# TODO: revisit to see if restriction can be removed
if options.low_memory_mode==True or self.state.point_file_contains_polygons==True:
parallelize = False
if (self.state.point_file_contains_polygons == True) or (options.write_cur_maps == True) or (options.write_volt_maps == True) or (options.use_included_pairs==True) or (options.data_type=='network'):
use_resistance_calc_shortcut = False
else:
use_resistance_calc_shortcut = True # We use this when there are no focal regions. It saves time when we are also not creating maps
shortcut_resistances = -1 * np.ones((numpoints, numpoints), dtype='float64')
solver_failed_somewhere = [False]
Compute.logger.info('Graph has ' + str(g_habitat.num_nodes) + ' nodes, ' + str(numpoints) + ' focal nodes and '+ str(g_habitat.num_components)+ ' components.')
resistances = -1 * np.ones((numpoints, numpoints), dtype = 'float64') #Inf creates trouble in python 2.5 on Windows. Use -1 instead.
# if use_resistance_calc_shortcut==True:
# num_points_to_solve = numpoints
# else:
# num_points_to_solve = numpoints*(numpoints-1)/2
num_points_to_solve = 0
max_parallel = 0
for c in range(1, int(g_habitat.num_components+1)):
if not fp.exists_points_in_component(c, g_habitat):
continue
num_parallel = 0
for (pt1_idx, pt2_idx) in fp.point_pair_idxs_in_component(c, g_habitat):
if pt2_idx == -1:
if (use_resistance_calc_shortcut==True):
max_parallel = max(max_parallel, num_parallel)
num_parallel = 0
break
else:
max_parallel = max(max_parallel, num_parallel)
num_parallel = 0
continue
num_parallel += 1
num_points_to_solve += 1
Compute.logger.debug('max parallel possible = ' + str(max_parallel) + ', will parallelize = ' + str(parallelize))
num_points_solved = 0
for c in range(1, int(g_habitat.num_components+1)):
if not fp.exists_points_in_component(c, g_habitat):
continue
G_pruned, local_node_map = g_habitat.prune_nodes_for_component(c)
G = ComputeBase.laplacian(G_pruned)
del G_pruned
if use_resistance_calc_shortcut:
voltmatrix = np.zeros((numpoints,numpoints), dtype='float64') #For resistance calc shortcut
G_dst_dst = local_dst = None
for (pt1_idx, pt2_idx) in fp.point_pair_idxs_in_component(c, g_habitat):
if pt2_idx == -1:
if parallelize:
self.state.worker_pool_wait()
self.state.del_amg_hierarchy()
if (local_dst != None) and (G_dst_dst != None):
G[local_dst, local_dst] = G_dst_dst
local_dst = G_dst_dst = None
if (use_resistance_calc_shortcut==True):
Compute.get_shortcut_resistances(pt1_idx, voltmatrix, numpoints, resistances, shortcut_resistances)
break #No need to continue, we've got what we need to calculate resistances
else:
continue
if parallelize:
self.state.worker_pool_create(options.max_parallel, True)
if report_status==True:
num_points_solved += 1
if use_resistance_calc_shortcut==True:
Compute.logger.info('solving focal node ' + str(num_points_solved) + ' of '+ str(num_points_to_solve))
else:
Compute.logger.info('solving focal pair ' + str(num_points_solved) + ' of '+ str(num_points_to_solve))
local_src = fp.get_graph_node_idx(pt2_idx, local_node_map)
if None == local_dst:
local_dst = fp.get_graph_node_idx(pt1_idx, local_node_map)
G_dst_dst = G[local_dst, local_dst]
G[local_dst, local_dst] = 0
if self.state.amg_hierarchy == None:
self.state.create_amg_hierarchy(G, self.options.solver)
if use_resistance_calc_shortcut:
post_solve = self._post_single_ground_solve(G, fp, cs, resistances, numpoints, pt1_idx, pt2_idx, local_src, local_dst, local_node_map, solver_failed_somewhere, use_resistance_calc_shortcut, voltmatrix)
else:
post_solve = self._post_single_ground_solve(G, fp, cs, resistances, numpoints, pt1_idx, pt2_idx, local_src, local_dst, local_node_map, solver_failed_somewhere)
if parallelize:
self.state.worker_pool.apply_async(Compute.parallel_single_ground_solver, args=(G, local_src, local_dst, options.solver, self.state.amg_hierarchy), callback=post_solve)
#post_solve(self.state.worker_pool.apply(Compute.parallel_single_ground_solver, args=(G, local_src, local_dst, options.solver, self.state.amg_hierarchy)))
else:
try:
voltages = Compute.single_ground_solver(G, local_src, local_dst, options.solver, self.state.amg_hierarchy)
except:
voltages = None
post_solve(voltages)
if options.low_memory_mode==True or self.state.point_file_contains_polygons==True:
self.state.del_amg_hierarchy()
(hours,mins,_secs) = ComputeBase.elapsed_time(last_write_time)
if mins > 2 or hours > 0:
last_write_time = time.time()
CSIO.write_resistances(options.output_file, resistances, incomplete=True)# Save incomplete resistances
self.state.del_amg_hierarchy()
# Finally, resistance to self is 0.
if use_resistance_calc_shortcut==True:
resistances = shortcut_resistances
for i in range(0, numpoints):
resistances[i, i] = 0
return resistances, solver_failed_somewhere[0]
def one_to_all_module(self, g_map, poly_map, points_rc):
"""Overhead module for one-to-all AND all-to-one modes with raster data."""
last_write_time = time.time()
out = Output(self.options, self.state, False)
if self.options.write_cur_maps:
out.alloc_c_map('')
if self.options.write_max_cur_maps:
out.alloc_c_map('max')
fp = FocalPoints(points_rc, self.state.included_pairs, False)
fp = FocalPoints(fp.get_unique_coordinates(), self.state.included_pairs, False)
point_ids = fp.point_ids
points_rc_unique = fp.points_rc
included_pairs = self.state.included_pairs
resistance_vector = np.zeros((point_ids.size,2), float)
solver_failed_somewhere = False
if self.options.use_included_pairs==False: #Will do this each time later if using included pairs
point_map = np.zeros((self.state.nrows, self.state.ncols), int)
point_map[points_rc[:,1], points_rc[:,2]] = points_rc[:,0]
#combine point map and poly map
poly_map_temp = self.get_poly_map_temp(poly_map, point_map, point_ids)
unique_point_map = np.zeros((self.state.nrows, self.state.ncols), int)
unique_point_map[points_rc_unique[:,1], points_rc_unique[:,2]] = points_rc_unique[:,0]
(strength_map, strengths_rc) = self.get_strength_map(points_rc_unique, self.state.point_strengths)
g_habitat = HabitatGraph(g_map=g_map, poly_map=poly_map_temp, connect_using_avg_resistances=self.options.connect_using_avg_resistances, connect_four_neighbors_only=self.options.connect_four_neighbors_only)
Compute.logger.info('Graph has ' + str(g_habitat.num_nodes) + ' nodes and '+ str(g_habitat.num_components) + ' components.')
component_with_points = g_habitat.unique_component_with_points(unique_point_map)
else:
if Compute.logger.isEnabledFor(logging.DEBUG):
g_habitat = HabitatGraph(g_map=g_map, poly_map=poly_map, connect_using_avg_resistances=self.options.connect_using_avg_resistances, connect_four_neighbors_only=self.options.connect_four_neighbors_only)
Compute.logger.info('Graph has ' + str(g_habitat.num_nodes) + ' nodes and '+ str(g_habitat.num_components) + ' components.')
component_with_points = None
for pt_idx in range(0, point_ids.size): # These are the 'src' nodes, pt_idx.e. the 'one' in all-to-one and one-to-all
Compute.logger.info('solving focal node ' + str(pt_idx+1) + ' of ' + str(point_ids.size))
if self.options.use_included_pairs==True: # Done above otherwise
#######################
points_rc_unique_temp = np.copy(points_rc_unique)
point_map = np.zeros((self.state.nrows, self.state.ncols), int)
point_map[points_rc[:,1], points_rc[:,2]] = points_rc[:,0]
#loop thru exclude[point,:], delete included pairs of focal point from point_map and points_rc_unique_temp
for pair in range(0, point_ids.size):
if (pt_idx != pair) and not self.state.included_pairs.is_included_pair(point_ids[pt_idx], point_ids[pair]):
pt_id = point_ids[pair]
point_map = np.where(point_map==pt_id, 0, point_map)
points_rc_unique_temp[pair, 0] = 0 #point will not be burned in to unique_point_map
poly_map_temp = self.get_poly_map_temp2(poly_map, point_map, points_rc_unique_temp, included_pairs, pt_idx)
g_habitat = HabitatGraph(g_map=g_map, poly_map=poly_map_temp, connect_using_avg_resistances=self.options.connect_using_avg_resistances, connect_four_neighbors_only=self.options.connect_four_neighbors_only)
unique_point_map = np.zeros((self.state.nrows, self.state.ncols),int)
unique_point_map[points_rc_unique_temp[:,1], points_rc_unique_temp[:,2]] = points_rc_unique_temp[:,0]
(strength_map, strengths_rc) = self.get_strength_map(points_rc_unique_temp, self.state.point_strengths)
###########################################
src = point_ids[pt_idx]
if unique_point_map.sum() == src: # src is the only point
resistance_vector[pt_idx,0] = src
resistance_vector[pt_idx,1] = -1
else: # there are points to connect with src point
if self.options.scenario == 'one-to-all':
strength = strengths_rc[pt_idx,0] if self.options.use_variable_source_strengths else 1
source_map = np.where(unique_point_map == src, strength, 0)
ground_map = np.where(unique_point_map == src, 0, unique_point_map)
ground_map = np.where(ground_map, np.Inf, 0)
self.options.remove_src_or_gnd = 'rmvgnd'
else: # all-to-one
if self.options.use_variable_source_strengths==True:
source_map = np.where(unique_point_map == src, 0, strength_map)
else:
source_map = np.where(unique_point_map, 1, 0)
source_map = np.where(unique_point_map == src, 0, source_map)
ground_map = np.where(unique_point_map == src, np.Inf, 0)
self.options.remove_src_or_gnd = 'rmvsrc'
#FIXME: right now one-to-all *might* fail if there is just one node that is not grounded (I haven't encountered this problem lately BHM Nov 2009).
resistance, current_map, solver_failed = self.advanced_module(g_habitat, out, source_map, ground_map, src, component_with_points)
if not solver_failed:
if self.options.write_cur_maps:
out.accumulate_c_map_with_values('', current_map)
if self.options.write_max_cur_maps:
out.store_max_c_map_values('max', current_map)
else:
Compute.logger.warning('Solver failed for at least one focal node. Focal nodes with failed solves will be marked with value of -777 in output resistance list.')
resistance_vector[pt_idx,0] = src
resistance_vector[pt_idx,1] = resistance
if solver_failed==True:
solver_failed_somewhere = True
(hours,mins,_secs) = ComputeBase.elapsed_time(last_write_time)
if mins > 2 or hours > 0:
last_write_time = time.time()
CSIO.write_resistances(self.options.output_file, resistance_vector, incomplete=True)
if not solver_failed_somewhere:
if self.options.write_cur_maps:
out.write_c_map('', True)
if self.options.write_max_cur_maps:
out.write_c_map('max', True)
CSIO.write_resistances(self.options.output_file, resistance_vector)
return resistance_vector, solver_failed_somewhere
def __init__(self, filename):
mode, point_ids, mat = CSIO.read_included_pairs(filename)
self.is_include = (mode == "include")
self.point_ids = point_ids
self.mat = mat.tocoo()
self.max_id = int(np.max(point_ids)) + 1
