def rec_submesh(p, verbose=True):
from anuga.utilities import parallel_abstraction as pypar
numproc = pypar.size()
myid = pypar.rank()
[
submesh_cell, triangles_per_proc, number_of_full_nodes,
number_of_full_triangles
] = rec_submesh_flat(p, verbose)
# find the full triangles assigned to this processor
lower_t = 0
for i in range(myid):
lower_t = lower_t + triangles_per_proc[i]
upper_t = lower_t + triangles_per_proc[myid]
# convert the information into a form needed by the GA
# datastructure
[GAnodes, GAtriangles, boundary, quantities,
ghost_rec, full_send,
tri_map, node_map, tri_l2g, node_l2g,
ghost_layer_width] = \
build_local_mesh(submesh_cell, lower_t, upper_t, numproc)
return GAnodes, GAtriangles, boundary, quantities,\
ghost_rec, full_send,\
number_of_full_nodes, number_of_full_triangles, tri_map, node_map,\
tri_l2g, node_l2g, ghost_layer_width
def __init__(self,
coordinates,
vertices,
boundary=None,
full_send_dict=None,
ghost_recv_dict=None,
velocity=None):
Domain.__init__(self,
coordinates,
vertices,
boundary,
velocity=velocity,
full_send_dict=full_send_dict,
ghost_recv_dict=ghost_recv_dict,
processor=pypar.rank(),
numproc=pypar.size())
N = self.number_of_elements
self.communication_time = 0.0
self.communication_reduce_time = 0.0
print('processor', self.processor)
print('numproc', self.numproc)
def print_l2_stats(full_edge):
numprocs = pypar.size()
myid = pypar.rank()
tri_norm = zeros(3, Float)
recv_norm = zeros(3, Float)
tri_norm[0] = pow(l2_norm(full_edge[:, 0]), 2)
tri_norm[1] = pow(l2_norm(full_edge[:, 1]), 2)
tri_norm[2] = pow(l2_norm(full_edge[:, 2]), 2)
if myid == 0:
for p in range(numprocs - 1):
pypar.receive(p + 1, recv_norm)
tri_norm[0] = tri_norm[0] + recv_norm[0]
tri_norm[1] = tri_norm[1] + recv_norm[1]
tri_norm[2] = tri_norm[2] + recv_norm[2]
print('l2_norm along each axis : [', pow(tri_norm[0], 0.5),', ', pow(tri_norm[1], 0.5), \
', ', pow(tri_norm[2], 0.5), ']')
else:
pypar.send(tri_norm, 0)
def print_linf_stats(full_edge):
numprocs = pypar.size()
myid = pypar.rank()
tri_norm = zeros(3, Float)
recv_norm = zeros(3, Float)
tri_norm[0] = linf_norm(full_edge[:, 0])
tri_norm[1] = linf_norm(full_edge[:, 1])
tri_norm[2] = linf_norm(full_edge[:, 2])
if myid == 0:
for p in range(numprocs - 1):
pypar.receive(p + 1, recv_norm)
tri_norm[0] = max(tri_norm[0], recv_norm[0])
tri_norm[1] = max(tri_norm[1], recv_norm[1])
tri_norm[2] = max(tri_norm[2], recv_norm[2])
print('linf_norm along each axis : [', tri_norm[0], ', ', tri_norm[1],
', ', tri_norm[2], ']')
else:
pypar.send(tri_norm, 0)
def send_submesh(submesh, triangles_per_proc, p, verbose=True):
from anuga.utilities import parallel_abstraction as pypar
myid = pypar.rank()
nprocs = pypar.size()
if verbose:
print('P%d: Sending submesh to P%d' % (myid, p))
# build and send the tagmap for the boundary conditions
tagmap = {}
counter = 1
for b in submesh["full_boundary"][p]:
bkey = submesh["full_boundary"][p][b]
if bkey not in tagmap:
tagmap[bkey] = counter
counter = counter + 1
for b in submesh["ghost_boundary"][p]:
bkey = submesh["ghost_boundary"][p][b]
if bkey not in tagmap:
tagmap[bkey] = counter
counter = counter + 1
# send boundary tags
pypar.send(tagmap, p)
# send the quantities key information
pypar.send(list(submesh["full_quan"].keys()), p)
# compress full_commun
flat_full_commun = []
for c in submesh["full_commun"][p]:
for i in range(len(submesh["full_commun"][p][c])):
flat_full_commun.append([c, submesh["full_commun"][p][c][i]])
# send the array sizes so memory can be allocated
setup_array = num.zeros((9, ), num.int)
setup_array[0] = len(submesh["full_nodes"][p])
setup_array[1] = len(submesh["ghost_nodes"][p])
setup_array[2] = len(submesh["full_triangles"][p])
setup_array[3] = len(submesh["ghost_triangles"][p])
setup_array[4] = len(submesh["full_boundary"][p])
setup_array[5] = len(submesh["ghost_boundary"][p])
setup_array[6] = len(submesh["ghost_commun"][p])
setup_array[7] = len(flat_full_commun)
setup_array[8] = len(submesh["full_quan"])
x = num.array(setup_array, num.int)
pypar.send(x, p, bypass=True)
# ghost layer width
x = num.array(submesh["ghost_layer_width"][p], num.int)
pypar.send(x, p, bypass=True)
# send the number of triangles per processor
x = num.array(triangles_per_proc, num.int)
pypar.send(x, p, bypass=True)
# send the nodes
x = num.array(submesh["full_nodes"][p], num.float)
pypar.send(x, p, bypass=True)
x = num.array(submesh["ghost_nodes"][p], num.float)
pypar.send(x, p, bypass=True)
# send the triangles
x = num.array(submesh["full_triangles"][p], num.int)
pypar.send(x, p, bypass=True)
# send ghost triangles
x = num.array(submesh["ghost_triangles"][p], num.int)
pypar.send(x, p, bypass=True)
# send the boundary
bc = []
for b in submesh["full_boundary"][p]:
bc.append([b[0], b[1], tagmap[submesh["full_boundary"][p][b]]])
x = num.array(bc, num.int)
pypar.send(x, p, bypass=True)
bc = []
for b in submesh["ghost_boundary"][p]:
bc.append([b[0], b[1], tagmap[submesh["ghost_boundary"][p][b]]])
x = num.array(bc, num.int)
pypar.send(x, p, bypass=True)
# send the communication pattern
x = num.array(submesh["ghost_commun"][p], num.int)
pypar.send(x, p, bypass=True)
x = num.array(flat_full_commun, num.int)
pypar.send(x, p, bypass=True)
# send the quantities
for k in submesh["full_quan"]:
x = num.array(submesh["full_quan"][k][p], num.float)
pypar.send(x, p, bypass=True)
for k in submesh["ghost_quan"]:
x = num.array(submesh["ghost_quan"][k][p], num.float)
pypar.send(x, p, bypass=True)
def distribute_mesh(domain, verbose=False, debug=False, parameters=None):
""" Distribute andsend the mesh info to all the processors.
Should only be run from processor 0 and will send info to the other
processors.
There should be a corresponding rec_submesh called from all the other
processors
"""
if debug:
verbose = True
numprocs = size()
# Subdivide the mesh
if verbose: print 'Subdivide mesh'
new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
s2p_map, p2s_map = \
pmesh_divide_metis_with_map(domain, numprocs)
#PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
# sp2_map (maps parallel domain triangles to domain triangles)
# Build the mesh that should be assigned to each processor,
# this includes ghost nodes and the communication pattern
if verbose: print 'Build submeshes'
submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities,
triangles_per_proc, parameters)
if verbose:
for p in range(numprocs):
N = len(submesh['ghost_nodes'][p])
M = len(submesh['ghost_triangles'][p])
print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
%(N, M, p)
#if debug:
# from pprint import pprint
# pprint(submesh)
# Send the mesh partition to the appropriate processor
if verbose: print 'Distribute submeshes'
for p in range(1, numprocs):
send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose)
# Build the local mesh for processor 0
points, vertices, boundary, quantities, \
ghost_recv_dict, full_send_dict, \
tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
extract_submesh(submesh, triangles_per_proc, p2s_map, 0)
# Keep track of the number full nodes and triangles.
# This is useful later if one needs access to a ghost-free domain
# Here, we do it for process 0. The others are done in rec_submesh.
number_of_full_nodes = len(submesh['full_nodes'][0])
number_of_full_triangles = len(submesh['full_triangles'][0])
#print
#for p in range(numprocs):
# print 'Process %d:' %(p)
#
# print 'full_triangles:'
# print submesh['full_triangles'][p]
#
# print 'full_nodes:'
# print submesh['full_nodes'][p]
#
# print 'ghost_triangles:'
# print submesh['ghost_triangles'][p]#
#
# print 'ghost_nodes:'
# print submesh['ghost_nodes'][p]
# print
#
#print 'Receive dict'
#print ghost_recv_dict
#
#print 'Send dict'
#print full_send_dict
# Return structures necessary for building the parallel domain
return points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles, \
s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width
from anuga.parallel.distribute_mesh import rec_submesh
from anuga.parallel.distribute_mesh import extract_submesh
# Mesh partitioning using Metis
from anuga.parallel.distribute_mesh import build_submesh
from anuga.parallel.distribute_mesh import pmesh_divide_metis_with_map
from anuga.parallel.parallel_shallow_water import Parallel_domain
from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh
#------------------------------------------------------------------------------
# Read in processor information
#------------------------------------------------------------------------------
numprocs = size()
myid = rank()
processor_name = get_processor_name()
#print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name)
def collect_value(value):
value = value
if myid == 0:
for i in range(numprocs):
if i == 0: continue
val = receive(i)
value = value + val
else:
def allocate_inlet_procs(domain,
poly,
enquiry_point=None,
master_proc=0,
procs=None,
verbose=False):
from anuga.utilities import parallel_abstraction as pypar
if procs is None:
procs = list(range(0, pypar.size()))
myid = pypar.rank()
vertex_coordinates = domain.get_full_vertex_coordinates(absolute=True)
domain_centroids = domain.centroid_coordinates
size = 0
has_enq_point = False
numprocs = pypar.size()
inlet_procs = []
max_size = -1
inlet_master_proc = -1
inlet_enq_proc = -1
# Calculate the number of points of the line inside full polygon
#tri_id = line_intersect(vertex_coordinates, poly)
if len(poly) == 2: # poly is a line
if verbose: print("======================")
tri_id = line_intersect(vertex_coordinates, poly)
else: # poly is a polygon
if verbose: print("+++++++++++++++++++++++")
tris_0 = line_intersect(vertex_coordinates, [poly[0], poly[1]])
tris_1 = inside_polygon(domain_centroids, poly)
tri_id = num.union1d(tris_0, tris_1)
if verbose:
print("P%d has %d triangles in poly %s" % (myid, len(tri_id), poly))
size = len(tri_id)
if enquiry_point is not None:
try:
k = domain.get_triangle_containing_point(enquiry_point)
if domain.tri_full_flag[k] == 1:
size = size + 1
has_enq_point = True
if verbose:
print("P%d has enq point %s" % (myid, enquiry_point))
else:
if verbose:
print("P%d contains ghost copy of enq point %s" %
(myid, enquiry_point))
has_enq_point = False
except:
if verbose:
print("P%d does not contain enq point %s" %
(myid, enquiry_point))
has_enq_point = False
if myid == master_proc:
# Recieve size of overlap from each processor
# Initialize line_master_proc and inlet_procs
if size > 0:
inlet_procs = [master_proc]
max_size = size
inlet_master_proc = master_proc
if has_enq_point:
inlet_enq_proc = master_proc
# Recieve size of overlap
for i in procs:
if i == master_proc: continue
x = pypar.receive(i)
y = pypar.receive(i)
if x > 0:
inlet_procs.append(i)
# Choose inlet_master_proc as the one with the most overlap
if x > max_size:
max_size = x
inlet_master_proc = i
if y is True:
assert inlet_enq_proc == -1, "Enquiry point correspond to more than one proc"
inlet_enq_proc = i
assert len(inlet_procs) > 0, "Line does not intersect any domain"
assert inlet_master_proc >= 0, "No master processor assigned"
if enquiry_point is not None:
msg = "Enquiry point %s doesn't intersect mesh, maybe inside a building, try reducing enquiry_gap" % str(
enquiry_point)
if inlet_enq_proc < 0:
raise Exception(msg)
# Send inlet_master_proc and inlet_procs to all processors in inlet_procs
for i in procs:
if i != master_proc:
pypar.send(inlet_master_proc, i)
pypar.send(inlet_procs, i)
pypar.send(inlet_enq_proc, i)
else:
pypar.send(size, master_proc)
pypar.send(has_enq_point, master_proc)
inlet_master_proc = pypar.receive(master_proc)
inlet_procs = pypar.receive(master_proc)
inlet_enq_proc = pypar.receive(master_proc)
if has_enq_point:
assert inlet_enq_proc == myid, "Enquiry found in proc, but not declared globally"
if size > 0:
return True, inlet_master_proc, inlet_procs, inlet_enq_proc
else:
return False, inlet_master_proc, inlet_procs, inlet_enq_proc
def Internal_boundary_operator(domain,
internal_boundary_function,
width=1.,
height=1.,
end_points=None,
exchange_lines=None,
enquiry_points=None,
invert_elevation=None,
apron=0.0,
enquiry_gap=0.0,
use_velocity_head=False,
zero_outflow_momentum=False,
force_constant_inlet_elevations=True,
smoothing_timescale=0.0,
compute_discharge_implicitly=True,
description=None,
label=None,
structure_type='internal_boundary',
logging=False,
verbose=True,
master_proc=0,
procs=None,
inlet_master_proc=[0, 0],
inlet_procs=None,
enquiry_proc=[0, 0]):
# If not parallel domain then allocate serial Internal boundary operator
if isinstance(domain, Parallel_domain) is False:
if verbose:
print("Allocating non parallel internal_boundary operator .....")
return anuga.structures.internal_boundary_operator.Internal_boundary_operator(
domain=domain,
internal_boundary_function=internal_boundary_function,
width=width,
height=height,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevation=invert_elevation,
apron=apron,
enquiry_gap=enquiry_gap,
use_velocity_head=use_velocity_head,
zero_outflow_momentum=zero_outflow_momentum,
force_constant_inlet_elevations=force_constant_inlet_elevations,
smoothing_timescale=smoothing_timescale,
compute_discharge_implicitly=compute_discharge_implicitly,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose)
from anuga.utilities import parallel_abstraction as pypar
if procs is None:
procs = list(range(0, pypar.size()))
myid = pypar.rank()
end_points = ensure_numeric(end_points)
exchange_lines = ensure_numeric(exchange_lines)
enquiry_points = ensure_numeric(enquiry_points)
if height is None:
height = width
diameter = None
if apron is None:
apron = width
# Calculate location of inlet enquiry points and exchange lines
if myid == master_proc:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = __process_skew_culvert(
exchange_lines, end_points, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(enquiry_points_tmp, i)
elif end_points is not None:
exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert(
end_points, width, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(exchange_lines_tmp, i)
pypar.send(enquiry_points_tmp, i)
else:
raise Exception('Define either exchange_lines or end_points')
else:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = pypar.receive(master_proc)
elif end_points is not None:
exchange_lines_tmp = pypar.receive(master_proc)
enquiry_points_tmp = pypar.receive(master_proc)
# Determine processors associated with first inlet
line0 = exchange_lines_tmp[0]
enquiry_point0 = enquiry_points_tmp[0]
alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs(
domain,
line0,
enquiry_point=enquiry_point0,
master_proc=master_proc,
procs=procs,
verbose=verbose)
# Determine processors associated with second inlet
line1 = exchange_lines_tmp[1]
enquiry_point1 = enquiry_points_tmp[1]
alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs(
domain,
line1,
enquiry_point=enquiry_point1,
master_proc=master_proc,
procs=procs,
verbose=verbose)
structure_procs = list(set(inlet0_procs + inlet1_procs))
inlet_master_proc = [inlet0_master_proc, inlet1_master_proc]
inlet_procs = [inlet0_procs, inlet1_procs]
enquiry_proc = [enquiry0_proc, enquiry1_proc]
if myid == master_proc and verbose:
print(
"Parallel Internal boundary Operator ============================="
)
print("Structure Master Proc is P" + str(inlet0_master_proc))
print("Structure Procs are P" + str(structure_procs))
print("Inlet Master Procs are P" + str(inlet_master_proc))
print("Inlet Procs are P" + str(inlet_procs[0]) + " and " +
str(inlet_procs[1]))
print("Inlet Enquiry Procs are P" + str(enquiry_proc))
print("Enquiry Points are " + str(enquiry_point0) + " and " +
str(enquiry_point1))
print("Inlet Exchange Lines are " + str(line0) + " and " + str(line1))
print("========================================================")
if alloc0 or alloc1:
return Parallel_Internal_boundary_operator(
domain=domain,
internal_boundary_function=internal_boundary_function,
width=width,
height=height,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevation=invert_elevation,
apron=apron,
enquiry_gap=enquiry_gap,
use_velocity_head=use_velocity_head,
zero_outflow_momentum=zero_outflow_momentum,
force_constant_inlet_elevations=force_constant_inlet_elevations,
smoothing_timescale=smoothing_timescale,
compute_discharge_implicitly=compute_discharge_implicitly,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose,
master_proc=inlet0_master_proc,
procs=structure_procs,
inlet_master_proc=inlet_master_proc,
inlet_procs=inlet_procs,
enquiry_proc=enquiry_proc)
else:
return None
def Weir_orifice_trapezoid_operator(
domain,
losses,
width,
blockage=0.0,
barrels=1.0,
z1=None,
z2=None,
height=None,
end_points=None,
exchange_lines=None,
enquiry_points=None,
invert_elevations=None,
#culvert_slope=None,
apron=0.1,
manning=0.013,
enquiry_gap=0.0,
smoothing_timescale=0.0,
use_momentum_jet=True,
use_velocity_head=True,
description=None,
label=None,
structure_type='weir_orifice_trapezoid',
logging=False,
verbose=False,
master_proc=0,
procs=None):
# If not parallel domain then allocate serial Weir orifice trapezoid operator
if isinstance(domain, Parallel_domain) is False:
if verbose:
print(
"Allocating non parallel weir orifice trapzezoid operator ....."
)
return anuga.structures.weir_orifice_trapezoid_operator.Weir_orifice_trapezoid_operator(
domain=domain,
losses=losses,
width=width,
height=height,
blockage=blockage,
barrels=barrels,
z1=z1,
z2=z2,
#culvert_slope=culvert_slope,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevations=invert_elevations,
apron=apron,
manning=manning,
enquiry_gap=enquiry_gap,
smoothing_timescale=smoothing_timescale,
use_momentum_jet=use_momentum_jet,
use_velocity_head=use_velocity_head,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose)
from anuga.utilities import parallel_abstraction as pypar
if procs is None:
procs = list(range(0, pypar.size()))
myid = pypar.rank()
end_points = ensure_numeric(end_points)
exchange_lines = ensure_numeric(exchange_lines)
enquiry_points = ensure_numeric(enquiry_points)
if height is None:
height = width
diameter = None
if apron is None:
apron = width
# Calculate location of inlet enquiry points and exchange lines
if myid == master_proc:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = __process_skew_culvert(
exchange_lines, end_points, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(enquiry_points_tmp, i)
elif end_points is not None:
exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert(
end_points, width, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(exchange_lines_tmp, i)
pypar.send(enquiry_points_tmp, i)
else:
raise Exception('Define either exchange_lines or end_points')
else:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = pypar.receive(master_proc)
elif end_points is not None:
exchange_lines_tmp = pypar.receive(master_proc)
enquiry_points_tmp = pypar.receive(master_proc)
# Determine processors associated with first inlet
line0 = exchange_lines_tmp[0]
enquiry_point0 = enquiry_points_tmp[0]
alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs(
domain,
line0,
enquiry_point=enquiry_point0,
master_proc=master_proc,
procs=procs,
verbose=verbose)
# Determine processors associated with second inlet
line1 = exchange_lines_tmp[1]
enquiry_point1 = enquiry_points_tmp[1]
alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs(
domain,
line1,
enquiry_point=enquiry_point1,
master_proc=master_proc,
procs=procs,
verbose=verbose)
structure_procs = list(set(inlet0_procs + inlet1_procs))
inlet_master_proc = [inlet0_master_proc, inlet1_master_proc]
inlet_procs = [inlet0_procs, inlet1_procs]
enquiry_proc = [enquiry0_proc, enquiry1_proc]
if myid == master_proc and verbose:
print(
"Parallel Weir Orifice Trapezoid Operator ============================="
)
print("Structure Master Proc is P" + str(inlet0_master_proc))
print("Structure Procs are P" + str(structure_procs))
print("Inlet Master Procs are P" + str(inlet_master_proc))
print("Inlet Procs are P" + str(inlet_procs[0]) + " and " +
str(inlet_procs[1]))
print("Inlet Enquiry Procs are P" + str(enquiry_proc))
print("Enquiry Points are " + str(enquiry_point0) + " and " +
str(enquiry_point1))
print("Inlet Exchange Lines are " + str(line0) + " and " + str(line1))
print("========================================================")
if alloc0 or alloc1:
return Parallel_Weir_orifice_trapezoid_operator(
domain=domain,
losses=losses,
width=width,
height=height,
blockage=blockage,
barrels=barrels,
z1=z1,
z2=z2,
#culvert_slope=culvert_slope,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevations=invert_elevations,
apron=apron,
manning=manning,
enquiry_gap=enquiry_gap,
smoothing_timescale=smoothing_timescale,
use_momentum_jet=use_momentum_jet,
use_velocity_head=use_velocity_head,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose,
master_proc=inlet0_master_proc,
procs=structure_procs,
inlet_master_proc=inlet_master_proc,
inlet_procs=inlet_procs,
enquiry_proc=enquiry_proc)
else:
return None
def __init__(self, coordinates, vertices,
boundary=None,
full_send_dict=None,
ghost_recv_dict=None,
number_of_full_nodes=None,
number_of_full_triangles=None,
geo_reference=None,
processor = None,
numproc = None,
number_of_global_triangles=None, ## SR added this
number_of_global_nodes= None, ## SR added this
s2p_map=None,
p2s_map=None, #jj added this
tri_l2g = None, ## SR added this
node_l2g = None, #): ## SR added this
ghost_layer_width = 2): ## SR added this
#-----------------------------------------
# Sometimes we want to manually
# create instances of the parallel_domain
# otherwise ...
#----------------------------------------
if processor is None:
processor = pypar.rank()
if numproc is None:
numproc = pypar.size()
Domain.__init__(self,
coordinates,
vertices,
boundary,
full_send_dict=full_send_dict,
ghost_recv_dict=ghost_recv_dict,
processor=processor,
numproc=numproc,
number_of_full_nodes=number_of_full_nodes,
number_of_full_triangles=number_of_full_triangles,
geo_reference=geo_reference, #) #jj added this
ghost_layer_width = ghost_layer_width)
self.parallel = True
# PETE: Find the number of full nodes and full triangles, this is a temporary fix
# until the bug with get_number_of_full_[nodes|triangles]() is fixed.
if number_of_full_nodes is not None:
self.number_of_full_nodes_tmp = number_of_full_nodes
else:
self.number_of_full_nodes_tmp = self.get_number_of_nodes()
if number_of_full_triangles is not None:
self.number_of_full_triangles_tmp = number_of_full_triangles
else:
self.number_of_full_triangles_tmp = self.get_number_of_triangles()
generic_comms.setup_buffers(self)
self.global_name = 'domain'
self.number_of_global_triangles=number_of_global_triangles
self.number_of_global_nodes = number_of_global_nodes
self.s2p_map = s2p_map
self.p2s_map = p2s_map
self.s2p_map = None
self.p2s_map = None
self.tri_l2g = tri_l2g
self.node_l2g = node_l2g
self.ghost_counter = 0
def dump_triangulation(self, filename="domain.png"):
# Get vertex coordinates, partition full and ghost triangles based on self.tri_full_flag
try:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.tri as tri
except:
print(
"Couldn't import module from matplotlib, probably you need to update matplotlib"
)
raise
vertices = self.get_vertex_coordinates()
full_mask = num.repeat(self.tri_full_flag == 1, 3)
ghost_mask = num.repeat(self.tri_full_flag == 0, 3)
myid = pypar.rank()
numprocs = pypar.size()
if myid == 0:
fig = plt.figure()
fx = {}
fy = {}
gx = {}
gy = {}
# Proc 0 gathers full and ghost nodes from self and other processors
fx[0] = vertices[full_mask, 0]
fy[0] = vertices[full_mask, 1]
gx[0] = vertices[ghost_mask, 0]
gy[0] = vertices[ghost_mask, 1]
for i in range(1, numprocs):
fx[i] = pypar.receive(i)
fy[i] = pypar.receive(i)
gx[i] = pypar.receive(i)
gy[i] = pypar.receive(i)
# Plot full triangles
for i in range(0, numprocs):
n = int(old_div(len(fx[i]), 3))
triang = num.array(list(range(0, 3 * n)))
triang.shape = (n, 3)
plt.triplot(fx[i], fy[i], triang, 'g-', linewidth=0.5)
# Plot ghost triangles
for i in range(0, numprocs):
n = int(old_div(len(gx[i]), 3))
if n > 0:
triang = num.array(list(range(0, 3 * n)))
triang.shape = (n, 3)
plt.triplot(gx[i], gy[i], triang, 'b--', linewidth=0.5)
# Save triangulation to location pointed by filename
plt.savefig(filename, dpi=600)
else:
# Proc 1..numprocs send full and ghost triangles to Proc 0
pypar.send(vertices[full_mask, 0], 0)
pypar.send(vertices[full_mask, 1], 0)
pypar.send(vertices[ghost_mask, 0], 0)
pypar.send(vertices[ghost_mask, 1], 0)
# Mesh partitioning using Metis
from anuga.parallel.distribute_mesh import build_submesh
from anuga.parallel.distribute_mesh import pmesh_divide_metis_with_map
from anuga.parallel.parallel_shallow_water import Parallel_domain
from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh
#------------------------------------------------------------------------------
# Read in processor information
#------------------------------------------------------------------------------
numprocs = size()
myid = rank()
processor_name = get_processor_name()
#print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name)
def collect_value(value):
value = value
if myid == 0:
for i in range(numprocs):
if i == 0: continue
val = receive(i)
value = value + val
def distibute_three_processors():
"""
Do a parallel test of distributing a rectangle onto 3 processors
"""
# FIXME: Need to update expected values on macos
if sys.platform == 'darwin':
return
# FIXME: Need to update expected values on macos
#if sys.platform == 'win32':
# return
from anuga.utilities import parallel_abstraction as pypar
myid = pypar.rank()
numprocs = pypar.size()
if not numprocs == 3:
return
try:
import pymetis
metis_version = 5
except:
metis_version = 4
#print numprocs
#barrier()
if myid == 0:
nodes_0, triangles_0, boundary_0 = rectangular_cross(2, 2)
domain = Domain(nodes_0, triangles_0, boundary_0)
domain.set_quantity('elevation',
topography) # Use function for elevation
domain.set_quantity('friction', 0.0) # Constant friction
domain.set_quantity('stage',
expression='elevation') # Dry initial stage
domain.set_quantity('xmomentum', expression='friction + 2.0')
domain.set_quantity('ymomentum', ycoord)
#----------------------------------------------------------------------------------
# Test pmesh_divide_metis
#----------------------------------------------------------------------------------
vertices, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(
domain, numprocs)
if False:
print_seq_values(vertices, triangles, triangles_per_proc)
true_seq_values = get_true_seq_values(metis_version=metis_version)
if False:
print("True Seq Values = \\")
pprint(true_seq_values)
assert_allclose(vertices, true_seq_values['vertices'])
assert_allclose(triangles, true_seq_values['triangles'])
assert_allclose(triangles_per_proc,
true_seq_values['triangles_per_proc'])
#----------------------------------------------------------------------------------
# Test build_submesh
#----------------------------------------------------------------------------------
submesh = build_submesh(vertices, triangles, boundary, quantities,
triangles_per_proc)
if False:
print('submesh_values = \\')
print_submesh_values(submesh)
true_values = get_true_submesh_values(metis_version)
assert_allclose(submesh['full_nodes'][0], true_values['full_nodes_0'])
assert_allclose(submesh['full_nodes'][1], true_values['full_nodes_1'])
assert_allclose(submesh['full_nodes'][2], true_values['full_nodes_2'])
assert_allclose(submesh['ghost_nodes'][0],
true_values['ghost_nodes_0'])
assert_allclose(submesh['ghost_nodes'][1],
true_values['ghost_nodes_1'])
assert_allclose(submesh['ghost_nodes'][2],
true_values['ghost_nodes_2'])
assert_allclose(submesh['full_triangles'][0],
true_values['full_triangles_0'])
assert_allclose(submesh['full_triangles'][1],
true_values['full_triangles_1'])
assert_allclose(submesh['full_triangles'][2],
true_values['full_triangles_2'])
assert_allclose(submesh['ghost_triangles'][0],
true_values['ghost_triangles_0'])
assert_allclose(submesh['ghost_triangles'][1],
true_values['ghost_triangles_1'])
assert_allclose(submesh['ghost_triangles'][2],
true_values['ghost_triangles_2'])
assert_allclose(submesh['ghost_commun'][0],
true_values['ghost_commun_0'])
assert_allclose(submesh['ghost_commun'][1],
true_values['ghost_commun_1'])
assert_allclose(submesh['ghost_commun'][2],
true_values['ghost_commun_2'])
assert_(submesh['full_commun'] == true_values['full_commun'])
barrier()
#--------------------------------
# Now do the comunnication part
#--------------------------------
if myid == 0:
points, vertices, boundary, quantities, \
ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width =\
extract_submesh(submesh, triangles_per_proc)
#----------------------------------------------------------------------------------
# Test send_submesh
#----------------------------------------------------------------------------------
for p in range(1, numprocs):
send_submesh(submesh, triangles_per_proc, p, verbose=False)
else:
#----------------------------------------------------------------------------------
# Test rec_submesh
#----------------------------------------------------------------------------------
points, triangles, boundary, quantities, \
ghost_recv_dict, full_send_dict, \
no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width = \
rec_submesh(0, verbose=False)
barrier()
#--------------------------------
# Now do the test
#--------------------------------
if myid == 0:
if False:
print('extract_values = \\')
print_extract_submesh(points, triangles, ghost_recv_dict, \
full_send_dict, tri_map, node_map, ghost_layer_width)
true_values = get_true_extract_submesh(metis_version)
assert_allclose(points, true_values['points'])
assert_allclose(triangles, true_values['triangles'])
assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
assert_allclose(tri_map, true_values['tri_map'])
assert_allclose(node_map, true_values['node_map'])
assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])
if myid == 1:
if False:
print("rec_submesh_1 = \\")
print_rec_submesh_1(points, triangles, ghost_recv_dict, full_send_dict, \
tri_map, node_map, ghost_layer_width)
true_values = get_true_rec_submesh_1(metis_version)
if False:
print('true_rec_values_1 = \\')
pprint(true_values)
assert_allclose(points, true_values['points'])
assert_allclose(triangles, true_values['triangles'])
assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
assert_allclose(tri_map, true_values['tri_map'])
assert_allclose(node_map, true_values['node_map'])
assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])
if myid == 2:
if False:
print("rec_submesh_2 = \\")
print_rec_submesh_2(points, triangles, ghost_recv_dict, full_send_dict, \
tri_map, node_map, ghost_layer_width)
true_values = get_true_rec_submesh_2(metis_version)
if False:
print('true_rec_values_2 = \\')
pprint(true_values)
assert_allclose(points, true_values['points'])
assert_allclose(triangles, true_values['triangles'])
assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
assert_allclose(tri_map, true_values['tri_map'])
assert_allclose(node_map, true_values['node_map'])
assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])
finalize()
def plotCentroidError(domain, control_data, rthr = 1E-7, athr = 1E-12,
quantity = 'stage', filename = 'centroid_error.png'):
n_triangles = num.sum(domain.tri_full_flag)
if size() > 1:
# If parallel, translate control data to parallel indexing
local_control_data = num.zeros(n_triangles)
inv_tri_map = domain.get_inv_tri_map()
for i in range(n_triangles):
local_control_data[i] = control_data[inv_tri_map[(rank(), i)]]
else:
local_control_data = control_data
# Evaluate absolute and relative difference between control and actual values
stage = domain.get_quantity(quantity)
actual_data = stage.centroid_values[:n_triangles]
adiff = num.fabs((actual_data - local_control_data))
rdiff = adiff/num.fabs(local_control_data)
# Compute masks for error (err_mask) and non-error (acc_mask) vertex indices based on thresholds
vertices = domain.get_vertex_coordinates()
err_mask = rdiff > rthr
err_mask[adiff <= athr] = False
err_mask = num.repeat(err_mask, 3)
acc_mask = ~err_mask
inv_tri_map = domain.get_inv_tri_map()
# Plot error and non-error triangle
if rank() == 0:
fx = {}
fy = {}
gx = {}
gy = {}
fx[0] = vertices[acc_mask,0]
fy[0] = vertices[acc_mask,1]
gx[0] = vertices[err_mask,0]
gy[0] = vertices[err_mask,1]
# Receive vertex indices of non-error triangles (fx, fy) and error triangles (gx, gy)
for i in range(1,size()):
fx[i] = receive(i)
fy[i] = receive(i)
gx[i] = receive(i)
gy[i] = receive(i)
# Plot non-error triangles in green
for i in range(0,size()):
n = int(len(fx[i])/3)
triang = num.array(range(0,3*n))
triang.shape = (n, 3)
if len(fx[i]) > 0:
plt.triplot(fx[i], fy[i], triang, 'g-')
# Plot error triangles in blue
for i in range(0,size()):
n = int(len(gx[i])/3)
triang = num.array(range(0,3*n))
triang.shape = (n, 3)
if len(gx[i]) > 0:
plt.triplot(gx[i], gy[i], triang, 'b--')
# Save plot
plt.savefig(filename)
else:
# Send error and non-error vertex indices to Proc 0
send(vertices[acc_mask,0], 0)
send(vertices[acc_mask,1], 0)
send(vertices[err_mask,0], 0)
send(vertices[err_mask,1], 0)
def plotCentroidError(domain,
control_data,
rthr=1E-7,
athr=1E-12,
quantity='stage',
filename='centroid_error.png'):
n_triangles = num.sum(domain.tri_full_flag)
if size() > 1:
# If parallel, translate control data to parallel indexing
local_control_data = num.zeros(n_triangles)
inv_tri_map = domain.get_inv_tri_map()
for i in range(n_triangles):
local_control_data[i] = control_data[inv_tri_map[(rank(), i)]]
else:
local_control_data = control_data
# Evaluate absolute and relative difference between control and actual values
stage = domain.get_quantity(quantity)
actual_data = stage.centroid_values[:n_triangles]
adiff = num.fabs((actual_data - local_control_data))
rdiff = adiff / num.fabs(local_control_data)
# Compute masks for error (err_mask) and non-error (acc_mask) vertex indices based on thresholds
vertices = domain.get_vertex_coordinates()
err_mask = rdiff > rthr
err_mask[adiff <= athr] = False
err_mask = num.repeat(err_mask, 3)
acc_mask = ~err_mask
inv_tri_map = domain.get_inv_tri_map()
# Plot error and non-error triangle
if rank() == 0:
fx = {}
fy = {}
gx = {}
gy = {}
fx[0] = vertices[acc_mask, 0]
fy[0] = vertices[acc_mask, 1]
gx[0] = vertices[err_mask, 0]
gy[0] = vertices[err_mask, 1]
# Receive vertex indices of non-error triangles (fx, fy) and error triangles (gx, gy)
for i in range(1, size()):
fx[i] = receive(i)
fy[i] = receive(i)
gx[i] = receive(i)
gy[i] = receive(i)
# Plot non-error triangles in green
for i in range(0, size()):
n = int(len(fx[i]) / 3)
triang = num.array(range(0, 3 * n))
triang.shape = (n, 3)
if len(fx[i]) > 0:
plt.triplot(fx[i], fy[i], triang, 'g-')
# Plot error triangles in blue
for i in range(0, size()):
n = int(len(gx[i]) / 3)
triang = num.array(range(0, 3 * n))
triang.shape = (n, 3)
if len(gx[i]) > 0:
plt.triplot(gx[i], gy[i], triang, 'b--')
# Save plot
plt.savefig(filename)
else:
# Send error and non-error vertex indices to Proc 0
send(vertices[acc_mask, 0], 0)
send(vertices[acc_mask, 1], 0)
send(vertices[err_mask, 0], 0)
send(vertices[err_mask, 1], 0)
def rec_submesh_flat(p, verbose=True):
from anuga.utilities import parallel_abstraction as pypar
numprocs = pypar.size()
myid = pypar.rank()
submesh_cell = {}
if verbose:
print(indent + 'P%d: Receiving submesh from P%d' % (myid, p))
# receive the tagmap for the boundary conditions
tagmap = pypar.receive(p)
itagmap = {}
for t in tagmap:
itagmap[tagmap[t]] = t
# receive the quantities key information
qkeys = pypar.receive(p)
# recieve information about the array sizes
x = num.zeros((9, ), num.int)
pypar.receive(p, buffer=x, bypass=True)
setup_array = x
no_full_nodes = setup_array[0]
no_ghost_nodes = setup_array[1]
no_full_triangles = setup_array[2]
no_ghost_triangles = setup_array[3]
no_full_boundary = setup_array[4]
no_ghost_boundary = setup_array[5]
no_ghost_commun = setup_array[6]
no_full_commun = setup_array[7]
no_quantities = setup_array[8]
# ghost layer width
x = num.zeros((1, ), num.int)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["ghost_layer_width"] = x[0]
# receive the number of triangles per processor
x = num.zeros((numprocs, ), num.int)
pypar.receive(p, buffer=x, bypass=True)
triangles_per_proc = x
# receive the full nodes
x = num.zeros((no_full_nodes, 3), num.float)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["full_nodes"] = x
# receive the ghost nodes
x = num.zeros((no_ghost_nodes, 3), num.float)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["ghost_nodes"] = x
# receive the full triangles
x = num.zeros((no_full_triangles, 3), num.int)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["full_triangles"] = x
# receive the ghost triangles
x = num.zeros((no_ghost_triangles, 4), num.int)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["ghost_triangles"] = x
# receive the full boundary
x = num.zeros((no_full_boundary, 3), num.int)
pypar.receive(p, buffer=x, bypass=True)
bnd_c = x
submesh_cell["full_boundary"] = {}
for b in bnd_c:
submesh_cell["full_boundary"][b[0], b[1]] = itagmap[b[2]]
# receive the ghost boundary
x = num.zeros((no_ghost_boundary, 3), num.int)
pypar.receive(p, buffer=x, bypass=True)
bnd_c = x
submesh_cell["ghost_boundary"] = {}
for b in bnd_c:
submesh_cell["ghost_boundary"][b[0], b[1]] = itagmap[b[2]]
# receive the ghost communication pattern
x = num.zeros((no_ghost_commun, 2), num.int)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["ghost_commun"] = x
# receive the full communication pattern
x = num.zeros((no_full_commun, 2), num.int)
pypar.receive(p, buffer=x, bypass=True)
full_commun = x
submesh_cell["full_commun"] = {}
for c in full_commun:
submesh_cell["full_commun"][c[0]] = []
for c in full_commun:
submesh_cell["full_commun"][c[0]].append(c[1])
# receive the quantities
submesh_cell["full_quan"] = {}
for i in range(no_quantities):
x = num.zeros((no_full_triangles, 3), num.float)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["full_quan"][qkeys[i]] = x
submesh_cell["ghost_quan"] = {}
for i in range(no_quantities):
x = num.zeros((no_ghost_triangles, 3), num.float)
pypar.receive(p, buffer=x, bypass=True)
submesh_cell["ghost_quan"][qkeys[i]] = x
return submesh_cell, triangles_per_proc,\
no_full_nodes, no_full_triangles
def __init__(self,
domain,
end_points,
exchange_lines,
enquiry_points,
invert_elevations,
width,
height,
diameter,
z1,
z2,
blockage,
barrels,
apron,
manning,
enquiry_gap,
use_momentum_jet,
zero_outflow_momentum,
use_old_momentum_method,
always_use_Q_wetdry_adjustment,
force_constant_inlet_elevations,
description,
label,
structure_type,
logging,
verbose,
master_proc=0,
procs=None,
inlet_master_proc=[0, 0],
inlet_procs=None,
enquiry_proc=None):
self.myid = pypar.rank()
self.num_procs = pypar.size()
anuga.Operator.__init__(self, domain)
# Allocate default processor associations if not specified in arguments
# although we assume that such associations are provided correctly by the
# parallel_operator_factory.
self.master_proc = master_proc
self.inlet_master_proc = inlet_master_proc
if procs is None:
self.procs = [master_proc]
else:
self.procs = procs
if inlet_procs is None:
self.inlet_procs = [[inlet_master_proc[0]], [inlet_master_proc[0]]]
else:
self.inlet_procs = inlet_procs
if enquiry_proc is None:
self.enquiry_proc = [[inlet_master_proc[0]],
[inlet_master_proc[0]]]
else:
self.enquiry_proc = enquiry_proc
self.end_points = ensure_numeric(end_points)
self.exchange_lines = ensure_numeric(exchange_lines)
self.enquiry_points = ensure_numeric(enquiry_points)
self.invert_elevations = ensure_numeric(invert_elevations)
assert (width is not None
and diameter is None) or (width is None
and diameter is not None)
if width is None:
width = diameter
if diameter is None:
diameter = width
if height is None:
height = width
if apron is None:
apron = width
self.width = width
self.height = height
self.diameter = diameter
self.z1 = z1
self.z2 = z2
self.blockage = blockage
self.barrels = barrels
self.apron = apron
self.manning = manning
self.enquiry_gap = enquiry_gap
self.use_momentum_jet = use_momentum_jet
self.zero_outflow_momentum = zero_outflow_momentum
if use_momentum_jet and zero_outflow_momentum:
msg = "Can't have use_momentum_jet and zero_outflow_momentum both True"
raise Exception(msg)
self.use_old_momentum_method = use_old_momentum_method
self.always_use_Q_wetdry_adjustment = always_use_Q_wetdry_adjustment
if description is None:
self.description = ' '
else:
self.description = description
if label is None:
self.label = "structure_%g" % Parallel_Structure_operator.counter + "_P" + str(
self.myid)
else:
self.label = label + '_%g' % Parallel_Structure_operator.counter + "_P" + str(
self.myid)
if structure_type is None:
self.structure_type = 'generic structure'
else:
self.structure_type = structure_type
self.verbose = verbose
# Keep count of structures
if self.myid == master_proc:
Parallel_Structure_operator.counter += 1
# Slots for recording current statistics
self.accumulated_flow = 0.0
self.discharge = 0.0
self.discharge_abs_timemean = 0.0
self.velocity = 0.0
self.outlet_depth = 0.0
self.delta_total_energy = 0.0
self.driving_energy = 0.0
if exchange_lines is not None:
self.__process_skew_culvert()
elif end_points is not None:
self.__process_non_skew_culvert()
else:
raise Exception('Define either exchange_lines or end_points')
self.inlets = []
# Allocate parallel inlet enquiry, assign None if processor is not associated with particular
# inlet.
if self.myid in self.inlet_procs[0]:
line0 = self.exchange_lines[0]
if self.apron is None:
poly0 = line0
else:
offset = -self.apron * self.outward_vector_0
poly0 = num.array(
[line0[0], line0[1], line0[1] + offset, line0[0] + offset])
if self.invert_elevations is None:
invert_elevation0 = None
else:
invert_elevation0 = self.invert_elevations[0]
enquiry_point0 = self.enquiry_points[0]
outward_vector0 = self.culvert_vector
self.inlets.append(
parallel_inlet_enquiry.Parallel_Inlet_enquiry(
self.domain,
line0,
enquiry_point0,
invert_elevation=invert_elevation0,
outward_culvert_vector=outward_vector0,
master_proc=self.inlet_master_proc[0],
procs=self.inlet_procs[0],
enquiry_proc=self.enquiry_proc[0],
verbose=self.verbose))
if force_constant_inlet_elevations:
# Try to enforce a constant inlet elevation
inlet_global_elevation = self.inlets[
-1].get_global_average_elevation()
self.inlets[-1].set_elevations(inlet_global_elevation)
else:
self.inlets.append(None)
if self.myid in self.inlet_procs[1]:
line1 = self.exchange_lines[1]
if self.apron is None:
poly1 = line1
else:
offset = -self.apron * self.outward_vector_1
poly1 = num.array(
[line1[0], line1[1], line1[1] + offset, line1[0] + offset])
if self.invert_elevations is None:
invert_elevation1 = None
else:
invert_elevation1 = self.invert_elevations[1]
enquiry_point1 = self.enquiry_points[1]
outward_vector1 = -self.culvert_vector
self.inlets.append(
parallel_inlet_enquiry.Parallel_Inlet_enquiry(
self.domain,
line1,
enquiry_point1,
invert_elevation=invert_elevation1,
outward_culvert_vector=outward_vector1,
master_proc=self.inlet_master_proc[1],
procs=self.inlet_procs[1],
enquiry_proc=self.enquiry_proc[1],
verbose=self.verbose))
if force_constant_inlet_elevations:
# Try to enforce a constant inlet elevation
inlet_global_elevation = self.inlets[
-1].get_global_average_elevation()
self.inlets[-1].set_elevations(inlet_global_elevation)
else:
self.inlets.append(None)
self.inflow_index = 0
self.outflow_index = 1
self.set_parallel_logging(logging)
def distribute_mesh(domain, verbose=False, debug=False, parameters=None):
""" Distribute andsend the mesh info to all the processors.
Should only be run from processor 0 and will send info to the other
processors.
There should be a corresponding rec_submesh called from all the other
processors
"""
if debug:
verbose = True
numprocs = size()
# Subdivide the mesh
if verbose: print 'Subdivide mesh'
new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
s2p_map, p2s_map = \
pmesh_divide_metis_with_map(domain, numprocs)
#PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
# sp2_map (maps parallel domain triangles to domain triangles)
# Build the mesh that should be assigned to each processor,
# this includes ghost nodes and the communication pattern
if verbose: print 'Build submeshes'
submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities, triangles_per_proc, parameters)
if verbose:
for p in range(numprocs):
N = len(submesh['ghost_nodes'][p])
M = len(submesh['ghost_triangles'][p])
print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
%(N, M, p)
#if debug:
# from pprint import pprint
# pprint(submesh)
# Send the mesh partition to the appropriate processor
if verbose: print 'Distribute submeshes'
for p in range(1, numprocs):
send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose)
# Build the local mesh for processor 0
points, vertices, boundary, quantities, \
ghost_recv_dict, full_send_dict, \
tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
extract_submesh(submesh, triangles_per_proc, p2s_map, 0)
# Keep track of the number full nodes and triangles.
# This is useful later if one needs access to a ghost-free domain
# Here, we do it for process 0. The others are done in rec_submesh.
number_of_full_nodes = len(submesh['full_nodes'][0])
number_of_full_triangles = len(submesh['full_triangles'][0])
#print
#for p in range(numprocs):
# print 'Process %d:' %(p)
#
# print 'full_triangles:'
# print submesh['full_triangles'][p]
#
# print 'full_nodes:'
# print submesh['full_nodes'][p]
#
# print 'ghost_triangles:'
# print submesh['ghost_triangles'][p]#
#
# print 'ghost_nodes:'
# print submesh['ghost_nodes'][p]
# print
#
#print 'Receive dict'
#print ghost_recv_dict
#
#print 'Send dict'
#print full_send_dict
# Return structures necessary for building the parallel domain
return points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles, \
s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width
def dump_triangulation(self, filename="domain.png"):
# Get vertex coordinates, partition full and ghost triangles based on self.tri_full_flag
try:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.tri as tri
except:
print "Couldn't import module from matplotlib, probably you need to update matplotlib"
raise
vertices = self.get_vertex_coordinates()
full_mask = num.repeat(self.tri_full_flag == 1, 3)
ghost_mask = num.repeat(self.tri_full_flag == 0, 3)
myid = pypar.rank()
numprocs = pypar.size()
if myid == 0:
fig = plt.figure()
fx = {}
fy = {}
gx = {}
gy = {}
# Proc 0 gathers full and ghost nodes from self and other processors
fx[0] = vertices[full_mask,0]
fy[0] = vertices[full_mask,1]
gx[0] = vertices[ghost_mask,0]
gy[0] = vertices[ghost_mask,1]
for i in range(1,numprocs):
fx[i] = pypar.receive(i)
fy[i] = pypar.receive(i)
gx[i] = pypar.receive(i)
gy[i] = pypar.receive(i)
# Plot full triangles
for i in range(0, numprocs):
n = int(len(fx[i])/3)
triang = num.array(range(0,3*n))
triang.shape = (n, 3)
plt.triplot(fx[i], fy[i], triang, 'g-', linewidth = 0.5)
# Plot ghost triangles
for i in range(0, numprocs):
n = int(len(gx[i])/3)
if n > 0:
triang = num.array(range(0,3*n))
triang.shape = (n, 3)
plt.triplot(gx[i], gy[i], triang, 'b--', linewidth = 0.5)
# Save triangulation to location pointed by filename
plt.savefig(filename, dpi=600)
else:
# Proc 1..numprocs send full and ghost triangles to Proc 0
pypar.send(vertices[full_mask,0], 0)
pypar.send(vertices[full_mask,1], 0)
pypar.send(vertices[ghost_mask,0], 0)
pypar.send(vertices[ghost_mask,1], 0)
def __init__(
self,
coordinates,
vertices,
boundary=None,
full_send_dict=None,
ghost_recv_dict=None,
number_of_full_nodes=None,
number_of_full_triangles=None,
geo_reference=None,
processor=None,
numproc=None,
number_of_global_triangles=None, ## SR added this
number_of_global_nodes=None, ## SR added this
s2p_map=None,
p2s_map=None, #jj added this
tri_l2g=None, ## SR added this
node_l2g=None, #): ## SR added this
ghost_layer_width=2): ## SR added this
#-----------------------------------------
# Sometimes we want to manually
# create instances of the parallel_domain
# otherwise ...
#----------------------------------------
if processor is None:
processor = pypar.rank()
if numproc is None:
numproc = pypar.size()
Domain.__init__(
self,
coordinates,
vertices,
boundary,
full_send_dict=full_send_dict,
ghost_recv_dict=ghost_recv_dict,
processor=processor,
numproc=numproc,
number_of_full_nodes=number_of_full_nodes,
number_of_full_triangles=number_of_full_triangles,
geo_reference=geo_reference, #) #jj added this
ghost_layer_width=ghost_layer_width)
self.parallel = True
# PETE: Find the number of full nodes and full triangles, this is a temporary fix
# until the bug with get_number_of_full_[nodes|triangles]() is fixed.
if number_of_full_nodes is not None:
self.number_of_full_nodes_tmp = number_of_full_nodes
else:
self.number_of_full_nodes_tmp = self.get_number_of_nodes()
if number_of_full_triangles is not None:
self.number_of_full_triangles_tmp = number_of_full_triangles
else:
self.number_of_full_triangles_tmp = self.get_number_of_triangles()
generic_comms.setup_buffers(self)
self.global_name = 'domain'
self.number_of_global_triangles = number_of_global_triangles
self.number_of_global_nodes = number_of_global_nodes
self.s2p_map = s2p_map
self.p2s_map = p2s_map
self.s2p_map = None
self.p2s_map = None
self.tri_l2g = tri_l2g
self.node_l2g = node_l2g
self.ghost_counter = 0
def Inlet_operator(domain,
poly,
Q,
velocity=None,
zero_velocity=False,
default=0.0,
description=None,
label=None,
logging=False,
master_proc=0,
procs=None,
verbose=False):
# If not parallel domain then allocate serial Inlet operator
if isinstance(domain, Parallel_domain) is False:
if verbose: print("Allocating non parallel inlet operator .....")
return anuga.structures.inlet_operator.Inlet_operator(
domain,
poly,
Q,
velocity=velocity,
zero_velocity=zero_velocity,
default=default,
description=description,
label=label,
logging=logging,
verbose=verbose)
from anuga.utilities import parallel_abstraction as pypar
if procs is None:
procs = list(range(0, pypar.size()))
myid = pypar.rank()
poly = num.array(poly, dtype='d')
alloc, inlet_master_proc, inlet_procs, enquiry_proc = allocate_inlet_procs(
domain, poly, master_proc=master_proc, procs=procs, verbose=verbose)
if alloc:
if verbose and myid == inlet_master_proc:
print("Parallel Inlet Operator =================")
print("Poly = " + str(poly))
print("Master Processor is P%d" % (inlet_master_proc))
print("Processors are P%s" % (inlet_procs))
print("=========================================")
return Parallel_Inlet_operator(domain,
poly,
Q,
velocity=velocity,
zero_velocity=zero_velocity,
default=default,
description=description,
label=label,
logging=logging,
master_proc=inlet_master_proc,
procs=inlet_procs,
verbose=verbose)
else:
return None