def sww_merge(self, verbose=False, delete_old=False):
# make sure all the computations have finished
pypar.barrier()
# now on processor 0 pull all the separate sww files together
if self.processor == 0 and self.numproc > 1 and self.store :
import anuga.utilities.sww_merge as merge
global_name = join(self.get_datadir(),self.get_global_name())
merge.sww_merge_parallel(global_name,self.numproc,verbose,delete_old)
# make sure all the merge completes on processor 0 before other
# processors complete (like when finalize is forgotten in main script)
pypar.barrier()
def sww_merge(self, verbose=False, delete_old=False):
# make sure all the computations have finished
pypar.barrier()
# now on processor 0 pull all the separate sww files together
if self.processor == 0 and self.numproc > 1 and self.store:
import anuga.utilities.sww_merge as merge
global_name = join(self.get_datadir(), self.get_global_name())
merge.sww_merge_parallel(global_name, self.numproc, verbose,
delete_old)
# make sure all the merge completes on processor 0 before other
# processors complete (like when finalize is forgotten in main script)
pypar.barrier()
#pypar.finalize()
raise_(AssertionError, msg)
if __name__ == "__main__":
if numprocs == 1:
runner = unittest.TextTestRunner()
suite = unittest.makeSuite(Test_parallel_boyd_box_apron, 'test')
#print "Running for numproc = 1"
runner.run(suite)
else:
#print "Running for numproc > 1"
import atexit
atexit.register(finalize)
pypar.barrier()
test_points = []
if myid == 0:
if verbose: print('PARALLEL START')
for i in range(samples):
x = random.randrange(0, 1000) / 1000.0 * length
y = random.randrange(0, 1000) / 1000.0 * width
point = [x, y]
test_points.append(point)
for i in range(1, numprocs):
pypar.send(test_points, i)
else:
test_points = pypar.receive(0)
def old_distribute(domain, verbose=False, debug=False, parameters=None):
""" Distribute the domain to all processes
parameters values
"""
if debug:
verbose = True
barrier()
# FIXME: Dummy assignment (until boundaries are refactored to
# be independent of domains until they are applied)
if myid == 0:
bdmap = {}
for tag in domain.get_boundary_tags():
bdmap[tag] = None
domain.set_boundary(bdmap)
if not pypar_available or numprocs == 1: return domain # Bypass
# For some obscure reason this communication must happen prior to
# the more complex mesh distribution - Oh Well!
if myid == 0:
domain_name = domain.get_name()
domain_dir = domain.get_datadir()
domain_store = domain.get_store()
domain_store_centroids = domain.get_store_centroids()
domain_smooth = domain.smooth
domain_reduction = domain.reduction
domain_minimum_storable_height = domain.minimum_storable_height
domain_flow_algorithm = domain.get_flow_algorithm()
domain_minimum_allowed_height = domain.get_minimum_allowed_height()
georef = domain.geo_reference
number_of_global_triangles = domain.number_of_triangles
number_of_global_nodes = domain.number_of_nodes
# FIXME - what other attributes need to be transferred?
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send((domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, number_of_global_nodes), p)
else:
if verbose: print 'P%d: Receiving domain attributes' % (myid)
domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, \
number_of_global_nodes = receive(0)
# Distribute boundary conditions
# FIXME: This cannot handle e.g. Time_boundaries due to
# difficulties pickling functions
if myid == 0:
boundary_map = domain.boundary_map
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send(boundary_map, p)
else:
if verbose: print 'P%d: Receiving boundary map' % (myid)
boundary_map = receive(0)
send_s2p = False
if myid == 0:
# Partition and distribute mesh.
# Structures returned is in the
# correct form for the ANUGA data structure
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles,\
s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width =\
distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
if debug:
print 'P%d' % myid
print 'tri_map ', tri_map
print 'node_map', node_map
print 'tri_l2g', tri_l2g
print 'node_l2g', node_l2g
print 's2p_map', s2p_map
print 'p2s_map', p2s_map
def protocol(x):
vanilla = False
import pypar
control_info, x = pypar.create_control_info(x,
vanilla,
return_object=True)
print 'protocol', control_info[0]
# Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs
if send_s2p:
n = len(s2p_map)
s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(), num.int),
(n, 1))
s2p_map_values_flat = num.array(s2p_map.values(), num.int)
s2p_map_flat = num.concatenate(
(s2p_map_keys_flat, s2p_map_values_flat), axis=1)
n = len(p2s_map)
p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(), num.int),
(n, 2))
p2s_map_values_flat = num.reshape(
num.array(p2s_map.values(), num.int), (n, 1))
p2s_map_flat = num.concatenate(
(p2s_map_keys_flat, p2s_map_values_flat), axis=1)
for p in range(1, numprocs):
# FIXME SR: Creates cPickle dump
send(s2p_map_flat, p)
# FIXME SR: Creates cPickle dump
#print p2s_map
send(p2s_map_flat, p)
else:
if verbose: print 'Not sending s2p_map and p2s_map'
s2p_map = None
p2s_map = None
if verbose: print 'Communication done'
else:
# Read in the mesh partition that belongs to this
# processor
if verbose: print 'P%d: Receiving submeshes' % (myid)
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles, \
tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
rec_submesh(0, verbose)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
# Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping
if send_s2p:
s2p_map_flat = receive(0)
s2p_map = dict.fromkeys(s2p_map_flat[:, 0], s2p_map_flat[:, 1:2])
p2s_map_flat = receive(0)
p2s_map_keys = [tuple(x) for x in p2s_map_flat[:, 0:1]]
p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:, 2])
else:
s2p_map = None
p2s_map = None
#------------------------------------------------------------------------
# Build the domain for this processor using partion structures
#------------------------------------------------------------------------
if verbose:
print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (
myid, number_of_full_nodes, number_of_full_triangles)
domain = Parallel_domain(
points,
vertices,
boundary,
full_send_dict=full_send_dict,
ghost_recv_dict=ghost_recv_dict,
number_of_full_nodes=number_of_full_nodes,
number_of_full_triangles=number_of_full_triangles,
geo_reference=georef,
number_of_global_triangles=number_of_global_triangles,
number_of_global_nodes=number_of_global_nodes,
s2p_map=s2p_map,
p2s_map=p2s_map, ## jj added this
tri_l2g=tri_l2g, ## SR added this
node_l2g=node_l2g,
ghost_layer_width=ghost_layer_width)
#------------------------------------------------------------------------
# Transfer initial conditions to each subdomain
#------------------------------------------------------------------------
for q in quantities:
domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
domain.set_name(domain_name)
domain.set_datadir(domain_dir)
domain.set_store(domain_store)
domain.set_store_centroids(domain_store_centroids)
domain.set_store_vertices_smoothly(domain_smooth, domain_reduction)
domain.set_minimum_storable_height(domain_minimum_storable_height)
domain.set_minimum_allowed_height(domain_minimum_allowed_height)
domain.set_flow_algorithm(domain_flow_algorithm)
domain.geo_reference = georef
#------------------------------------------------------------------------
# Return parallel domain to all nodes
#------------------------------------------------------------------------
return domain
def old_distribute(domain, verbose=False, debug=False, parameters = None):
""" Distribute the domain to all processes
parameters values
"""
if debug:
verbose = True
barrier()
# FIXME: Dummy assignment (until boundaries are refactored to
# be independent of domains until they are applied)
if myid == 0:
bdmap = {}
for tag in domain.get_boundary_tags():
bdmap[tag] = None
domain.set_boundary(bdmap)
if not pypar_available or numprocs == 1 : return domain # Bypass
# For some obscure reason this communication must happen prior to
# the more complex mesh distribution - Oh Well!
if myid == 0:
domain_name = domain.get_name()
domain_dir = domain.get_datadir()
domain_store = domain.get_store()
domain_store_centroids = domain.get_store_centroids()
domain_smooth = domain.smooth
domain_reduction = domain.reduction
domain_minimum_storable_height = domain.minimum_storable_height
domain_flow_algorithm = domain.get_flow_algorithm()
domain_minimum_allowed_height = domain.get_minimum_allowed_height()
georef = domain.geo_reference
number_of_global_triangles = domain.number_of_triangles
number_of_global_nodes = domain.number_of_nodes
# FIXME - what other attributes need to be transferred?
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send((domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, number_of_global_nodes), p)
else:
if verbose: print 'P%d: Receiving domain attributes' %(myid)
domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, \
number_of_global_nodes = receive(0)
# Distribute boundary conditions
# FIXME: This cannot handle e.g. Time_boundaries due to
# difficulties pickling functions
if myid == 0:
boundary_map = domain.boundary_map
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send(boundary_map, p)
else:
if verbose: print 'P%d: Receiving boundary map' %(myid)
boundary_map = receive(0)
send_s2p = False
if myid == 0:
# Partition and distribute mesh.
# Structures returned is in the
# correct form for the ANUGA data structure
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles,\
s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width =\
distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
if debug:
print 'P%d' %myid
print 'tri_map ',tri_map
print 'node_map',node_map
print 'tri_l2g', tri_l2g
print 'node_l2g', node_l2g
print 's2p_map', s2p_map
print 'p2s_map', p2s_map
def protocol(x):
vanilla=False
import pypar
control_info, x = pypar.create_control_info(x, vanilla, return_object=True)
print 'protocol', control_info[0]
# Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs
if send_s2p :
n = len(s2p_map)
s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(),num.int), (n,1) )
s2p_map_values_flat = num.array(s2p_map.values(),num.int)
s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1 )
n = len(p2s_map)
p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(),num.int), (n,2) )
p2s_map_values_flat = num.reshape(num.array(p2s_map.values(),num.int) , (n,1))
p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1 )
for p in range(1, numprocs):
# FIXME SR: Creates cPickle dump
send(s2p_map_flat, p)
# FIXME SR: Creates cPickle dump
#print p2s_map
send(p2s_map_flat, p)
else:
if verbose: print 'Not sending s2p_map and p2s_map'
s2p_map = None
p2s_map = None
if verbose: print 'Communication done'
else:
# Read in the mesh partition that belongs to this
# processor
if verbose: print 'P%d: Receiving submeshes' %(myid)
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles, \
tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
rec_submesh(0, verbose)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
# Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping
if send_s2p :
s2p_map_flat = receive(0)
s2p_map = dict.fromkeys(s2p_map_flat[:,0], s2p_map_flat[:,1:2])
p2s_map_flat = receive(0)
p2s_map_keys = [tuple(x) for x in p2s_map_flat[:,0:1]]
p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:,2])
else:
s2p_map = None
p2s_map = None
#------------------------------------------------------------------------
# Build the domain for this processor using partion structures
#------------------------------------------------------------------------
if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (myid, number_of_full_nodes, number_of_full_triangles)
domain = Parallel_domain(points, vertices, boundary,
full_send_dict=full_send_dict,
ghost_recv_dict=ghost_recv_dict,
number_of_full_nodes=number_of_full_nodes,
number_of_full_triangles=number_of_full_triangles,
geo_reference=georef,
number_of_global_triangles = number_of_global_triangles,
number_of_global_nodes = number_of_global_nodes,
s2p_map = s2p_map,
p2s_map = p2s_map, ## jj added this
tri_l2g = tri_l2g, ## SR added this
node_l2g = node_l2g,
ghost_layer_width = ghost_layer_width)
#------------------------------------------------------------------------
# Transfer initial conditions to each subdomain
#------------------------------------------------------------------------
for q in quantities:
domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
domain.set_name(domain_name)
domain.set_datadir(domain_dir)
domain.set_store(domain_store)
domain.set_store_centroids(domain_store_centroids)
domain.set_store_vertices_smoothly(domain_smooth,domain_reduction)
domain.set_minimum_storable_height(domain_minimum_storable_height)
domain.set_minimum_allowed_height(domain_minimum_allowed_height)
domain.set_flow_algorithm(domain_flow_algorithm)
domain.geo_reference = georef
#------------------------------------------------------------------------
# Return parallel domain to all nodes
#------------------------------------------------------------------------
return domain
