def distribute(domain, verbose=False, debug=False, parameters=None):
""" Distribute the domain to all processes
parameters allows user to change size of ghost layer
"""
if not pypar_available or numprocs == 1: return domain # Bypass
if myid == 0:
from sequential_distribute import Sequential_distribute
partition = Sequential_distribute(domain, verbose, debug, parameters)
partition.distribute(numprocs)
kwargs, points, vertices, boundary, quantities, boundary_map, \
domain_name, domain_dir, domain_store, domain_store_centroids, \
domain_minimum_storable_height, domain_minimum_allowed_height, \
domain_flow_algorithm, domain_georef, \
domain_quantities_to_be_stored, domain_smooth \
= partition.extract_submesh(0)
for p in range(1, numprocs):
tostore = partition.extract_submesh(p)
send(tostore, p)
else:
kwargs, points, vertices, boundary, quantities, boundary_map, \
domain_name, domain_dir, domain_store, domain_store_centroids, \
domain_minimum_storable_height, domain_minimum_allowed_height, \
domain_flow_algorithm, domain_georef, \
domain_quantities_to_be_stored, domain_smooth \
= receive(0)
#---------------------------------------------------------------------------
# Now Create parallel domain
#---------------------------------------------------------------------------
parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs)
#------------------------------------------------------------------------
# Copy in quantity data
#------------------------------------------------------------------------
for q in quantities:
try:
parallel_domain.set_quantity(q, quantities[q])
except KeyError:
#print 'Try to create quantity %s'% q
from anuga import Quantity
Q = Quantity(parallel_domain, name=q, register=True)
parallel_domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
parallel_domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
parallel_domain.set_flow_algorithm(domain_flow_algorithm)
parallel_domain.set_name(domain_name)
parallel_domain.set_datadir(domain_dir)
parallel_domain.set_store(domain_store)
parallel_domain.set_store_centroids(domain_store_centroids)
parallel_domain.set_minimum_storable_height(domain_minimum_storable_height)
parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height)
parallel_domain.geo_reference = domain_georef
parallel_domain.set_quantities_to_be_stored(domain_quantities_to_be_stored)
parallel_domain.smooth = domain_smooth
return parallel_domain
def old_distribute(domain, verbose=False, debug=False, parameters=None):
""" Distribute the domain to all processes
parameters values
"""
if debug:
verbose = True
barrier()
# FIXME: Dummy assignment (until boundaries are refactored to
# be independent of domains until they are applied)
if myid == 0:
bdmap = {}
for tag in domain.get_boundary_tags():
bdmap[tag] = None
domain.set_boundary(bdmap)
if not pypar_available or numprocs == 1: return domain # Bypass
# For some obscure reason this communication must happen prior to
# the more complex mesh distribution - Oh Well!
if myid == 0:
domain_name = domain.get_name()
domain_dir = domain.get_datadir()
domain_store = domain.get_store()
domain_store_centroids = domain.get_store_centroids()
domain_smooth = domain.smooth
domain_reduction = domain.reduction
domain_minimum_storable_height = domain.minimum_storable_height
domain_flow_algorithm = domain.get_flow_algorithm()
domain_minimum_allowed_height = domain.get_minimum_allowed_height()
georef = domain.geo_reference
number_of_global_triangles = domain.number_of_triangles
number_of_global_nodes = domain.number_of_nodes
# FIXME - what other attributes need to be transferred?
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send((domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, number_of_global_nodes), p)
else:
if verbose: print 'P%d: Receiving domain attributes' % (myid)
domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, \
number_of_global_nodes = receive(0)
# Distribute boundary conditions
# FIXME: This cannot handle e.g. Time_boundaries due to
# difficulties pickling functions
if myid == 0:
boundary_map = domain.boundary_map
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send(boundary_map, p)
else:
if verbose: print 'P%d: Receiving boundary map' % (myid)
boundary_map = receive(0)
send_s2p = False
if myid == 0:
# Partition and distribute mesh.
# Structures returned is in the
# correct form for the ANUGA data structure
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles,\
s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width =\
distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
if debug:
print 'P%d' % myid
print 'tri_map ', tri_map
print 'node_map', node_map
print 'tri_l2g', tri_l2g
print 'node_l2g', node_l2g
print 's2p_map', s2p_map
print 'p2s_map', p2s_map
def protocol(x):
vanilla = False
import pypar
control_info, x = pypar.create_control_info(x,
vanilla,
return_object=True)
print 'protocol', control_info[0]
# Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs
if send_s2p:
n = len(s2p_map)
s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(), num.int),
(n, 1))
s2p_map_values_flat = num.array(s2p_map.values(), num.int)
s2p_map_flat = num.concatenate(
(s2p_map_keys_flat, s2p_map_values_flat), axis=1)
n = len(p2s_map)
p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(), num.int),
(n, 2))
p2s_map_values_flat = num.reshape(
num.array(p2s_map.values(), num.int), (n, 1))
p2s_map_flat = num.concatenate(
(p2s_map_keys_flat, p2s_map_values_flat), axis=1)
for p in range(1, numprocs):
# FIXME SR: Creates cPickle dump
send(s2p_map_flat, p)
# FIXME SR: Creates cPickle dump
#print p2s_map
send(p2s_map_flat, p)
else:
if verbose: print 'Not sending s2p_map and p2s_map'
s2p_map = None
p2s_map = None
if verbose: print 'Communication done'
else:
# Read in the mesh partition that belongs to this
# processor
if verbose: print 'P%d: Receiving submeshes' % (myid)
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles, \
tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
rec_submesh(0, verbose)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
# Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping
if send_s2p:
s2p_map_flat = receive(0)
s2p_map = dict.fromkeys(s2p_map_flat[:, 0], s2p_map_flat[:, 1:2])
p2s_map_flat = receive(0)
p2s_map_keys = [tuple(x) for x in p2s_map_flat[:, 0:1]]
p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:, 2])
else:
s2p_map = None
p2s_map = None
#------------------------------------------------------------------------
# Build the domain for this processor using partion structures
#------------------------------------------------------------------------
if verbose:
print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (
myid, number_of_full_nodes, number_of_full_triangles)
domain = Parallel_domain(
points,
vertices,
boundary,
full_send_dict=full_send_dict,
ghost_recv_dict=ghost_recv_dict,
number_of_full_nodes=number_of_full_nodes,
number_of_full_triangles=number_of_full_triangles,
geo_reference=georef,
number_of_global_triangles=number_of_global_triangles,
number_of_global_nodes=number_of_global_nodes,
s2p_map=s2p_map,
p2s_map=p2s_map, ## jj added this
tri_l2g=tri_l2g, ## SR added this
node_l2g=node_l2g,
ghost_layer_width=ghost_layer_width)
#------------------------------------------------------------------------
# Transfer initial conditions to each subdomain
#------------------------------------------------------------------------
for q in quantities:
domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
domain.set_name(domain_name)
domain.set_datadir(domain_dir)
domain.set_store(domain_store)
domain.set_store_centroids(domain_store_centroids)
domain.set_store_vertices_smoothly(domain_smooth, domain_reduction)
domain.set_minimum_storable_height(domain_minimum_storable_height)
domain.set_minimum_allowed_height(domain_minimum_allowed_height)
domain.set_flow_algorithm(domain_flow_algorithm)
domain.geo_reference = georef
#------------------------------------------------------------------------
# Return parallel domain to all nodes
#------------------------------------------------------------------------
return domain
def sequential_distribute_load_pickle_file(pickle_name, np=1, verbose=False):
"""
Open pickle files
"""
import cPickle
f = file(pickle_name, 'rb')
kwargs, points, vertices, boundary, quantities, boundary_map, \
domain_name, domain_dir, domain_store, domain_store_centroids, \
domain_minimum_storable_height, domain_minimum_allowed_height, \
domain_flow_algorithm, georef = cPickle.load(f)
f.close()
#---------------------------------------------------------------------------
# Create domain (parallel if np>1)
#---------------------------------------------------------------------------
if np > 1:
domain = Parallel_domain(points, vertices, boundary, **kwargs)
else:
domain = Domain(points, vertices, boundary, **kwargs)
#------------------------------------------------------------------------
# Copy in quantity data
#------------------------------------------------------------------------
for q in quantities:
domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
domain.set_name(domain_name)
domain.set_datadir(domain_dir)
domain.set_flow_algorithm(domain_flow_algorithm)
domain.set_store(domain_store)
domain.set_store_centroids(domain_store_centroids)
domain.set_minimum_storable_height(domain_minimum_storable_height)
domain.set_minimum_allowed_height(domain_minimum_allowed_height)
domain.geo_reference = georef
return domain
def distribute(domain, verbose=False, debug=False, parameters = None):
""" Distribute the domain to all processes
parameters allows user to change size of ghost layer
"""
if not pypar_available or numprocs == 1 : return domain # Bypass
if myid == 0:
from sequential_distribute import Sequential_distribute
partition = Sequential_distribute(domain, verbose, debug, parameters)
partition.distribute(numprocs)
kwargs, points, vertices, boundary, quantities, boundary_map, \
domain_name, domain_dir, domain_store, domain_store_centroids, \
domain_minimum_storable_height, domain_minimum_allowed_height, \
domain_flow_algorithm, domain_georef, \
domain_quantities_to_be_stored, domain_smooth \
= partition.extract_submesh(0)
for p in range(1, numprocs):
tostore = partition.extract_submesh(p)
send(tostore,p)
else:
kwargs, points, vertices, boundary, quantities, boundary_map, \
domain_name, domain_dir, domain_store, domain_store_centroids, \
domain_minimum_storable_height, domain_minimum_allowed_height, \
domain_flow_algorithm, domain_georef, \
domain_quantities_to_be_stored, domain_smooth \
= receive(0)
#---------------------------------------------------------------------------
# Now Create parallel domain
#---------------------------------------------------------------------------
parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs)
#------------------------------------------------------------------------
# Copy in quantity data
#------------------------------------------------------------------------
for q in quantities:
try:
parallel_domain.set_quantity(q, quantities[q])
except KeyError:
#print 'Try to create quantity %s'% q
from anuga import Quantity
Q = Quantity(parallel_domain, name=q, register=True)
parallel_domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
parallel_domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
parallel_domain.set_flow_algorithm(domain_flow_algorithm)
parallel_domain.set_name(domain_name)
parallel_domain.set_datadir(domain_dir)
parallel_domain.set_store(domain_store)
parallel_domain.set_store_centroids(domain_store_centroids)
parallel_domain.set_minimum_storable_height(domain_minimum_storable_height)
parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height)
parallel_domain.geo_reference = domain_georef
parallel_domain.set_quantities_to_be_stored(domain_quantities_to_be_stored)
parallel_domain.smooth = domain_smooth
return parallel_domain
def old_distribute(domain, verbose=False, debug=False, parameters = None):
""" Distribute the domain to all processes
parameters values
"""
if debug:
verbose = True
barrier()
# FIXME: Dummy assignment (until boundaries are refactored to
# be independent of domains until they are applied)
if myid == 0:
bdmap = {}
for tag in domain.get_boundary_tags():
bdmap[tag] = None
domain.set_boundary(bdmap)
if not pypar_available or numprocs == 1 : return domain # Bypass
# For some obscure reason this communication must happen prior to
# the more complex mesh distribution - Oh Well!
if myid == 0:
domain_name = domain.get_name()
domain_dir = domain.get_datadir()
domain_store = domain.get_store()
domain_store_centroids = domain.get_store_centroids()
domain_smooth = domain.smooth
domain_reduction = domain.reduction
domain_minimum_storable_height = domain.minimum_storable_height
domain_flow_algorithm = domain.get_flow_algorithm()
domain_minimum_allowed_height = domain.get_minimum_allowed_height()
georef = domain.geo_reference
number_of_global_triangles = domain.number_of_triangles
number_of_global_nodes = domain.number_of_nodes
# FIXME - what other attributes need to be transferred?
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send((domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, number_of_global_nodes), p)
else:
if verbose: print 'P%d: Receiving domain attributes' %(myid)
domain_name, domain_dir, domain_store, \
domain_store_centroids, domain_smooth, domain_reduction, \
domain_minimum_storable_height, domain_flow_algorithm, \
domain_minimum_allowed_height, georef, \
number_of_global_triangles, \
number_of_global_nodes = receive(0)
# Distribute boundary conditions
# FIXME: This cannot handle e.g. Time_boundaries due to
# difficulties pickling functions
if myid == 0:
boundary_map = domain.boundary_map
for p in xrange(1, numprocs):
# FIXME SR: Creates cPickle dump
send(boundary_map, p)
else:
if verbose: print 'P%d: Receiving boundary map' %(myid)
boundary_map = receive(0)
send_s2p = False
if myid == 0:
# Partition and distribute mesh.
# Structures returned is in the
# correct form for the ANUGA data structure
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles,\
s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
ghost_layer_width =\
distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
if debug:
print 'P%d' %myid
print 'tri_map ',tri_map
print 'node_map',node_map
print 'tri_l2g', tri_l2g
print 'node_l2g', node_l2g
print 's2p_map', s2p_map
print 'p2s_map', p2s_map
def protocol(x):
vanilla=False
import pypar
control_info, x = pypar.create_control_info(x, vanilla, return_object=True)
print 'protocol', control_info[0]
# Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs
if send_s2p :
n = len(s2p_map)
s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(),num.int), (n,1) )
s2p_map_values_flat = num.array(s2p_map.values(),num.int)
s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1 )
n = len(p2s_map)
p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(),num.int), (n,2) )
p2s_map_values_flat = num.reshape(num.array(p2s_map.values(),num.int) , (n,1))
p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1 )
for p in range(1, numprocs):
# FIXME SR: Creates cPickle dump
send(s2p_map_flat, p)
# FIXME SR: Creates cPickle dump
#print p2s_map
send(p2s_map_flat, p)
else:
if verbose: print 'Not sending s2p_map and p2s_map'
s2p_map = None
p2s_map = None
if verbose: print 'Communication done'
else:
# Read in the mesh partition that belongs to this
# processor
if verbose: print 'P%d: Receiving submeshes' %(myid)
points, vertices, boundary, quantities,\
ghost_recv_dict, full_send_dict,\
number_of_full_nodes, number_of_full_triangles, \
tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
rec_submesh(0, verbose)
# Extract l2g maps
#tri_l2g = extract_l2g_map(tri_map)
#node_l2g = extract_l2g_map(node_map)
#tri_l2g = p2s_map[tri_l2g]
# Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping
if send_s2p :
s2p_map_flat = receive(0)
s2p_map = dict.fromkeys(s2p_map_flat[:,0], s2p_map_flat[:,1:2])
p2s_map_flat = receive(0)
p2s_map_keys = [tuple(x) for x in p2s_map_flat[:,0:1]]
p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:,2])
else:
s2p_map = None
p2s_map = None
#------------------------------------------------------------------------
# Build the domain for this processor using partion structures
#------------------------------------------------------------------------
if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (myid, number_of_full_nodes, number_of_full_triangles)
domain = Parallel_domain(points, vertices, boundary,
full_send_dict=full_send_dict,
ghost_recv_dict=ghost_recv_dict,
number_of_full_nodes=number_of_full_nodes,
number_of_full_triangles=number_of_full_triangles,
geo_reference=georef,
number_of_global_triangles = number_of_global_triangles,
number_of_global_nodes = number_of_global_nodes,
s2p_map = s2p_map,
p2s_map = p2s_map, ## jj added this
tri_l2g = tri_l2g, ## SR added this
node_l2g = node_l2g,
ghost_layer_width = ghost_layer_width)
#------------------------------------------------------------------------
# Transfer initial conditions to each subdomain
#------------------------------------------------------------------------
for q in quantities:
domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
domain.set_name(domain_name)
domain.set_datadir(domain_dir)
domain.set_store(domain_store)
domain.set_store_centroids(domain_store_centroids)
domain.set_store_vertices_smoothly(domain_smooth,domain_reduction)
domain.set_minimum_storable_height(domain_minimum_storable_height)
domain.set_minimum_allowed_height(domain_minimum_allowed_height)
domain.set_flow_algorithm(domain_flow_algorithm)
domain.geo_reference = georef
#------------------------------------------------------------------------
# Return parallel domain to all nodes
#------------------------------------------------------------------------
return domain
def sequential_distribute_load_pickle_file(pickle_name, np=1, verbose = False):
"""
Open pickle files
"""
import cPickle
f = file(pickle_name, 'rb')
kwargs, points, vertices, boundary, quantities, boundary_map, \
domain_name, domain_dir, domain_store, domain_store_centroids, \
domain_minimum_storable_height, domain_minimum_allowed_height, \
domain_flow_algorithm, georef = cPickle.load(f)
f.close()
#---------------------------------------------------------------------------
# Create domain (parallel if np>1)
#---------------------------------------------------------------------------
if np>1:
domain = Parallel_domain(points, vertices, boundary, **kwargs)
else:
domain = Domain(points, vertices, boundary, **kwargs)
#------------------------------------------------------------------------
# Copy in quantity data
#------------------------------------------------------------------------
for q in quantities:
domain.set_quantity(q, quantities[q])
#------------------------------------------------------------------------
# Transfer boundary conditions to each subdomain
#------------------------------------------------------------------------
boundary_map['ghost'] = None # Add binding to ghost boundary
domain.set_boundary(boundary_map)
#------------------------------------------------------------------------
# Transfer other attributes to each subdomain
#------------------------------------------------------------------------
domain.set_name(domain_name)
domain.set_datadir(domain_dir)
domain.set_flow_algorithm(domain_flow_algorithm)
domain.set_store(domain_store)
domain.set_store_centroids(domain_store_centroids)
domain.set_minimum_storable_height(domain_minimum_storable_height)
domain.set_minimum_allowed_height(domain_minimum_allowed_height)
domain.geo_reference = georef
return domain