def sequential_distribute_load_pickle_file(pickle_name, np=1, verbose = False): """ Open pickle files """ f = open(pickle_name, 'rb') import pickle kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth, \ domain_low_froude = pickle.load(f) f.close() for k in quantities: quantities[k] = num.load(quantities[k]) points = num.load(points) vertices = num.load(vertices) #--------------------------------------------------------------------------- # Create domain (parallel if np>1) #--------------------------------------------------------------------------- if np>1: domain = Parallel_domain(points, vertices, boundary, **kwargs) else: domain = Domain(points, vertices, boundary, **kwargs) #------------------------------------------------------------------------ # Copy in quantity data #------------------------------------------------------------------------ for q in quantities: domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ domain.set_name(domain_name) domain.set_datadir(domain_dir) domain.set_flow_algorithm(domain_flow_algorithm) domain.set_low_froude(domain_low_froude) domain.set_store(domain_store) domain.set_store_centroids(domain_store_centroids) domain.set_minimum_storable_height(domain_minimum_storable_height) domain.set_minimum_allowed_height(domain_minimum_allowed_height) domain.geo_reference = domain_georef domain.set_quantities_to_be_stored(domain_quantities_to_be_stored) domain.smooth = domain_smooth return domain
def sequential_distribute_load_pickle_file(pickle_name, np=1, verbose = False): """ Open pickle files """ import cPickle f = file(pickle_name, 'rb') kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, georef = cPickle.load(f) f.close() #--------------------------------------------------------------------------- # Create domain (parallel if np>1) #--------------------------------------------------------------------------- if np>1: domain = Parallel_domain(points, vertices, boundary, **kwargs) else: domain = Domain(points, vertices, boundary, **kwargs) #------------------------------------------------------------------------ # Copy in quantity data #------------------------------------------------------------------------ for q in quantities: domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ domain.set_name(domain_name) domain.set_datadir(domain_dir) domain.set_flow_algorithm(domain_flow_algorithm) domain.set_store(domain_store) domain.set_store_centroids(domain_store_centroids) domain.set_minimum_storable_height(domain_minimum_storable_height) domain.set_minimum_allowed_height(domain_minimum_allowed_height) domain.geo_reference = georef return domain
def distribute(domain, verbose=False, debug=False, parameters=None): """ Distribute the domain to all processes parameters allows user to change size of ghost layer """ if not pypar_available or numprocs == 1: return domain # Bypass if myid == 0: from sequential_distribute import Sequential_distribute partition = Sequential_distribute(domain, verbose, debug, parameters) partition.distribute(numprocs) kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = partition.extract_submesh(0) for p in range(1, numprocs): tostore = partition.extract_submesh(p) send(tostore, p) else: kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = receive(0) #--------------------------------------------------------------------------- # Now Create parallel domain #--------------------------------------------------------------------------- parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs) #------------------------------------------------------------------------ # Copy in quantity data #------------------------------------------------------------------------ for q in quantities: try: parallel_domain.set_quantity(q, quantities[q]) except KeyError: #print 'Try to create quantity %s'% q from anuga import Quantity Q = Quantity(parallel_domain, name=q, register=True) parallel_domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary parallel_domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ parallel_domain.set_flow_algorithm(domain_flow_algorithm) parallel_domain.set_name(domain_name) parallel_domain.set_datadir(domain_dir) parallel_domain.set_store(domain_store) parallel_domain.set_store_centroids(domain_store_centroids) parallel_domain.set_minimum_storable_height(domain_minimum_storable_height) parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height) parallel_domain.geo_reference = domain_georef parallel_domain.set_quantities_to_be_stored(domain_quantities_to_be_stored) parallel_domain.smooth = domain_smooth return parallel_domain
def old_distribute(domain, verbose=False, debug=False, parameters=None): """ Distribute the domain to all processes parameters values """ if debug: verbose = True barrier() # FIXME: Dummy assignment (until boundaries are refactored to # be independent of domains until they are applied) if myid == 0: bdmap = {} for tag in domain.get_boundary_tags(): bdmap[tag] = None domain.set_boundary(bdmap) if not pypar_available or numprocs == 1: return domain # Bypass # For some obscure reason this communication must happen prior to # the more complex mesh distribution - Oh Well! if myid == 0: domain_name = domain.get_name() domain_dir = domain.get_datadir() domain_store = domain.get_store() domain_store_centroids = domain.get_store_centroids() domain_smooth = domain.smooth domain_reduction = domain.reduction domain_minimum_storable_height = domain.minimum_storable_height domain_flow_algorithm = domain.get_flow_algorithm() domain_minimum_allowed_height = domain.get_minimum_allowed_height() georef = domain.geo_reference number_of_global_triangles = domain.number_of_triangles number_of_global_nodes = domain.number_of_nodes # FIXME - what other attributes need to be transferred? for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send((domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, number_of_global_nodes), p) else: if verbose: print 'P%d: Receiving domain attributes' % (myid) domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, \ number_of_global_nodes = receive(0) # Distribute boundary conditions # FIXME: This cannot handle e.g. Time_boundaries due to # difficulties pickling functions if myid == 0: boundary_map = domain.boundary_map for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send(boundary_map, p) else: if verbose: print 'P%d: Receiving boundary map' % (myid) boundary_map = receive(0) send_s2p = False if myid == 0: # Partition and distribute mesh. # Structures returned is in the # correct form for the ANUGA data structure points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles,\ s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width =\ distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] if debug: print 'P%d' % myid print 'tri_map ', tri_map print 'node_map', node_map print 'tri_l2g', tri_l2g print 'node_l2g', node_l2g print 's2p_map', s2p_map print 'p2s_map', p2s_map def protocol(x): vanilla = False import pypar control_info, x = pypar.create_control_info(x, vanilla, return_object=True) print 'protocol', control_info[0] # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs if send_s2p: n = len(s2p_map) s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(), num.int), (n, 1)) s2p_map_values_flat = num.array(s2p_map.values(), num.int) s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1) n = len(p2s_map) p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(), num.int), (n, 2)) p2s_map_values_flat = num.reshape( num.array(p2s_map.values(), num.int), (n, 1)) p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1) for p in range(1, numprocs): # FIXME SR: Creates cPickle dump send(s2p_map_flat, p) # FIXME SR: Creates cPickle dump #print p2s_map send(p2s_map_flat, p) else: if verbose: print 'Not sending s2p_map and p2s_map' s2p_map = None p2s_map = None if verbose: print 'Communication done' else: # Read in the mesh partition that belongs to this # processor if verbose: print 'P%d: Receiving submeshes' % (myid) points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles, \ tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\ rec_submesh(0, verbose) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] # Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping if send_s2p: s2p_map_flat = receive(0) s2p_map = dict.fromkeys(s2p_map_flat[:, 0], s2p_map_flat[:, 1:2]) p2s_map_flat = receive(0) p2s_map_keys = [tuple(x) for x in p2s_map_flat[:, 0:1]] p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:, 2]) else: s2p_map = None p2s_map = None #------------------------------------------------------------------------ # Build the domain for this processor using partion structures #------------------------------------------------------------------------ if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % ( myid, number_of_full_nodes, number_of_full_triangles) domain = Parallel_domain( points, vertices, boundary, full_send_dict=full_send_dict, ghost_recv_dict=ghost_recv_dict, number_of_full_nodes=number_of_full_nodes, number_of_full_triangles=number_of_full_triangles, geo_reference=georef, number_of_global_triangles=number_of_global_triangles, number_of_global_nodes=number_of_global_nodes, s2p_map=s2p_map, p2s_map=p2s_map, ## jj added this tri_l2g=tri_l2g, ## SR added this node_l2g=node_l2g, ghost_layer_width=ghost_layer_width) #------------------------------------------------------------------------ # Transfer initial conditions to each subdomain #------------------------------------------------------------------------ for q in quantities: domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ domain.set_name(domain_name) domain.set_datadir(domain_dir) domain.set_store(domain_store) domain.set_store_centroids(domain_store_centroids) domain.set_store_vertices_smoothly(domain_smooth, domain_reduction) domain.set_minimum_storable_height(domain_minimum_storable_height) domain.set_minimum_allowed_height(domain_minimum_allowed_height) domain.set_flow_algorithm(domain_flow_algorithm) domain.geo_reference = georef #------------------------------------------------------------------------ # Return parallel domain to all nodes #------------------------------------------------------------------------ return domain
def distribute(domain, verbose=False, debug=False, parameters = None): """ Distribute the domain to all processes parameters allows user to change size of ghost layer """ if not pypar_available or numprocs == 1 : return domain # Bypass if myid == 0: from sequential_distribute import Sequential_distribute partition = Sequential_distribute(domain, verbose, debug, parameters) partition.distribute(numprocs) kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = partition.extract_submesh(0) for p in range(1, numprocs): tostore = partition.extract_submesh(p) send(tostore,p) else: kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = receive(0) #--------------------------------------------------------------------------- # Now Create parallel domain #--------------------------------------------------------------------------- parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs) #------------------------------------------------------------------------ # Copy in quantity data #------------------------------------------------------------------------ for q in quantities: try: parallel_domain.set_quantity(q, quantities[q]) except KeyError: #print 'Try to create quantity %s'% q from anuga import Quantity Q = Quantity(parallel_domain, name=q, register=True) parallel_domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary parallel_domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ parallel_domain.set_flow_algorithm(domain_flow_algorithm) parallel_domain.set_name(domain_name) parallel_domain.set_datadir(domain_dir) parallel_domain.set_store(domain_store) parallel_domain.set_store_centroids(domain_store_centroids) parallel_domain.set_minimum_storable_height(domain_minimum_storable_height) parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height) parallel_domain.geo_reference = domain_georef parallel_domain.set_quantities_to_be_stored(domain_quantities_to_be_stored) parallel_domain.smooth = domain_smooth return parallel_domain
def old_distribute(domain, verbose=False, debug=False, parameters = None): """ Distribute the domain to all processes parameters values """ if debug: verbose = True barrier() # FIXME: Dummy assignment (until boundaries are refactored to # be independent of domains until they are applied) if myid == 0: bdmap = {} for tag in domain.get_boundary_tags(): bdmap[tag] = None domain.set_boundary(bdmap) if not pypar_available or numprocs == 1 : return domain # Bypass # For some obscure reason this communication must happen prior to # the more complex mesh distribution - Oh Well! if myid == 0: domain_name = domain.get_name() domain_dir = domain.get_datadir() domain_store = domain.get_store() domain_store_centroids = domain.get_store_centroids() domain_smooth = domain.smooth domain_reduction = domain.reduction domain_minimum_storable_height = domain.minimum_storable_height domain_flow_algorithm = domain.get_flow_algorithm() domain_minimum_allowed_height = domain.get_minimum_allowed_height() georef = domain.geo_reference number_of_global_triangles = domain.number_of_triangles number_of_global_nodes = domain.number_of_nodes # FIXME - what other attributes need to be transferred? for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send((domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, number_of_global_nodes), p) else: if verbose: print 'P%d: Receiving domain attributes' %(myid) domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, \ number_of_global_nodes = receive(0) # Distribute boundary conditions # FIXME: This cannot handle e.g. Time_boundaries due to # difficulties pickling functions if myid == 0: boundary_map = domain.boundary_map for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send(boundary_map, p) else: if verbose: print 'P%d: Receiving boundary map' %(myid) boundary_map = receive(0) send_s2p = False if myid == 0: # Partition and distribute mesh. # Structures returned is in the # correct form for the ANUGA data structure points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles,\ s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width =\ distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] if debug: print 'P%d' %myid print 'tri_map ',tri_map print 'node_map',node_map print 'tri_l2g', tri_l2g print 'node_l2g', node_l2g print 's2p_map', s2p_map print 'p2s_map', p2s_map def protocol(x): vanilla=False import pypar control_info, x = pypar.create_control_info(x, vanilla, return_object=True) print 'protocol', control_info[0] # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs if send_s2p : n = len(s2p_map) s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(),num.int), (n,1) ) s2p_map_values_flat = num.array(s2p_map.values(),num.int) s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1 ) n = len(p2s_map) p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(),num.int), (n,2) ) p2s_map_values_flat = num.reshape(num.array(p2s_map.values(),num.int) , (n,1)) p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1 ) for p in range(1, numprocs): # FIXME SR: Creates cPickle dump send(s2p_map_flat, p) # FIXME SR: Creates cPickle dump #print p2s_map send(p2s_map_flat, p) else: if verbose: print 'Not sending s2p_map and p2s_map' s2p_map = None p2s_map = None if verbose: print 'Communication done' else: # Read in the mesh partition that belongs to this # processor if verbose: print 'P%d: Receiving submeshes' %(myid) points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles, \ tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\ rec_submesh(0, verbose) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] # Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping if send_s2p : s2p_map_flat = receive(0) s2p_map = dict.fromkeys(s2p_map_flat[:,0], s2p_map_flat[:,1:2]) p2s_map_flat = receive(0) p2s_map_keys = [tuple(x) for x in p2s_map_flat[:,0:1]] p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:,2]) else: s2p_map = None p2s_map = None #------------------------------------------------------------------------ # Build the domain for this processor using partion structures #------------------------------------------------------------------------ if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (myid, number_of_full_nodes, number_of_full_triangles) domain = Parallel_domain(points, vertices, boundary, full_send_dict=full_send_dict, ghost_recv_dict=ghost_recv_dict, number_of_full_nodes=number_of_full_nodes, number_of_full_triangles=number_of_full_triangles, geo_reference=georef, number_of_global_triangles = number_of_global_triangles, number_of_global_nodes = number_of_global_nodes, s2p_map = s2p_map, p2s_map = p2s_map, ## jj added this tri_l2g = tri_l2g, ## SR added this node_l2g = node_l2g, ghost_layer_width = ghost_layer_width) #------------------------------------------------------------------------ # Transfer initial conditions to each subdomain #------------------------------------------------------------------------ for q in quantities: domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ domain.set_name(domain_name) domain.set_datadir(domain_dir) domain.set_store(domain_store) domain.set_store_centroids(domain_store_centroids) domain.set_store_vertices_smoothly(domain_smooth,domain_reduction) domain.set_minimum_storable_height(domain_minimum_storable_height) domain.set_minimum_allowed_height(domain_minimum_allowed_height) domain.set_flow_algorithm(domain_flow_algorithm) domain.geo_reference = georef #------------------------------------------------------------------------ # Return parallel domain to all nodes #------------------------------------------------------------------------ return domain