def distribute_mesh(domain, verbose=False, debug=False, parameters=None): """ Distribute andsend the mesh info to all the processors. Should only be run from processor 0 and will send info to the other processors. There should be a corresponding rec_submesh called from all the other processors """ if debug: verbose = True numprocs = size() # Subdivide the mesh if verbose: print 'Subdivide mesh' new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \ s2p_map, p2s_map = \ pmesh_divide_metis_with_map(domain, numprocs) #PETE: s2p_map (maps serial domain triangles to parallel domain triangles) # sp2_map (maps parallel domain triangles to domain triangles) # Build the mesh that should be assigned to each processor, # this includes ghost nodes and the communication pattern if verbose: print 'Build submeshes' submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities, triangles_per_proc, parameters) if verbose: for p in range(numprocs): N = len(submesh['ghost_nodes'][p]) M = len(submesh['ghost_triangles'][p]) print 'There are %d ghost nodes and %d ghost triangles on proc %d'\ %(N, M, p) #if debug: # from pprint import pprint # pprint(submesh) # Send the mesh partition to the appropriate processor if verbose: print 'Distribute submeshes' for p in range(1, numprocs): send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose) # Build the local mesh for processor 0 points, vertices, boundary, quantities, \ ghost_recv_dict, full_send_dict, \ tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\ extract_submesh(submesh, triangles_per_proc, p2s_map, 0) # Keep track of the number full nodes and triangles. # This is useful later if one needs access to a ghost-free domain # Here, we do it for process 0. The others are done in rec_submesh. number_of_full_nodes = len(submesh['full_nodes'][0]) number_of_full_triangles = len(submesh['full_triangles'][0]) #print #for p in range(numprocs): # print 'Process %d:' %(p) # # print 'full_triangles:' # print submesh['full_triangles'][p] # # print 'full_nodes:' # print submesh['full_nodes'][p] # # print 'ghost_triangles:' # print submesh['ghost_triangles'][p]# # # print 'ghost_nodes:' # print submesh['ghost_nodes'][p] # print # #print 'Receive dict' #print ghost_recv_dict # #print 'Send dict' #print full_send_dict # Return structures necessary for building the parallel domain return points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles, \ s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width
def distibute_three_processors(): """ Do a parallel test of distributing a rectangle onto 3 processors """ # FIXME: Need to update expected values on macos if sys.platform == 'darwin': return # FIXME: Need to update expected values on macos #if sys.platform == 'win32': # return from anuga.utilities import parallel_abstraction as pypar myid = pypar.rank() numprocs = pypar.size() if not numprocs == 3: return try: import pymetis metis_version = 5 except: metis_version = 4 #print numprocs #barrier() if myid == 0: nodes_0, triangles_0, boundary_0 = rectangular_cross(2, 2) domain = Domain(nodes_0, triangles_0, boundary_0) domain.set_quantity('elevation', topography) # Use function for elevation domain.set_quantity('friction', 0.0) # Constant friction domain.set_quantity('stage', expression='elevation') # Dry initial stage domain.set_quantity('xmomentum', expression='friction + 2.0') domain.set_quantity('ymomentum', ycoord) #---------------------------------------------------------------------------------- # Test pmesh_divide_metis #---------------------------------------------------------------------------------- vertices, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis( domain, numprocs) if False: print_seq_values(vertices, triangles, triangles_per_proc) true_seq_values = get_true_seq_values(metis_version=metis_version) if False: print("True Seq Values = \\") pprint(true_seq_values) assert_allclose(vertices, true_seq_values['vertices']) assert_allclose(triangles, true_seq_values['triangles']) assert_allclose(triangles_per_proc, true_seq_values['triangles_per_proc']) #---------------------------------------------------------------------------------- # Test build_submesh #---------------------------------------------------------------------------------- submesh = build_submesh(vertices, triangles, boundary, quantities, triangles_per_proc) if False: print('submesh_values = \\') print_submesh_values(submesh) true_values = get_true_submesh_values(metis_version) assert_allclose(submesh['full_nodes'][0], true_values['full_nodes_0']) assert_allclose(submesh['full_nodes'][1], true_values['full_nodes_1']) assert_allclose(submesh['full_nodes'][2], true_values['full_nodes_2']) assert_allclose(submesh['ghost_nodes'][0], true_values['ghost_nodes_0']) assert_allclose(submesh['ghost_nodes'][1], true_values['ghost_nodes_1']) assert_allclose(submesh['ghost_nodes'][2], true_values['ghost_nodes_2']) assert_allclose(submesh['full_triangles'][0], true_values['full_triangles_0']) assert_allclose(submesh['full_triangles'][1], true_values['full_triangles_1']) assert_allclose(submesh['full_triangles'][2], true_values['full_triangles_2']) assert_allclose(submesh['ghost_triangles'][0], true_values['ghost_triangles_0']) assert_allclose(submesh['ghost_triangles'][1], true_values['ghost_triangles_1']) assert_allclose(submesh['ghost_triangles'][2], true_values['ghost_triangles_2']) assert_allclose(submesh['ghost_commun'][0], true_values['ghost_commun_0']) assert_allclose(submesh['ghost_commun'][1], true_values['ghost_commun_1']) assert_allclose(submesh['ghost_commun'][2], true_values['ghost_commun_2']) assert_(submesh['full_commun'] == true_values['full_commun']) barrier() #-------------------------------- # Now do the comunnication part #-------------------------------- if myid == 0: points, vertices, boundary, quantities, \ ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width =\ extract_submesh(submesh, triangles_per_proc) #---------------------------------------------------------------------------------- # Test send_submesh #---------------------------------------------------------------------------------- for p in range(1, numprocs): send_submesh(submesh, triangles_per_proc, p, verbose=False) else: #---------------------------------------------------------------------------------- # Test rec_submesh #---------------------------------------------------------------------------------- points, triangles, boundary, quantities, \ ghost_recv_dict, full_send_dict, \ no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width = \ rec_submesh(0, verbose=False) barrier() #-------------------------------- # Now do the test #-------------------------------- if myid == 0: if False: print('extract_values = \\') print_extract_submesh(points, triangles, ghost_recv_dict, \ full_send_dict, tri_map, node_map, ghost_layer_width) true_values = get_true_extract_submesh(metis_version) assert_allclose(points, true_values['points']) assert_allclose(triangles, true_values['triangles']) assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1']) assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2']) assert_allclose(full_send_dict[1], true_values['full_send_dict_1']) assert_allclose(full_send_dict[2], true_values['full_send_dict_2']) assert_allclose(tri_map, true_values['tri_map']) assert_allclose(node_map, true_values['node_map']) assert_allclose(ghost_layer_width, true_values['ghost_layer_width']) if myid == 1: if False: print("rec_submesh_1 = \\") print_rec_submesh_1(points, triangles, ghost_recv_dict, full_send_dict, \ tri_map, node_map, ghost_layer_width) true_values = get_true_rec_submesh_1(metis_version) if False: print('true_rec_values_1 = \\') pprint(true_values) assert_allclose(points, true_values['points']) assert_allclose(triangles, true_values['triangles']) assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0']) assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2']) assert_allclose(full_send_dict[0], true_values['full_send_dict_0']) assert_allclose(full_send_dict[2], true_values['full_send_dict_2']) assert_allclose(tri_map, true_values['tri_map']) assert_allclose(node_map, true_values['node_map']) assert_allclose(ghost_layer_width, true_values['ghost_layer_width']) if myid == 2: if False: print("rec_submesh_2 = \\") print_rec_submesh_2(points, triangles, ghost_recv_dict, full_send_dict, \ tri_map, node_map, ghost_layer_width) true_values = get_true_rec_submesh_2(metis_version) if False: print('true_rec_values_2 = \\') pprint(true_values) assert_allclose(points, true_values['points']) assert_allclose(triangles, true_values['triangles']) assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0']) assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1']) assert_allclose(full_send_dict[0], true_values['full_send_dict_0']) assert_allclose(full_send_dict[1], true_values['full_send_dict_1']) assert_allclose(tri_map, true_values['tri_map']) assert_allclose(node_map, true_values['node_map']) assert_allclose(ghost_layer_width, true_values['ghost_layer_width']) finalize()
def distibute_three_processors(): """ Do a parallel test of distributing a rectangle onto 3 processors """ # FIXME: Need to update expected values on macos if sys.platform == 'darwin': return import pypar myid = pypar.rank() numprocs = pypar.size() if not numprocs == 3: return #print numprocs barrier() if myid == 0: points, vertices, boundary = rectangular_cross(2,2) domain = Domain(points, vertices, boundary) domain.set_quantity('elevation', topography) # Use function for elevation domain.set_quantity('friction', 0.0) # Constant friction domain.set_quantity('stage', expression='elevation') # Dry initial stage domain.set_quantity('xmomentum', expression='friction + 2.0') # domain.set_quantity('ymomentum', ycoord) # #---------------------------------------------------------------------------------- # Test pmesh_divide_metis #---------------------------------------------------------------------------------- nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,numprocs) assert_(num.allclose(nodes,points)) true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]] true_triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]] assert_(num.allclose(vertices,true_vertices)) assert_(num.allclose(triangles,true_triangles)) assert_(num.allclose(triangles_per_proc,[5,6,5])) #---------------------------------------------------------------------------------- # Test build_submesh #---------------------------------------------------------------------------------- submesh = build_submesh(nodes, triangles, boundary, quantities, triangles_per_proc) assert_(num.allclose(submesh['full_nodes'][0],[[3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [9.0, 0.25, 0.25], [12.0, 0.75, 0.75]])) assert_(num.allclose(submesh['full_nodes'][1],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [9.0, 0.25, 0.25], [10.0, 0.25, 0.75]])) assert_(num.allclose(submesh['full_nodes'][2],[[0.0, 0.0, 0.0], [3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [6.0, 1.0, 0.0], [7.0, 1.0, 0.5], [9.0, 0.25, 0.25], [11.0, 0.75, 0.25]])) assert_(num.allclose(submesh['ghost_nodes'][0],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [6.0, 1.0, 0.0], [10.0, 0.25, 0.75], [11.0, 0.75, 0.25]])) assert_(num.allclose(submesh['ghost_nodes'][1],[[3.0, 0.5, 0.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [11.0, 0.75, 0.25], [12.0, 0.75, 0.75]])) assert_(num.allclose(submesh['ghost_nodes'][2],[[1.0, 0.0, 0.5], [5.0, 0.5, 1.0], [8.0, 1.0, 1.0], [12.0, 0.75, 0.75]])) true_full_triangles = [num.array([[ 4, 9, 3], [ 4, 12, 5], [ 7, 12, 4], [ 8, 12, 7], [ 5, 12, 8]]), num.array([[ 0, 9, 1], [ 1, 9, 4], [ 1, 10, 2], [ 4, 10, 1], [ 5, 10, 4], [ 2, 10, 5]]), num.array([[ 3, 9, 0], [ 3, 11, 4], [ 6, 11, 3], [ 7, 11, 6], [ 4, 11, 7]])] assert_(num.allclose(submesh['full_triangles'][0],true_full_triangles[0])) assert_(num.allclose(submesh['full_triangles'][1],true_full_triangles[1])) assert_(num.allclose(submesh['full_triangles'][2],true_full_triangles[2])) true_ghost_triangles = [num.array([[ 5, 0, 9, 1], [ 6, 1, 9, 4], [ 8, 4, 10, 1], [ 9, 5, 10, 4], [10, 2, 10, 5], [11, 3, 9, 0], [12, 3, 11, 4], [13, 6, 11, 3], [14, 7, 11, 6], [15, 4, 11, 7]]), num.array([[ 0, 4, 9, 3], [ 1, 4, 12, 5], [ 2, 7, 12, 4], [ 4, 5, 12, 8], [11, 3, 9, 0], [12, 3, 11, 4]]), num.array([[ 0, 4, 9, 3], [ 1, 4, 12, 5], [ 2, 7, 12, 4], [ 3, 8, 12, 7], [ 5, 0, 9, 1], [ 6, 1, 9, 4]])] assert_(num.allclose(submesh['ghost_triangles'][0],true_ghost_triangles[0])) assert_(num.allclose(submesh['ghost_triangles'][1],true_ghost_triangles[1])) assert_(num.allclose(submesh['ghost_triangles'][2],true_ghost_triangles[2])) true_full_commun = [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}] assert_(true_full_commun == submesh['full_commun']) true_ghost_commun = [num.array([[ 5, 1], [ 6, 1], [ 8, 1], [ 9, 1], [10, 1], [11, 2], [12, 2], [13, 2], [14, 2], [15, 2]]), num.array([[ 0, 0], [ 1, 0], [ 2, 0], [ 4, 0], [11, 2], [12, 2]]), num.array([[0, 0], [1, 0], [2, 0], [3, 0], [5, 1], [6, 1]])] assert_(num.allclose(submesh['ghost_commun'][0],true_ghost_commun[0])) assert_(num.allclose(submesh['ghost_commun'][1],true_ghost_commun[1])) assert_(num.allclose(submesh['ghost_commun'][2],true_ghost_commun[2])) barrier() #-------------------------------- # Now do the comunnication part #-------------------------------- if myid == 0: #---------------------------------------------------------------------------------- # Test send_submesh #---------------------------------------------------------------------------------- for p in range(1, numprocs): send_submesh(submesh, triangles_per_proc, p, verbose=False) #---------------------------------------------------------------------------------- # Test extract_submesh #---------------------------------------------------------------------------------- points, vertices, boundary, quantities, \ ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width =\ extract_submesh(submesh, triangles_per_proc) true_points = [[0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.75, 0.75], [0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [1.0, 0.0], [0.25, 0.75], [0.75, 0.25]] true_vertices = [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [7, 5, 8], [8, 5, 1], [1, 11, 8], [2, 11, 1], [9, 11, 2], [0, 5, 7], [0, 12, 1], [10, 12, 0], [3, 12, 10], [1, 12, 3]] true_ghost_recv = {1: [num.array([5, 6, 7, 8, 9]), num.array([ 5, 6, 8, 9, 10])], 2: [num.array([10, 11, 12, 13, 14]), num.array([11, 12, 13, 14, 15])]} true_full_send = {1: [num.array([0, 1, 2, 4]), num.array([0, 1, 2, 4])], 2: [num.array([0, 1, 2, 3]), num.array([0, 1, 2, 3])]} assert_(num.allclose(ghost_layer_width, 2)) assert_(num.allclose(points, true_points)) assert_(num.allclose(vertices, true_vertices)) assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1])) assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2])) assert_(num.allclose(full_send_dict[1],true_full_send[1])) assert_(num.allclose(full_send_dict[2],true_full_send[2])) #print triangles_per_proc else: #---------------------------------------------------------------------------------- # Test rec_submesh #---------------------------------------------------------------------------------- points, vertices, boundary, quantities, \ ghost_recv_dict, full_send_dict, \ no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width = \ rec_submesh(0, verbose=False) if myid == 1: true_points = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.5], [0.5, 1.0], [0.25, 0.25], [0.25, 0.75], [0.5, 0.0], [1.0, 0.5], [1.0, 1.0], [0.75, 0.25], [0.75, 0.75]] true_vertices = [[0, 5, 1], [1, 5, 3], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [3, 5, 7], [3, 11, 4], [8, 11, 3], [4, 11, 9], [7, 5, 0], [7, 10, 3]] true_ghost_recv = {0: [num.array([6, 7, 8, 9]), num.array([0, 1, 2, 4])], 2: [num.array([10, 11]), num.array([11, 12])]} true_full_send = {0: [num.array([0, 1, 3, 4, 5]), num.array([ 5, 6, 8, 9, 10])], 2: [num.array([0, 1]), num.array([5, 6])]} true_tri_map = num.array([ 6, 7, 8, -1, 9, 0, 1, 2, 3, 4, 5, 10, 11]) true_node_map = num.array([ 0, 1, 2, 7, 3, 4, -1, 8, 9, 5, 6, 10, 11]) assert_(num.allclose(ghost_layer_width, 2)) assert_(num.allclose(tri_map, true_tri_map)) assert_(num.allclose(node_map, true_node_map)) assert_(num.allclose(points, true_points)) assert_(num.allclose(vertices, true_vertices)) assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0])) assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2])) assert_(num.allclose(full_send_dict[0],true_full_send[0])) assert_(num.allclose(full_send_dict[2],true_full_send[2])) if myid == 2: true_points = [[0.0, 0.0], [0.5, 0.0], [0.5, 0.5], [1.0, 0.0], [1.0, 0.5], [0.25, 0.25], [0.75, 0.25], [0.0, 0.5], [0.5, 1.0], [1.0, 1.0], [0.75, 0.75]] true_vertices = [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [2, 5, 1], [2, 10, 8], [4, 10, 2], [9, 10, 4], [0, 5, 7], [7, 5, 2]] true_ghost_recv = {0: [num.array([5, 6, 7, 8]), num.array([0, 1, 2, 3])], 1: [num.array([ 9, 10]), num.array([5, 6])]} true_full_send = {0: [num.array([0, 1, 2, 3, 4]), num.array([11, 12, 13, 14, 15])], 1: [num.array([0, 1]), num.array([11, 12])]} true_tri_map = num.array([5, 6, 7, 8, -1, 9, 10, -1, -1, -1, -1, 0, 1, 2, 3, 4, -1]) true_node_map = num.array([ 0, 7, -1, 1, 2, 8 , 3, 4, 9, 5, -1, 6, 10]) assert_(num.allclose(ghost_layer_width, 2)) assert_(num.allclose(tri_map, true_tri_map)) assert_(num.allclose(node_map, true_node_map)) assert_(num.allclose(points, true_points)) assert_(num.allclose(vertices, true_vertices)) assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0])) assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1])) assert_(num.allclose(full_send_dict[0],true_full_send[0])) assert_(num.allclose(full_send_dict[1],true_full_send[1]))