def distribute(self, numprocs=1):

        self.numprocs = numprocs
        
        domain = self.domain
        verbose = self.verbose
        debug = self.debug
        parameters = self.parameters

        # FIXME: Dummy assignment (until boundaries are refactored to
        # be independent of domains until they are applied)
        bdmap = {}
        for tag in domain.get_boundary_tags():
            bdmap[tag] = None

        domain.set_boundary(bdmap)


        self.domain_name = domain.get_name()
        self.domain_dir = domain.get_datadir()
        self.domain_store = domain.get_store()
        self.domain_store_centroids = domain.get_store_centroids()
        self.domain_minimum_storable_height = domain.minimum_storable_height
        self.domain_flow_algorithm = domain.get_flow_algorithm()
        self.domain_minimum_allowed_height = domain.get_minimum_allowed_height()
        self.domain_georef = domain.geo_reference
        self.number_of_global_triangles = domain.number_of_triangles
        self.number_of_global_nodes = domain.number_of_nodes
        self.boundary_map = domain.boundary_map


        # Subdivide the mesh
        if verbose: print 'sequential_distribute: Subdivide mesh'

        new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
               s2p_map, p2s_map = \
               pmesh_divide_metis_with_map(domain, numprocs)


        # Build the mesh that should be assigned to each processor,
        # this includes ghost nodes and the communication pattern
        if verbose: print 'sequential_distribute: Build submeshes'
        if verbose: print 'sequential_distribute: parameters = ',parameters

        submesh = build_submesh(new_nodes, new_triangles, new_boundary, \
                                quantities, triangles_per_proc, parameters=parameters)

        if verbose:
            for p in range(numprocs):
                N = len(submesh['ghost_nodes'][p])
                M = len(submesh['ghost_triangles'][p])
                print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
                      %(N, M, p)


        self.submesh = submesh
        self.triangles_per_proc = triangles_per_proc
        self.p2s_map =  p2s_map
    def distribute(self, numprocs=1):

        self.numprocs = numprocs

        domain = self.domain
        verbose = self.verbose
        debug = self.debug
        parameters = self.parameters

        # FIXME: Dummy assignment (until boundaries are refactored to
        # be independent of domains until they are applied)
        bdmap = {}
        for tag in domain.get_boundary_tags():
            bdmap[tag] = None

        domain.set_boundary(bdmap)

        self.domain_name = domain.get_name()
        self.domain_dir = domain.get_datadir()
        self.domain_store = domain.get_store()
        self.domain_store_centroids = domain.get_store_centroids()
        self.domain_minimum_storable_height = domain.minimum_storable_height
        self.domain_flow_algorithm = domain.get_flow_algorithm()
        self.domain_minimum_allowed_height = domain.get_minimum_allowed_height(
        )
        self.domain_georef = domain.geo_reference
        self.domain_quantities_to_be_stored = domain.quantities_to_be_stored
        self.domain_smooth = domain.smooth
        self.domain_low_froude = domain.low_froude
        self.number_of_global_triangles = domain.number_of_triangles
        self.number_of_global_nodes = domain.number_of_nodes
        self.boundary_map = domain.boundary_map

        # Subdivide the mesh
        if verbose: print 'sequential_distribute: Subdivide mesh'

        new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
               s2p_map, p2s_map = \
               pmesh_divide_metis_with_map(domain, numprocs)

        # Build the mesh that should be assigned to each processor,
        # this includes ghost nodes and the communication pattern
        if verbose: print 'sequential_distribute: Build submeshes'
        if verbose: print 'sequential_distribute: parameters = ', parameters

        submesh = build_submesh(new_nodes, new_triangles, new_boundary, \
                                quantities, triangles_per_proc, parameters=parameters)

        if verbose:
            for p in range(numprocs):
                N = len(submesh['ghost_nodes'][p])
                M = len(submesh['ghost_triangles'][p])
                print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
                      %(N, M, p)

        self.submesh = submesh
        self.triangles_per_proc = triangles_per_proc
        self.p2s_map = p2s_map
def distribute_mesh(domain, verbose=False, debug=False, parameters=None):
    """ Distribute andsend the mesh info to all the processors.
    Should only be run from processor 0 and will send info to the other
    processors.
    There should be a corresponding  rec_submesh called from all the other
    processors
    """

    if debug:
        verbose = True

    numprocs = size()

    # Subdivide the mesh
    if verbose: print 'Subdivide mesh'
    new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
           s2p_map, p2s_map = \
           pmesh_divide_metis_with_map(domain, numprocs)

    #PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
    #      sp2_map (maps parallel domain triangles to domain triangles)

    # Build the mesh that should be assigned to each processor,
    # this includes ghost nodes and the communication pattern
    if verbose: print 'Build submeshes'
    submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities,
                            triangles_per_proc, parameters)

    if verbose:
        for p in range(numprocs):
            N = len(submesh['ghost_nodes'][p])
            M = len(submesh['ghost_triangles'][p])
            print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
                  %(N, M, p)

    #if debug:
    #    from pprint import pprint
    #    pprint(submesh)

    # Send the mesh partition to the appropriate processor
    if verbose: print 'Distribute submeshes'
    for p in range(1, numprocs):
        send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose)

    # Build the local mesh for processor 0
    points, vertices, boundary, quantities, \
            ghost_recv_dict, full_send_dict, \
            tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
              extract_submesh(submesh, triangles_per_proc, p2s_map, 0)

    # Keep track of the number full nodes and triangles.
    # This is useful later if one needs access to a ghost-free domain
    # Here, we do it for process 0. The others are done in rec_submesh.
    number_of_full_nodes = len(submesh['full_nodes'][0])
    number_of_full_triangles = len(submesh['full_triangles'][0])

    #print
    #for p in range(numprocs):
    #    print 'Process %d:' %(p)
    #
    #    print 'full_triangles:'
    #    print submesh['full_triangles'][p]
    #
    #    print 'full_nodes:'
    #    print submesh['full_nodes'][p]
    #
    #    print 'ghost_triangles:'
    #    print submesh['ghost_triangles'][p]#
    #
    #    print 'ghost_nodes:'
    #   print submesh['ghost_nodes'][p]
    #    print
    #
    #print 'Receive dict'
    #print ghost_recv_dict
    #
    #print 'Send dict'
    #print full_send_dict

    # Return structures necessary for building the parallel domain
    return points, vertices, boundary, quantities,\
           ghost_recv_dict, full_send_dict,\
           number_of_full_nodes, number_of_full_triangles, \
           s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
           ghost_layer_width
Exemple #4
0
def distribute_mesh(domain, verbose=False, debug=False, parameters=None):
    """ Distribute andsend the mesh info to all the processors.
    Should only be run from processor 0 and will send info to the other
    processors.
    There should be a corresponding  rec_submesh called from all the other
    processors
    """

    if debug:
        verbose = True

    numprocs = size()

    
    # Subdivide the mesh
    if verbose: print 'Subdivide mesh'
    new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
           s2p_map, p2s_map = \
           pmesh_divide_metis_with_map(domain, numprocs)

    #PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
    #      sp2_map (maps parallel domain triangles to domain triangles)



    # Build the mesh that should be assigned to each processor,
    # this includes ghost nodes and the communication pattern
    if verbose: print 'Build submeshes'    
    submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities, triangles_per_proc, parameters)

    if verbose:
        for p in range(numprocs):
            N = len(submesh['ghost_nodes'][p])                
            M = len(submesh['ghost_triangles'][p])
            print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
                  %(N, M, p)

    #if debug:
    #    from pprint import pprint
    #    pprint(submesh)


    # Send the mesh partition to the appropriate processor
    if verbose: print 'Distribute submeshes'        
    for p in range(1, numprocs):
        send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose)

    # Build the local mesh for processor 0
    points, vertices, boundary, quantities, \
            ghost_recv_dict, full_send_dict, \
            tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
              extract_submesh(submesh, triangles_per_proc, p2s_map, 0)


              
    # Keep track of the number full nodes and triangles.
    # This is useful later if one needs access to a ghost-free domain
    # Here, we do it for process 0. The others are done in rec_submesh.
    number_of_full_nodes = len(submesh['full_nodes'][0])
    number_of_full_triangles = len(submesh['full_triangles'][0])
        
    #print
    #for p in range(numprocs):
    #    print 'Process %d:' %(p)
    #
    #    print 'full_triangles:'
    #    print submesh['full_triangles'][p]
    #
    #    print 'full_nodes:'
    #    print submesh['full_nodes'][p]
    #
    #    print 'ghost_triangles:'
    #    print submesh['ghost_triangles'][p]#
    #
    #    print 'ghost_nodes:'
    #   print submesh['ghost_nodes'][p]                                
    #    print
    #
    #print 'Receive dict'
    #print ghost_recv_dict
    #
    #print 'Send dict'
    #print full_send_dict        


    # Return structures necessary for building the parallel domain
    return points, vertices, boundary, quantities,\
           ghost_recv_dict, full_send_dict,\
           number_of_full_nodes, number_of_full_triangles, \
           s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
           ghost_layer_width
def distibute_three_processors():
    """
    Do a parallel test of distributing a rectangle onto 3 processors

    """

    # FIXME: Need to update expected values on macos
    if sys.platform == 'darwin':
        return

    # FIXME: Need to update expected values on macos
    #if sys.platform == 'win32':
    #	return

    from anuga.utilities import parallel_abstraction as pypar

    myid = pypar.rank()
    numprocs = pypar.size()

    if not numprocs == 3:
        return

    try:
        import pymetis
        metis_version = 5
    except:
        metis_version = 4

    #print numprocs

    #barrier()

    if myid == 0:

        nodes_0, triangles_0, boundary_0 = rectangular_cross(2, 2)

        domain = Domain(nodes_0, triangles_0, boundary_0)

        domain.set_quantity('elevation',
                            topography)  # Use function for elevation
        domain.set_quantity('friction', 0.0)  # Constant friction
        domain.set_quantity('stage',
                            expression='elevation')  # Dry initial stage
        domain.set_quantity('xmomentum', expression='friction + 2.0')
        domain.set_quantity('ymomentum', ycoord)

        #----------------------------------------------------------------------------------
        # Test pmesh_divide_metis
        #----------------------------------------------------------------------------------
        vertices, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(
            domain, numprocs)

        if False:
            print_seq_values(vertices, triangles, triangles_per_proc)

        true_seq_values = get_true_seq_values(metis_version=metis_version)

        if False:
            print("True Seq Values = \\")
            pprint(true_seq_values)

        assert_allclose(vertices, true_seq_values['vertices'])
        assert_allclose(triangles, true_seq_values['triangles'])
        assert_allclose(triangles_per_proc,
                        true_seq_values['triangles_per_proc'])

        #----------------------------------------------------------------------------------
        # Test build_submesh
        #----------------------------------------------------------------------------------
        submesh = build_submesh(vertices, triangles, boundary, quantities,
                                triangles_per_proc)

        if False:
            print('submesh_values = \\')
            print_submesh_values(submesh)

        true_values = get_true_submesh_values(metis_version)

        assert_allclose(submesh['full_nodes'][0], true_values['full_nodes_0'])
        assert_allclose(submesh['full_nodes'][1], true_values['full_nodes_1'])
        assert_allclose(submesh['full_nodes'][2], true_values['full_nodes_2'])

        assert_allclose(submesh['ghost_nodes'][0],
                        true_values['ghost_nodes_0'])
        assert_allclose(submesh['ghost_nodes'][1],
                        true_values['ghost_nodes_1'])
        assert_allclose(submesh['ghost_nodes'][2],
                        true_values['ghost_nodes_2'])

        assert_allclose(submesh['full_triangles'][0],
                        true_values['full_triangles_0'])
        assert_allclose(submesh['full_triangles'][1],
                        true_values['full_triangles_1'])
        assert_allclose(submesh['full_triangles'][2],
                        true_values['full_triangles_2'])

        assert_allclose(submesh['ghost_triangles'][0],
                        true_values['ghost_triangles_0'])
        assert_allclose(submesh['ghost_triangles'][1],
                        true_values['ghost_triangles_1'])
        assert_allclose(submesh['ghost_triangles'][2],
                        true_values['ghost_triangles_2'])

        assert_allclose(submesh['ghost_commun'][0],
                        true_values['ghost_commun_0'])
        assert_allclose(submesh['ghost_commun'][1],
                        true_values['ghost_commun_1'])
        assert_allclose(submesh['ghost_commun'][2],
                        true_values['ghost_commun_2'])

        assert_(submesh['full_commun'] == true_values['full_commun'])

    barrier()
    #--------------------------------
    # Now do the comunnication part
    #--------------------------------

    if myid == 0:

        points, vertices, boundary, quantities, \
                    ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
                    ghost_layer_width =\
                    extract_submesh(submesh, triangles_per_proc)

        #----------------------------------------------------------------------------------
        # Test send_submesh
        #----------------------------------------------------------------------------------
        for p in range(1, numprocs):
            send_submesh(submesh, triangles_per_proc, p, verbose=False)
    else:
        #----------------------------------------------------------------------------------
        # Test rec_submesh
        #----------------------------------------------------------------------------------
        points, triangles, boundary, quantities, \
                ghost_recv_dict, full_send_dict, \
                no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width = \
                rec_submesh(0, verbose=False)

    barrier()

    #--------------------------------
    # Now do the test
    #--------------------------------
    if myid == 0:

        if False:
            print('extract_values = \\')
            print_extract_submesh(points, triangles, ghost_recv_dict, \
                                  full_send_dict, tri_map, node_map, ghost_layer_width)

        true_values = get_true_extract_submesh(metis_version)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
        assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
        assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
        assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    if myid == 1:

        if False:
            print("rec_submesh_1 = \\")
            print_rec_submesh_1(points, triangles, ghost_recv_dict, full_send_dict, \
                         tri_map, node_map, ghost_layer_width)

        true_values = get_true_rec_submesh_1(metis_version)

        if False:
            print('true_rec_values_1 = \\')
            pprint(true_values)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
        assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
        assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
        assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    if myid == 2:

        if False:
            print("rec_submesh_2 = \\")
            print_rec_submesh_2(points, triangles, ghost_recv_dict, full_send_dict, \
                         tri_map, node_map, ghost_layer_width)

        true_values = get_true_rec_submesh_2(metis_version)

        if False:
            print('true_rec_values_2 = \\')
            pprint(true_values)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
        assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
        assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
        assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    finalize()
Exemple #6
0
def distibute_three_processors():
    """
    Do a parallel test of distributing a rectangle onto 3 processors
   
    """
    
	# FIXME: Need to update expected values on macos
    if sys.platform == 'darwin':
    	return

    import pypar

    myid = pypar.rank()
    numprocs = pypar.size()

    if not numprocs == 3: return

    #print numprocs

    barrier()

    if myid == 0:

        points, vertices, boundary = rectangular_cross(2,2)

        domain = Domain(points, vertices, boundary)


        domain.set_quantity('elevation', topography) # Use function for elevation
        domain.set_quantity('friction', 0.0)         # Constant friction 
        domain.set_quantity('stage', expression='elevation') # Dry initial stage
        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
        domain.set_quantity('ymomentum', ycoord) #

        #----------------------------------------------------------------------------------
        # Test pmesh_divide_metis
        #----------------------------------------------------------------------------------
        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,numprocs)

        assert_(num.allclose(nodes,points))

        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]]

        true_triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]

        assert_(num.allclose(vertices,true_vertices))
        assert_(num.allclose(triangles,true_triangles))

        assert_(num.allclose(triangles_per_proc,[5,6,5]))


        #----------------------------------------------------------------------------------
        # Test build_submesh
        #----------------------------------------------------------------------------------
        submesh = build_submesh(nodes, triangles, boundary, quantities, triangles_per_proc)


        assert_(num.allclose(submesh['full_nodes'][0],[[3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [9.0, 0.25, 0.25], [12.0, 0.75, 0.75]]))
        assert_(num.allclose(submesh['full_nodes'][1],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [9.0, 0.25, 0.25], [10.0, 0.25, 0.75]]))
        assert_(num.allclose(submesh['full_nodes'][2],[[0.0, 0.0, 0.0], [3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [6.0, 1.0, 0.0], [7.0, 1.0, 0.5], [9.0, 0.25, 0.25], [11.0, 0.75, 0.25]]))


        assert_(num.allclose(submesh['ghost_nodes'][0],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [6.0, 1.0, 0.0], [10.0, 0.25, 0.75], [11.0, 0.75, 0.25]]))
        assert_(num.allclose(submesh['ghost_nodes'][1],[[3.0, 0.5, 0.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [11.0, 0.75, 0.25], [12.0, 0.75, 0.75]]))
        assert_(num.allclose(submesh['ghost_nodes'][2],[[1.0, 0.0, 0.5], [5.0, 0.5, 1.0], [8.0, 1.0, 1.0], [12.0, 0.75, 0.75]]))



        true_full_triangles = [num.array([[ 4,  9,  3],
                                          [ 4, 12,  5],
                                          [ 7, 12,  4],
                                          [ 8, 12,  7],
                                          [ 5, 12,  8]]),
                               num.array([[ 0,  9,  1],
                                          [ 1,  9,  4],
                                          [ 1, 10,  2],
                                          [ 4, 10,  1],
                                          [ 5, 10,  4],
                                          [ 2, 10,  5]]),
                               num.array([[ 3,  9,  0],
                                          [ 3, 11,  4],
                                          [ 6, 11,  3],
                                          [ 7, 11,  6],
                                          [ 4, 11,  7]])]


        assert_(num.allclose(submesh['full_triangles'][0],true_full_triangles[0]))
        assert_(num.allclose(submesh['full_triangles'][1],true_full_triangles[1]))
        assert_(num.allclose(submesh['full_triangles'][2],true_full_triangles[2]))

        true_ghost_triangles = [num.array([[ 5,  0,  9,  1],
                                           [ 6,  1,  9,  4],
                                           [ 8,  4, 10,  1],
                                           [ 9,  5, 10,  4],
                                           [10,  2, 10,  5],
                                           [11,  3,  9,  0],
                                           [12,  3, 11,  4],
                                           [13,  6, 11,  3],
                                           [14,  7, 11,  6],
                                           [15,  4, 11,  7]]),
                                num.array([[ 0,  4,  9,  3],
                                           [ 1,  4, 12,  5],
                                           [ 2,  7, 12,  4],
                                           [ 4,  5, 12,  8],
                                           [11,  3,  9,  0],
                                           [12,  3, 11,  4]]),
                                num.array([[ 0,  4,  9,  3],
                                           [ 1,  4, 12,  5],
                                           [ 2,  7, 12,  4],
                                           [ 3,  8, 12,  7],
                                           [ 5,  0,  9,  1],
                                           [ 6,  1,  9,  4]])]



        assert_(num.allclose(submesh['ghost_triangles'][0],true_ghost_triangles[0]))
        assert_(num.allclose(submesh['ghost_triangles'][1],true_ghost_triangles[1]))
        assert_(num.allclose(submesh['ghost_triangles'][2],true_ghost_triangles[2]))

        true_full_commun = [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}]

        assert_(true_full_commun == submesh['full_commun'])


        true_ghost_commun = [num.array([[ 5,  1],
                                        [ 6,  1],
                                        [ 8,  1],
                                        [ 9,  1],
                                        [10,  1],
                                        [11,  2],
                                        [12,  2],
                                        [13,  2],
                                        [14,  2],
                                        [15,  2]]),
                             num.array([[ 0,  0],
                                        [ 1,  0],
                                        [ 2,  0],
                                        [ 4,  0],
                                        [11,  2],
                                        [12,  2]]),
                             num.array([[0, 0],
                                        [1, 0],
                                        [2, 0],
                                        [3, 0],
                                        [5, 1],
                                        [6, 1]])]

        assert_(num.allclose(submesh['ghost_commun'][0],true_ghost_commun[0]))
        assert_(num.allclose(submesh['ghost_commun'][1],true_ghost_commun[1]))
        assert_(num.allclose(submesh['ghost_commun'][2],true_ghost_commun[2]))



    barrier()
    #--------------------------------
    # Now do the comunnication part
    #--------------------------------


    if myid == 0:
        #----------------------------------------------------------------------------------
        # Test send_submesh
        #----------------------------------------------------------------------------------
        for p in range(1, numprocs):
            send_submesh(submesh, triangles_per_proc, p, verbose=False)

        #----------------------------------------------------------------------------------
        # Test extract_submesh
        #----------------------------------------------------------------------------------
        points, vertices, boundary, quantities, \
                ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width =\
        extract_submesh(submesh, triangles_per_proc)


        true_points =  [[0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.75, 0.75], [0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [1.0, 0.0], [0.25, 0.75], [0.75, 0.25]]

        true_vertices = [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [7, 5, 8], [8, 5, 1], [1, 11, 8], [2, 11, 1], [9, 11, 2], [0, 5, 7], [0, 12, 1], [10, 12, 0], [3, 12, 10], [1, 12, 3]]


        true_ghost_recv = {1: [num.array([5, 6, 7, 8, 9]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([10, 11, 12, 13, 14]), num.array([11, 12, 13, 14, 15])]}


        true_full_send = {1: [num.array([0, 1, 2, 4]), num.array([0, 1, 2, 4])], 2: [num.array([0, 1, 2, 3]), num.array([0, 1, 2, 3])]}

        assert_(num.allclose(ghost_layer_width,  2))
        assert_(num.allclose(points,   true_points))
        assert_(num.allclose(vertices, true_vertices))
        assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
        assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
        assert_(num.allclose(full_send_dict[1],true_full_send[1]))
        assert_(num.allclose(full_send_dict[2],true_full_send[2]))

        #print triangles_per_proc

    else:
        #----------------------------------------------------------------------------------
        # Test rec_submesh
        #----------------------------------------------------------------------------------
        points, vertices, boundary, quantities, \
                ghost_recv_dict, full_send_dict, \
                no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width  = \
                rec_submesh(0, verbose=False)    

        if myid == 1:


            true_points =  [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.5], [0.5, 1.0], [0.25, 0.25], [0.25, 0.75], [0.5, 0.0], [1.0, 0.5], [1.0, 1.0], [0.75, 0.25], [0.75, 0.75]] 

            true_vertices =  [[0, 5, 1], [1, 5, 3], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [3, 5, 7], [3, 11, 4], [8, 11, 3], [4, 11, 9], [7, 5, 0], [7, 10, 3]]

            true_ghost_recv =  {0: [num.array([6, 7, 8, 9]), num.array([0, 1, 2, 4])], 2: [num.array([10, 11]), num.array([11, 12])]}

            true_full_send =  {0: [num.array([0, 1, 3, 4, 5]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([0, 1]), num.array([5, 6])]}

            true_tri_map  = num.array([ 6,  7,  8, -1,  9,  0, 1,  2,  3,  4,  5, 10, 11])

            true_node_map = num.array([ 0,  1,  2,  7,  3,  4, -1,  8,  9,  5,  6, 10, 11])

            assert_(num.allclose(ghost_layer_width,  2))
            assert_(num.allclose(tri_map,   true_tri_map))
            assert_(num.allclose(node_map,   true_node_map))
            assert_(num.allclose(points,   true_points))
            assert_(num.allclose(vertices, true_vertices))
            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
            assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
            assert_(num.allclose(full_send_dict[2],true_full_send[2])) 


        if myid == 2:

            true_points =   [[0.0, 0.0], [0.5, 0.0], [0.5, 0.5], [1.0, 0.0], [1.0, 0.5], [0.25, 0.25], [0.75, 0.25], [0.0, 0.5], [0.5, 1.0], [1.0, 1.0], [0.75, 0.75]]

            true_vertices =  [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [2, 5, 1], [2, 10, 8], [4, 10, 2], [9, 10, 4], [0, 5, 7], [7, 5, 2]]

            true_ghost_recv =   {0: [num.array([5, 6, 7, 8]), num.array([0, 1, 2, 3])], 1: [num.array([ 9, 10]), num.array([5, 6])]}

            true_full_send =   {0: [num.array([0, 1, 2, 3, 4]), num.array([11, 12, 13, 14, 15])], 1: [num.array([0, 1]), num.array([11, 12])]}

            true_tri_map  = num.array([5, 6, 7, 8, -1, 9, 10, -1, -1, -1, -1, 0, 1, 2, 3, 4, -1])

            true_node_map = num.array([ 0,  7, -1,  1,  2,  8 , 3,  4,  9,  5, -1,  6, 10])

            assert_(num.allclose(ghost_layer_width,  2))
            assert_(num.allclose(tri_map,   true_tri_map))
            assert_(num.allclose(node_map,   true_node_map))
            assert_(num.allclose(points,   true_points))
            assert_(num.allclose(vertices, true_vertices))
            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
            assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
            assert_(num.allclose(full_send_dict[1],true_full_send[1]))