Beispiel #1
0
def old_distribute(domain, verbose=False, debug=False, parameters=None):
    """ Distribute the domain to all processes

    parameters values  
    """

    if debug:
        verbose = True

    barrier()

    # FIXME: Dummy assignment (until boundaries are refactored to
    # be independent of domains until they are applied)
    if myid == 0:
        bdmap = {}
        for tag in domain.get_boundary_tags():
            bdmap[tag] = None

        domain.set_boundary(bdmap)

    if not pypar_available or numprocs == 1: return domain  # Bypass

    # For some obscure reason this communication must happen prior to
    # the more complex mesh distribution - Oh Well!
    if myid == 0:
        domain_name = domain.get_name()
        domain_dir = domain.get_datadir()
        domain_store = domain.get_store()
        domain_store_centroids = domain.get_store_centroids()
        domain_smooth = domain.smooth
        domain_reduction = domain.reduction
        domain_minimum_storable_height = domain.minimum_storable_height
        domain_flow_algorithm = domain.get_flow_algorithm()
        domain_minimum_allowed_height = domain.get_minimum_allowed_height()
        georef = domain.geo_reference
        number_of_global_triangles = domain.number_of_triangles
        number_of_global_nodes = domain.number_of_nodes

        # FIXME - what other attributes need to be transferred?

        for p in xrange(1, numprocs):
            # FIXME SR: Creates cPickle dump
            send((domain_name, domain_dir, domain_store, \
                  domain_store_centroids, domain_smooth, domain_reduction, \
                  domain_minimum_storable_height, domain_flow_algorithm, \
                  domain_minimum_allowed_height, georef, \
                  number_of_global_triangles, number_of_global_nodes), p)
    else:
        if verbose: print 'P%d: Receiving domain attributes' % (myid)

        domain_name, domain_dir, domain_store, \
                  domain_store_centroids, domain_smooth, domain_reduction, \
                  domain_minimum_storable_height, domain_flow_algorithm, \
                  domain_minimum_allowed_height, georef, \
                  number_of_global_triangles, \
                  number_of_global_nodes = receive(0)

    # Distribute boundary conditions
    # FIXME: This cannot handle e.g. Time_boundaries due to
    # difficulties pickling functions
    if myid == 0:
        boundary_map = domain.boundary_map
        for p in xrange(1, numprocs):
            # FIXME SR: Creates cPickle dump
            send(boundary_map, p)
    else:
        if verbose: print 'P%d: Receiving boundary map' % (myid)

        boundary_map = receive(0)

    send_s2p = False

    if myid == 0:
        # Partition and distribute mesh.
        # Structures returned is in the
        # correct form for the ANUGA data structure


        points, vertices, boundary, quantities,\
                ghost_recv_dict, full_send_dict,\
                number_of_full_nodes, number_of_full_triangles,\
                s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width =\
                distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters)

        # Extract l2g maps
        #tri_l2g  = extract_l2g_map(tri_map)
        #node_l2g = extract_l2g_map(node_map)
        #tri_l2g = p2s_map[tri_l2g]

        if debug:
            print 'P%d' % myid
            print 'tri_map ', tri_map
            print 'node_map', node_map
            print 'tri_l2g', tri_l2g
            print 'node_l2g', node_l2g
            print 's2p_map', s2p_map
            print 'p2s_map', p2s_map

        def protocol(x):
            vanilla = False
            import pypar
            control_info, x = pypar.create_control_info(x,
                                                        vanilla,
                                                        return_object=True)
            print 'protocol', control_info[0]

        # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs

        if send_s2p:
            n = len(s2p_map)
            s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(), num.int),
                                            (n, 1))
            s2p_map_values_flat = num.array(s2p_map.values(), num.int)
            s2p_map_flat = num.concatenate(
                (s2p_map_keys_flat, s2p_map_values_flat), axis=1)

            n = len(p2s_map)
            p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(), num.int),
                                            (n, 2))
            p2s_map_values_flat = num.reshape(
                num.array(p2s_map.values(), num.int), (n, 1))
            p2s_map_flat = num.concatenate(
                (p2s_map_keys_flat, p2s_map_values_flat), axis=1)

            for p in range(1, numprocs):

                # FIXME SR: Creates cPickle dump
                send(s2p_map_flat, p)
                # FIXME SR: Creates cPickle dump
                #print p2s_map
                send(p2s_map_flat, p)
        else:
            if verbose: print 'Not sending s2p_map and p2s_map'
            s2p_map = None
            p2s_map = None

        if verbose: print 'Communication done'

    else:
        # Read in the mesh partition that belongs to this
        # processor
        if verbose: print 'P%d: Receiving submeshes' % (myid)
        points, vertices, boundary, quantities,\
                ghost_recv_dict, full_send_dict,\
                number_of_full_nodes, number_of_full_triangles, \
                tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
                rec_submesh(0, verbose)

        # Extract l2g maps
        #tri_l2g  = extract_l2g_map(tri_map)
        #node_l2g = extract_l2g_map(node_map)
        #tri_l2g = p2s_map[tri_l2g]

        # Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping
        if send_s2p:
            s2p_map_flat = receive(0)
            s2p_map = dict.fromkeys(s2p_map_flat[:, 0], s2p_map_flat[:, 1:2])

            p2s_map_flat = receive(0)
            p2s_map_keys = [tuple(x) for x in p2s_map_flat[:, 0:1]]

            p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:, 2])
        else:
            s2p_map = None
            p2s_map = None

    #------------------------------------------------------------------------
    # Build the domain for this processor using partion structures
    #------------------------------------------------------------------------

    if verbose:
        print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (
            myid, number_of_full_nodes, number_of_full_triangles)

    domain = Parallel_domain(
        points,
        vertices,
        boundary,
        full_send_dict=full_send_dict,
        ghost_recv_dict=ghost_recv_dict,
        number_of_full_nodes=number_of_full_nodes,
        number_of_full_triangles=number_of_full_triangles,
        geo_reference=georef,
        number_of_global_triangles=number_of_global_triangles,
        number_of_global_nodes=number_of_global_nodes,
        s2p_map=s2p_map,
        p2s_map=p2s_map,  ## jj added this
        tri_l2g=tri_l2g,  ## SR added this
        node_l2g=node_l2g,
        ghost_layer_width=ghost_layer_width)

    #------------------------------------------------------------------------
    # Transfer initial conditions to each subdomain
    #------------------------------------------------------------------------
    for q in quantities:
        domain.set_quantity(q, quantities[q])

    #------------------------------------------------------------------------
    # Transfer boundary conditions to each subdomain
    #------------------------------------------------------------------------
    boundary_map['ghost'] = None  # Add binding to ghost boundary
    domain.set_boundary(boundary_map)

    #------------------------------------------------------------------------
    # Transfer other attributes to each subdomain
    #------------------------------------------------------------------------
    domain.set_name(domain_name)
    domain.set_datadir(domain_dir)
    domain.set_store(domain_store)
    domain.set_store_centroids(domain_store_centroids)
    domain.set_store_vertices_smoothly(domain_smooth, domain_reduction)
    domain.set_minimum_storable_height(domain_minimum_storable_height)
    domain.set_minimum_allowed_height(domain_minimum_allowed_height)
    domain.set_flow_algorithm(domain_flow_algorithm)
    domain.geo_reference = georef

    #------------------------------------------------------------------------
    # Return parallel domain to all nodes
    #------------------------------------------------------------------------
    return domain
def distibute_three_processors():
    """
    Do a parallel test of distributing a rectangle onto 3 processors

    """

    # FIXME: Need to update expected values on macos
    if sys.platform == 'darwin':
        return

    # FIXME: Need to update expected values on macos
    #if sys.platform == 'win32':
    #	return

    from anuga.utilities import parallel_abstraction as pypar

    myid = pypar.rank()
    numprocs = pypar.size()

    if not numprocs == 3:
        return

    try:
        import pymetis
        metis_version = 5
    except:
        metis_version = 4

    #print numprocs

    #barrier()

    if myid == 0:

        nodes_0, triangles_0, boundary_0 = rectangular_cross(2, 2)

        domain = Domain(nodes_0, triangles_0, boundary_0)

        domain.set_quantity('elevation',
                            topography)  # Use function for elevation
        domain.set_quantity('friction', 0.0)  # Constant friction
        domain.set_quantity('stage',
                            expression='elevation')  # Dry initial stage
        domain.set_quantity('xmomentum', expression='friction + 2.0')
        domain.set_quantity('ymomentum', ycoord)

        #----------------------------------------------------------------------------------
        # Test pmesh_divide_metis
        #----------------------------------------------------------------------------------
        vertices, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(
            domain, numprocs)

        if False:
            print_seq_values(vertices, triangles, triangles_per_proc)

        true_seq_values = get_true_seq_values(metis_version=metis_version)

        if False:
            print("True Seq Values = \\")
            pprint(true_seq_values)

        assert_allclose(vertices, true_seq_values['vertices'])
        assert_allclose(triangles, true_seq_values['triangles'])
        assert_allclose(triangles_per_proc,
                        true_seq_values['triangles_per_proc'])

        #----------------------------------------------------------------------------------
        # Test build_submesh
        #----------------------------------------------------------------------------------
        submesh = build_submesh(vertices, triangles, boundary, quantities,
                                triangles_per_proc)

        if False:
            print('submesh_values = \\')
            print_submesh_values(submesh)

        true_values = get_true_submesh_values(metis_version)

        assert_allclose(submesh['full_nodes'][0], true_values['full_nodes_0'])
        assert_allclose(submesh['full_nodes'][1], true_values['full_nodes_1'])
        assert_allclose(submesh['full_nodes'][2], true_values['full_nodes_2'])

        assert_allclose(submesh['ghost_nodes'][0],
                        true_values['ghost_nodes_0'])
        assert_allclose(submesh['ghost_nodes'][1],
                        true_values['ghost_nodes_1'])
        assert_allclose(submesh['ghost_nodes'][2],
                        true_values['ghost_nodes_2'])

        assert_allclose(submesh['full_triangles'][0],
                        true_values['full_triangles_0'])
        assert_allclose(submesh['full_triangles'][1],
                        true_values['full_triangles_1'])
        assert_allclose(submesh['full_triangles'][2],
                        true_values['full_triangles_2'])

        assert_allclose(submesh['ghost_triangles'][0],
                        true_values['ghost_triangles_0'])
        assert_allclose(submesh['ghost_triangles'][1],
                        true_values['ghost_triangles_1'])
        assert_allclose(submesh['ghost_triangles'][2],
                        true_values['ghost_triangles_2'])

        assert_allclose(submesh['ghost_commun'][0],
                        true_values['ghost_commun_0'])
        assert_allclose(submesh['ghost_commun'][1],
                        true_values['ghost_commun_1'])
        assert_allclose(submesh['ghost_commun'][2],
                        true_values['ghost_commun_2'])

        assert_(submesh['full_commun'] == true_values['full_commun'])

    barrier()
    #--------------------------------
    # Now do the comunnication part
    #--------------------------------

    if myid == 0:

        points, vertices, boundary, quantities, \
                    ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
                    ghost_layer_width =\
                    extract_submesh(submesh, triangles_per_proc)

        #----------------------------------------------------------------------------------
        # Test send_submesh
        #----------------------------------------------------------------------------------
        for p in range(1, numprocs):
            send_submesh(submesh, triangles_per_proc, p, verbose=False)
    else:
        #----------------------------------------------------------------------------------
        # Test rec_submesh
        #----------------------------------------------------------------------------------
        points, triangles, boundary, quantities, \
                ghost_recv_dict, full_send_dict, \
                no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width = \
                rec_submesh(0, verbose=False)

    barrier()

    #--------------------------------
    # Now do the test
    #--------------------------------
    if myid == 0:

        if False:
            print('extract_values = \\')
            print_extract_submesh(points, triangles, ghost_recv_dict, \
                                  full_send_dict, tri_map, node_map, ghost_layer_width)

        true_values = get_true_extract_submesh(metis_version)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
        assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
        assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
        assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    if myid == 1:

        if False:
            print("rec_submesh_1 = \\")
            print_rec_submesh_1(points, triangles, ghost_recv_dict, full_send_dict, \
                         tri_map, node_map, ghost_layer_width)

        true_values = get_true_rec_submesh_1(metis_version)

        if False:
            print('true_rec_values_1 = \\')
            pprint(true_values)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
        assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
        assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
        assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    if myid == 2:

        if False:
            print("rec_submesh_2 = \\")
            print_rec_submesh_2(points, triangles, ghost_recv_dict, full_send_dict, \
                         tri_map, node_map, ghost_layer_width)

        true_values = get_true_rec_submesh_2(metis_version)

        if False:
            print('true_rec_values_2 = \\')
            pprint(true_values)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
        assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
        assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
        assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    finalize()
Beispiel #3
0
def old_distribute(domain, verbose=False, debug=False, parameters = None):
    """ Distribute the domain to all processes

    parameters values  
    """


    if debug:
        verbose = True
        
    barrier()

    # FIXME: Dummy assignment (until boundaries are refactored to
    # be independent of domains until they are applied)
    if myid == 0:
        bdmap = {}
        for tag in domain.get_boundary_tags():
            bdmap[tag] = None
    
    
        domain.set_boundary(bdmap)


    if not pypar_available or numprocs == 1 : return domain # Bypass

    # For some obscure reason this communication must happen prior to
    # the more complex mesh distribution - Oh Well!
    if myid == 0:
        domain_name = domain.get_name()
        domain_dir = domain.get_datadir()
        domain_store = domain.get_store()
        domain_store_centroids = domain.get_store_centroids()
        domain_smooth = domain.smooth
        domain_reduction = domain.reduction
        domain_minimum_storable_height = domain.minimum_storable_height
        domain_flow_algorithm = domain.get_flow_algorithm()
        domain_minimum_allowed_height = domain.get_minimum_allowed_height()
        georef = domain.geo_reference
        number_of_global_triangles = domain.number_of_triangles
        number_of_global_nodes = domain.number_of_nodes
        
        # FIXME - what other attributes need to be transferred?

        for p in xrange(1, numprocs):
            # FIXME SR: Creates cPickle dump
            send((domain_name, domain_dir, domain_store, \
                  domain_store_centroids, domain_smooth, domain_reduction, \
                  domain_minimum_storable_height, domain_flow_algorithm, \
                  domain_minimum_allowed_height, georef, \
                  number_of_global_triangles, number_of_global_nodes), p)
    else:
        if verbose: print 'P%d: Receiving domain attributes' %(myid)

        domain_name, domain_dir, domain_store, \
                  domain_store_centroids, domain_smooth, domain_reduction, \
                  domain_minimum_storable_height, domain_flow_algorithm, \
                  domain_minimum_allowed_height, georef, \
                  number_of_global_triangles, \
                  number_of_global_nodes = receive(0)



    # Distribute boundary conditions
    # FIXME: This cannot handle e.g. Time_boundaries due to
    # difficulties pickling functions
    if myid == 0:
        boundary_map = domain.boundary_map
        for p in xrange(1, numprocs):
            # FIXME SR: Creates cPickle dump
            send(boundary_map, p)
    else:
        if verbose: print 'P%d: Receiving boundary map' %(myid)        

        boundary_map = receive(0)
        

    send_s2p = False

    if myid == 0:
        # Partition and distribute mesh.
        # Structures returned is in the
        # correct form for the ANUGA data structure


        points, vertices, boundary, quantities,\
                ghost_recv_dict, full_send_dict,\
                number_of_full_nodes, number_of_full_triangles,\
                s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width =\
                distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters)
            
        # Extract l2g maps
        #tri_l2g  = extract_l2g_map(tri_map)
        #node_l2g = extract_l2g_map(node_map)
        #tri_l2g = p2s_map[tri_l2g]
        
        if debug:
            print 'P%d' %myid
            print 'tri_map ',tri_map
            print 'node_map',node_map
            print 'tri_l2g', tri_l2g
            print 'node_l2g', node_l2g
            print 's2p_map', s2p_map
            print 'p2s_map', p2s_map


        def protocol(x):
            vanilla=False
            import pypar
            control_info, x = pypar.create_control_info(x, vanilla, return_object=True)
            print 'protocol', control_info[0]
            
        # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs


        if send_s2p :
            n = len(s2p_map)
            s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(),num.int), (n,1) )
            s2p_map_values_flat = num.array(s2p_map.values(),num.int)
            s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1 )

            n = len(p2s_map)
            p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(),num.int), (n,2) )
            p2s_map_values_flat = num.reshape(num.array(p2s_map.values(),num.int) , (n,1))
            p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1 )

            for p in range(1, numprocs):

                # FIXME SR: Creates cPickle dump
                send(s2p_map_flat, p)
                # FIXME SR: Creates cPickle dump
                #print p2s_map
                send(p2s_map_flat, p)
        else:
            if verbose: print 'Not sending s2p_map and p2s_map'
            s2p_map = None
            p2s_map = None

        if verbose: print 'Communication done'
        
    else:
        # Read in the mesh partition that belongs to this
        # processor
        if verbose: print 'P%d: Receiving submeshes' %(myid)                
        points, vertices, boundary, quantities,\
                ghost_recv_dict, full_send_dict,\
                number_of_full_nodes, number_of_full_triangles, \
                tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
                rec_submesh(0, verbose)



        # Extract l2g maps
        #tri_l2g  = extract_l2g_map(tri_map)
        #node_l2g = extract_l2g_map(node_map)
        #tri_l2g = p2s_map[tri_l2g]
        
        # Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping
        if send_s2p :
            s2p_map_flat = receive(0)
            s2p_map = dict.fromkeys(s2p_map_flat[:,0], s2p_map_flat[:,1:2])

            p2s_map_flat = receive(0)
            p2s_map_keys = [tuple(x) for x in p2s_map_flat[:,0:1]]

            p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:,2])
        else:
            s2p_map = None
            p2s_map = None
            
    #------------------------------------------------------------------------
    # Build the domain for this processor using partion structures
    #------------------------------------------------------------------------

    if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (myid, number_of_full_nodes, number_of_full_triangles)

    
    domain = Parallel_domain(points, vertices, boundary,
                             full_send_dict=full_send_dict,
                             ghost_recv_dict=ghost_recv_dict,
                             number_of_full_nodes=number_of_full_nodes,
                             number_of_full_triangles=number_of_full_triangles,
                             geo_reference=georef,
                             number_of_global_triangles = number_of_global_triangles,
                             number_of_global_nodes = number_of_global_nodes,
                             s2p_map = s2p_map,
                             p2s_map = p2s_map, ## jj added this
                             tri_l2g = tri_l2g, ## SR added this
                             node_l2g = node_l2g,
                             ghost_layer_width = ghost_layer_width)

    #------------------------------------------------------------------------
    # Transfer initial conditions to each subdomain
    #------------------------------------------------------------------------
    for q in quantities:
        domain.set_quantity(q, quantities[q]) 


    #------------------------------------------------------------------------
    # Transfer boundary conditions to each subdomain
    #------------------------------------------------------------------------
    boundary_map['ghost'] = None  # Add binding to ghost boundary
    domain.set_boundary(boundary_map)


    #------------------------------------------------------------------------
    # Transfer other attributes to each subdomain
    #------------------------------------------------------------------------
    domain.set_name(domain_name)
    domain.set_datadir(domain_dir)
    domain.set_store(domain_store)
    domain.set_store_centroids(domain_store_centroids)
    domain.set_store_vertices_smoothly(domain_smooth,domain_reduction)
    domain.set_minimum_storable_height(domain_minimum_storable_height)
    domain.set_minimum_allowed_height(domain_minimum_allowed_height)
    domain.set_flow_algorithm(domain_flow_algorithm)
    domain.geo_reference = georef   

    #------------------------------------------------------------------------
    # Return parallel domain to all nodes
    #------------------------------------------------------------------------
    return domain    
Beispiel #4
0
def distibute_three_processors():
    """
    Do a parallel test of distributing a rectangle onto 3 processors
   
    """
    
	# FIXME: Need to update expected values on macos
    if sys.platform == 'darwin':
    	return

    import pypar

    myid = pypar.rank()
    numprocs = pypar.size()

    if not numprocs == 3: return

    #print numprocs

    barrier()

    if myid == 0:

        points, vertices, boundary = rectangular_cross(2,2)

        domain = Domain(points, vertices, boundary)


        domain.set_quantity('elevation', topography) # Use function for elevation
        domain.set_quantity('friction', 0.0)         # Constant friction 
        domain.set_quantity('stage', expression='elevation') # Dry initial stage
        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
        domain.set_quantity('ymomentum', ycoord) #

        #----------------------------------------------------------------------------------
        # Test pmesh_divide_metis
        #----------------------------------------------------------------------------------
        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,numprocs)

        assert_(num.allclose(nodes,points))

        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]]

        true_triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]

        assert_(num.allclose(vertices,true_vertices))
        assert_(num.allclose(triangles,true_triangles))

        assert_(num.allclose(triangles_per_proc,[5,6,5]))


        #----------------------------------------------------------------------------------
        # Test build_submesh
        #----------------------------------------------------------------------------------
        submesh = build_submesh(nodes, triangles, boundary, quantities, triangles_per_proc)


        assert_(num.allclose(submesh['full_nodes'][0],[[3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [9.0, 0.25, 0.25], [12.0, 0.75, 0.75]]))
        assert_(num.allclose(submesh['full_nodes'][1],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [9.0, 0.25, 0.25], [10.0, 0.25, 0.75]]))
        assert_(num.allclose(submesh['full_nodes'][2],[[0.0, 0.0, 0.0], [3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [6.0, 1.0, 0.0], [7.0, 1.0, 0.5], [9.0, 0.25, 0.25], [11.0, 0.75, 0.25]]))


        assert_(num.allclose(submesh['ghost_nodes'][0],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [6.0, 1.0, 0.0], [10.0, 0.25, 0.75], [11.0, 0.75, 0.25]]))
        assert_(num.allclose(submesh['ghost_nodes'][1],[[3.0, 0.5, 0.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [11.0, 0.75, 0.25], [12.0, 0.75, 0.75]]))
        assert_(num.allclose(submesh['ghost_nodes'][2],[[1.0, 0.0, 0.5], [5.0, 0.5, 1.0], [8.0, 1.0, 1.0], [12.0, 0.75, 0.75]]))



        true_full_triangles = [num.array([[ 4,  9,  3],
                                          [ 4, 12,  5],
                                          [ 7, 12,  4],
                                          [ 8, 12,  7],
                                          [ 5, 12,  8]]),
                               num.array([[ 0,  9,  1],
                                          [ 1,  9,  4],
                                          [ 1, 10,  2],
                                          [ 4, 10,  1],
                                          [ 5, 10,  4],
                                          [ 2, 10,  5]]),
                               num.array([[ 3,  9,  0],
                                          [ 3, 11,  4],
                                          [ 6, 11,  3],
                                          [ 7, 11,  6],
                                          [ 4, 11,  7]])]


        assert_(num.allclose(submesh['full_triangles'][0],true_full_triangles[0]))
        assert_(num.allclose(submesh['full_triangles'][1],true_full_triangles[1]))
        assert_(num.allclose(submesh['full_triangles'][2],true_full_triangles[2]))

        true_ghost_triangles = [num.array([[ 5,  0,  9,  1],
                                           [ 6,  1,  9,  4],
                                           [ 8,  4, 10,  1],
                                           [ 9,  5, 10,  4],
                                           [10,  2, 10,  5],
                                           [11,  3,  9,  0],
                                           [12,  3, 11,  4],
                                           [13,  6, 11,  3],
                                           [14,  7, 11,  6],
                                           [15,  4, 11,  7]]),
                                num.array([[ 0,  4,  9,  3],
                                           [ 1,  4, 12,  5],
                                           [ 2,  7, 12,  4],
                                           [ 4,  5, 12,  8],
                                           [11,  3,  9,  0],
                                           [12,  3, 11,  4]]),
                                num.array([[ 0,  4,  9,  3],
                                           [ 1,  4, 12,  5],
                                           [ 2,  7, 12,  4],
                                           [ 3,  8, 12,  7],
                                           [ 5,  0,  9,  1],
                                           [ 6,  1,  9,  4]])]



        assert_(num.allclose(submesh['ghost_triangles'][0],true_ghost_triangles[0]))
        assert_(num.allclose(submesh['ghost_triangles'][1],true_ghost_triangles[1]))
        assert_(num.allclose(submesh['ghost_triangles'][2],true_ghost_triangles[2]))

        true_full_commun = [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}]

        assert_(true_full_commun == submesh['full_commun'])


        true_ghost_commun = [num.array([[ 5,  1],
                                        [ 6,  1],
                                        [ 8,  1],
                                        [ 9,  1],
                                        [10,  1],
                                        [11,  2],
                                        [12,  2],
                                        [13,  2],
                                        [14,  2],
                                        [15,  2]]),
                             num.array([[ 0,  0],
                                        [ 1,  0],
                                        [ 2,  0],
                                        [ 4,  0],
                                        [11,  2],
                                        [12,  2]]),
                             num.array([[0, 0],
                                        [1, 0],
                                        [2, 0],
                                        [3, 0],
                                        [5, 1],
                                        [6, 1]])]

        assert_(num.allclose(submesh['ghost_commun'][0],true_ghost_commun[0]))
        assert_(num.allclose(submesh['ghost_commun'][1],true_ghost_commun[1]))
        assert_(num.allclose(submesh['ghost_commun'][2],true_ghost_commun[2]))



    barrier()
    #--------------------------------
    # Now do the comunnication part
    #--------------------------------


    if myid == 0:
        #----------------------------------------------------------------------------------
        # Test send_submesh
        #----------------------------------------------------------------------------------
        for p in range(1, numprocs):
            send_submesh(submesh, triangles_per_proc, p, verbose=False)

        #----------------------------------------------------------------------------------
        # Test extract_submesh
        #----------------------------------------------------------------------------------
        points, vertices, boundary, quantities, \
                ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width =\
        extract_submesh(submesh, triangles_per_proc)


        true_points =  [[0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.75, 0.75], [0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [1.0, 0.0], [0.25, 0.75], [0.75, 0.25]]

        true_vertices = [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [7, 5, 8], [8, 5, 1], [1, 11, 8], [2, 11, 1], [9, 11, 2], [0, 5, 7], [0, 12, 1], [10, 12, 0], [3, 12, 10], [1, 12, 3]]


        true_ghost_recv = {1: [num.array([5, 6, 7, 8, 9]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([10, 11, 12, 13, 14]), num.array([11, 12, 13, 14, 15])]}


        true_full_send = {1: [num.array([0, 1, 2, 4]), num.array([0, 1, 2, 4])], 2: [num.array([0, 1, 2, 3]), num.array([0, 1, 2, 3])]}

        assert_(num.allclose(ghost_layer_width,  2))
        assert_(num.allclose(points,   true_points))
        assert_(num.allclose(vertices, true_vertices))
        assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
        assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
        assert_(num.allclose(full_send_dict[1],true_full_send[1]))
        assert_(num.allclose(full_send_dict[2],true_full_send[2]))

        #print triangles_per_proc

    else:
        #----------------------------------------------------------------------------------
        # Test rec_submesh
        #----------------------------------------------------------------------------------
        points, vertices, boundary, quantities, \
                ghost_recv_dict, full_send_dict, \
                no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width  = \
                rec_submesh(0, verbose=False)    

        if myid == 1:


            true_points =  [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.5], [0.5, 1.0], [0.25, 0.25], [0.25, 0.75], [0.5, 0.0], [1.0, 0.5], [1.0, 1.0], [0.75, 0.25], [0.75, 0.75]] 

            true_vertices =  [[0, 5, 1], [1, 5, 3], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [3, 5, 7], [3, 11, 4], [8, 11, 3], [4, 11, 9], [7, 5, 0], [7, 10, 3]]

            true_ghost_recv =  {0: [num.array([6, 7, 8, 9]), num.array([0, 1, 2, 4])], 2: [num.array([10, 11]), num.array([11, 12])]}

            true_full_send =  {0: [num.array([0, 1, 3, 4, 5]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([0, 1]), num.array([5, 6])]}

            true_tri_map  = num.array([ 6,  7,  8, -1,  9,  0, 1,  2,  3,  4,  5, 10, 11])

            true_node_map = num.array([ 0,  1,  2,  7,  3,  4, -1,  8,  9,  5,  6, 10, 11])

            assert_(num.allclose(ghost_layer_width,  2))
            assert_(num.allclose(tri_map,   true_tri_map))
            assert_(num.allclose(node_map,   true_node_map))
            assert_(num.allclose(points,   true_points))
            assert_(num.allclose(vertices, true_vertices))
            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
            assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
            assert_(num.allclose(full_send_dict[2],true_full_send[2])) 


        if myid == 2:

            true_points =   [[0.0, 0.0], [0.5, 0.0], [0.5, 0.5], [1.0, 0.0], [1.0, 0.5], [0.25, 0.25], [0.75, 0.25], [0.0, 0.5], [0.5, 1.0], [1.0, 1.0], [0.75, 0.75]]

            true_vertices =  [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [2, 5, 1], [2, 10, 8], [4, 10, 2], [9, 10, 4], [0, 5, 7], [7, 5, 2]]

            true_ghost_recv =   {0: [num.array([5, 6, 7, 8]), num.array([0, 1, 2, 3])], 1: [num.array([ 9, 10]), num.array([5, 6])]}

            true_full_send =   {0: [num.array([0, 1, 2, 3, 4]), num.array([11, 12, 13, 14, 15])], 1: [num.array([0, 1]), num.array([11, 12])]}

            true_tri_map  = num.array([5, 6, 7, 8, -1, 9, 10, -1, -1, -1, -1, 0, 1, 2, 3, 4, -1])

            true_node_map = num.array([ 0,  7, -1,  1,  2,  8 , 3,  4,  9,  5, -1,  6, 10])

            assert_(num.allclose(ghost_layer_width,  2))
            assert_(num.allclose(tri_map,   true_tri_map))
            assert_(num.allclose(node_map,   true_node_map))
            assert_(num.allclose(points,   true_points))
            assert_(num.allclose(vertices, true_vertices))
            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
            assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
            assert_(num.allclose(full_send_dict[1],true_full_send[1]))