def rec_submesh(p, verbose=True):

    from anuga.utilities import parallel_abstraction as pypar

    numproc = pypar.size()
    myid = pypar.rank()

    [
        submesh_cell, triangles_per_proc, number_of_full_nodes,
        number_of_full_triangles
    ] = rec_submesh_flat(p, verbose)

    # find the full triangles assigned to this processor

    lower_t = 0
    for i in range(myid):
        lower_t = lower_t + triangles_per_proc[i]
    upper_t = lower_t + triangles_per_proc[myid]

    # convert the information into a form needed by the GA
    # datastructure

    [GAnodes, GAtriangles, boundary, quantities,
     ghost_rec, full_send,
     tri_map, node_map, tri_l2g, node_l2g,
     ghost_layer_width] = \
        build_local_mesh(submesh_cell, lower_t, upper_t, numproc)

    return GAnodes, GAtriangles, boundary, quantities,\
        ghost_rec, full_send,\
        number_of_full_nodes, number_of_full_triangles, tri_map, node_map,\
        tri_l2g, node_l2g, ghost_layer_width
    def __init__(self,
                 coordinates,
                 vertices,
                 boundary=None,
                 full_send_dict=None,
                 ghost_recv_dict=None,
                 velocity=None):

        Domain.__init__(self,
                        coordinates,
                        vertices,
                        boundary,
                        velocity=velocity,
                        full_send_dict=full_send_dict,
                        ghost_recv_dict=ghost_recv_dict,
                        processor=pypar.rank(),
                        numproc=pypar.size())

        N = self.number_of_elements

        self.communication_time = 0.0
        self.communication_reduce_time = 0.0

        print('processor', self.processor)
        print('numproc', self.numproc)
def print_l2_stats(full_edge):

    numprocs = pypar.size()
    myid = pypar.rank()

    tri_norm = zeros(3, Float)
    recv_norm = zeros(3, Float)
    tri_norm[0] = pow(l2_norm(full_edge[:, 0]), 2)
    tri_norm[1] = pow(l2_norm(full_edge[:, 1]), 2)
    tri_norm[2] = pow(l2_norm(full_edge[:, 2]), 2)
    if myid == 0:
        for p in range(numprocs - 1):
            pypar.receive(p + 1, recv_norm)
            tri_norm[0] = tri_norm[0] + recv_norm[0]
            tri_norm[1] = tri_norm[1] + recv_norm[1]
            tri_norm[2] = tri_norm[2] + recv_norm[2]
        print('l2_norm along each axis : [', pow(tri_norm[0], 0.5),', ', pow(tri_norm[1], 0.5), \
              ', ', pow(tri_norm[2], 0.5), ']')
    else:
        pypar.send(tri_norm, 0)
def print_linf_stats(full_edge):

    numprocs = pypar.size()
    myid = pypar.rank()

    tri_norm = zeros(3, Float)
    recv_norm = zeros(3, Float)

    tri_norm[0] = linf_norm(full_edge[:, 0])
    tri_norm[1] = linf_norm(full_edge[:, 1])
    tri_norm[2] = linf_norm(full_edge[:, 2])
    if myid == 0:
        for p in range(numprocs - 1):
            pypar.receive(p + 1, recv_norm)
            tri_norm[0] = max(tri_norm[0], recv_norm[0])
            tri_norm[1] = max(tri_norm[1], recv_norm[1])
            tri_norm[2] = max(tri_norm[2], recv_norm[2])
        print('linf_norm along each axis : [', tri_norm[0], ', ', tri_norm[1],
              ', ', tri_norm[2], ']')
    else:
        pypar.send(tri_norm, 0)
def send_submesh(submesh, triangles_per_proc, p, verbose=True):

    from anuga.utilities import parallel_abstraction as pypar

    myid = pypar.rank()
    nprocs = pypar.size()

    if verbose:
        print('P%d: Sending submesh to P%d' % (myid, p))

    # build and send the tagmap for the boundary conditions

    tagmap = {}
    counter = 1
    for b in submesh["full_boundary"][p]:
        bkey = submesh["full_boundary"][p][b]
        if bkey not in tagmap:
            tagmap[bkey] = counter
            counter = counter + 1
    for b in submesh["ghost_boundary"][p]:
        bkey = submesh["ghost_boundary"][p][b]
        if bkey not in tagmap:
            tagmap[bkey] = counter
            counter = counter + 1

    # send boundary tags
    pypar.send(tagmap, p)

    # send the quantities key information
    pypar.send(list(submesh["full_quan"].keys()), p)

    # compress full_commun
    flat_full_commun = []

    for c in submesh["full_commun"][p]:
        for i in range(len(submesh["full_commun"][p][c])):
            flat_full_commun.append([c, submesh["full_commun"][p][c][i]])

    # send the array sizes so memory can be allocated

    setup_array = num.zeros((9, ), num.int)
    setup_array[0] = len(submesh["full_nodes"][p])
    setup_array[1] = len(submesh["ghost_nodes"][p])
    setup_array[2] = len(submesh["full_triangles"][p])
    setup_array[3] = len(submesh["ghost_triangles"][p])
    setup_array[4] = len(submesh["full_boundary"][p])
    setup_array[5] = len(submesh["ghost_boundary"][p])
    setup_array[6] = len(submesh["ghost_commun"][p])
    setup_array[7] = len(flat_full_commun)
    setup_array[8] = len(submesh["full_quan"])

    x = num.array(setup_array, num.int)
    pypar.send(x, p, bypass=True)

    # ghost layer width
    x = num.array(submesh["ghost_layer_width"][p], num.int)
    pypar.send(x, p, bypass=True)

    # send the number of triangles per processor
    x = num.array(triangles_per_proc, num.int)
    pypar.send(x, p, bypass=True)

    # send the nodes
    x = num.array(submesh["full_nodes"][p], num.float)
    pypar.send(x, p, bypass=True)

    x = num.array(submesh["ghost_nodes"][p], num.float)
    pypar.send(x, p, bypass=True)

    # send the triangles
    x = num.array(submesh["full_triangles"][p], num.int)
    pypar.send(x, p, bypass=True)

    # send ghost triangles
    x = num.array(submesh["ghost_triangles"][p], num.int)
    pypar.send(x, p, bypass=True)

    # send the boundary
    bc = []
    for b in submesh["full_boundary"][p]:
        bc.append([b[0], b[1], tagmap[submesh["full_boundary"][p][b]]])

    x = num.array(bc, num.int)
    pypar.send(x, p, bypass=True)

    bc = []
    for b in submesh["ghost_boundary"][p]:
        bc.append([b[0], b[1], tagmap[submesh["ghost_boundary"][p][b]]])

    x = num.array(bc, num.int)
    pypar.send(x, p, bypass=True)

    # send the communication pattern
    x = num.array(submesh["ghost_commun"][p], num.int)
    pypar.send(x, p, bypass=True)

    x = num.array(flat_full_commun, num.int)
    pypar.send(x, p, bypass=True)

    # send the quantities
    for k in submesh["full_quan"]:
        x = num.array(submesh["full_quan"][k][p], num.float)
        pypar.send(x, p, bypass=True)

    for k in submesh["ghost_quan"]:
        x = num.array(submesh["ghost_quan"][k][p], num.float)
        pypar.send(x, p, bypass=True)
def distribute_mesh(domain, verbose=False, debug=False, parameters=None):
    """ Distribute andsend the mesh info to all the processors.
    Should only be run from processor 0 and will send info to the other
    processors.
    There should be a corresponding  rec_submesh called from all the other
    processors
    """

    if debug:
        verbose = True

    numprocs = size()

    # Subdivide the mesh
    if verbose: print 'Subdivide mesh'
    new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
           s2p_map, p2s_map = \
           pmesh_divide_metis_with_map(domain, numprocs)

    #PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
    #      sp2_map (maps parallel domain triangles to domain triangles)

    # Build the mesh that should be assigned to each processor,
    # this includes ghost nodes and the communication pattern
    if verbose: print 'Build submeshes'
    submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities,
                            triangles_per_proc, parameters)

    if verbose:
        for p in range(numprocs):
            N = len(submesh['ghost_nodes'][p])
            M = len(submesh['ghost_triangles'][p])
            print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
                  %(N, M, p)

    #if debug:
    #    from pprint import pprint
    #    pprint(submesh)

    # Send the mesh partition to the appropriate processor
    if verbose: print 'Distribute submeshes'
    for p in range(1, numprocs):
        send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose)

    # Build the local mesh for processor 0
    points, vertices, boundary, quantities, \
            ghost_recv_dict, full_send_dict, \
            tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
              extract_submesh(submesh, triangles_per_proc, p2s_map, 0)

    # Keep track of the number full nodes and triangles.
    # This is useful later if one needs access to a ghost-free domain
    # Here, we do it for process 0. The others are done in rec_submesh.
    number_of_full_nodes = len(submesh['full_nodes'][0])
    number_of_full_triangles = len(submesh['full_triangles'][0])

    #print
    #for p in range(numprocs):
    #    print 'Process %d:' %(p)
    #
    #    print 'full_triangles:'
    #    print submesh['full_triangles'][p]
    #
    #    print 'full_nodes:'
    #    print submesh['full_nodes'][p]
    #
    #    print 'ghost_triangles:'
    #    print submesh['ghost_triangles'][p]#
    #
    #    print 'ghost_nodes:'
    #   print submesh['ghost_nodes'][p]
    #    print
    #
    #print 'Receive dict'
    #print ghost_recv_dict
    #
    #print 'Send dict'
    #print full_send_dict

    # Return structures necessary for building the parallel domain
    return points, vertices, boundary, quantities,\
           ghost_recv_dict, full_send_dict,\
           number_of_full_nodes, number_of_full_triangles, \
           s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
           ghost_layer_width
    from anuga.parallel.distribute_mesh import rec_submesh
    from anuga.parallel.distribute_mesh import extract_submesh

    # Mesh partitioning using Metis
    from anuga.parallel.distribute_mesh import build_submesh
    from anuga.parallel.distribute_mesh import pmesh_divide_metis_with_map

    from anuga.parallel.parallel_shallow_water import Parallel_domain

from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh

#------------------------------------------------------------------------------
# Read in processor information
#------------------------------------------------------------------------------

numprocs = size()
myid = rank()
processor_name = get_processor_name()
#print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name)


def collect_value(value):

    value = value

    if myid == 0:
        for i in range(numprocs):
            if i == 0: continue
            val = receive(i)
            value = value + val
    else:
def allocate_inlet_procs(domain,
                         poly,
                         enquiry_point=None,
                         master_proc=0,
                         procs=None,
                         verbose=False):

    from anuga.utilities import parallel_abstraction as pypar
    if procs is None:
        procs = list(range(0, pypar.size()))

    myid = pypar.rank()
    vertex_coordinates = domain.get_full_vertex_coordinates(absolute=True)
    domain_centroids = domain.centroid_coordinates
    size = 0
    has_enq_point = False
    numprocs = pypar.size()

    inlet_procs = []
    max_size = -1
    inlet_master_proc = -1
    inlet_enq_proc = -1

    # Calculate the number of points of the line inside full polygon

    #tri_id = line_intersect(vertex_coordinates, poly)

    if len(poly) == 2:  # poly is a line
        if verbose: print("======================")
        tri_id = line_intersect(vertex_coordinates, poly)
    else:  # poly is a polygon
        if verbose: print("+++++++++++++++++++++++")
        tris_0 = line_intersect(vertex_coordinates, [poly[0], poly[1]])
        tris_1 = inside_polygon(domain_centroids, poly)
        tri_id = num.union1d(tris_0, tris_1)

    if verbose:
        print("P%d has %d triangles in poly %s" % (myid, len(tri_id), poly))

    size = len(tri_id)

    if enquiry_point is not None:
        try:
            k = domain.get_triangle_containing_point(enquiry_point)

            if domain.tri_full_flag[k] == 1:
                size = size + 1
                has_enq_point = True
                if verbose:
                    print("P%d has enq point %s" % (myid, enquiry_point))
            else:
                if verbose:
                    print("P%d contains ghost copy of enq point %s" %
                          (myid, enquiry_point))
                has_enq_point = False
        except:
            if verbose:
                print("P%d does not contain enq point %s" %
                      (myid, enquiry_point))
            has_enq_point = False

    if myid == master_proc:
        # Recieve size of overlap from each processor
        # Initialize line_master_proc and inlet_procs

        if size > 0:
            inlet_procs = [master_proc]
            max_size = size
            inlet_master_proc = master_proc
            if has_enq_point:
                inlet_enq_proc = master_proc

        # Recieve size of overlap
        for i in procs:
            if i == master_proc: continue
            x = pypar.receive(i)
            y = pypar.receive(i)

            if x > 0:
                inlet_procs.append(i)

                # Choose inlet_master_proc as the one with the most overlap
                if x > max_size:
                    max_size = x
                    inlet_master_proc = i

                if y is True:
                    assert inlet_enq_proc == -1, "Enquiry point correspond to more than one proc"
                    inlet_enq_proc = i

        assert len(inlet_procs) > 0, "Line does not intersect any domain"
        assert inlet_master_proc >= 0, "No master processor assigned"

        if enquiry_point is not None:
            msg = "Enquiry point %s doesn't intersect mesh, maybe inside a building, try reducing enquiry_gap" % str(
                enquiry_point)
            if inlet_enq_proc < 0:
                raise Exception(msg)

        # Send inlet_master_proc and inlet_procs to all processors in inlet_procs
        for i in procs:
            if i != master_proc:
                pypar.send(inlet_master_proc, i)
                pypar.send(inlet_procs, i)
                pypar.send(inlet_enq_proc, i)

    else:
        pypar.send(size, master_proc)
        pypar.send(has_enq_point, master_proc)

        inlet_master_proc = pypar.receive(master_proc)
        inlet_procs = pypar.receive(master_proc)
        inlet_enq_proc = pypar.receive(master_proc)
        if has_enq_point:
            assert inlet_enq_proc == myid, "Enquiry found in proc, but not declared globally"

    if size > 0:
        return True, inlet_master_proc, inlet_procs, inlet_enq_proc
    else:
        return False, inlet_master_proc, inlet_procs, inlet_enq_proc
def Internal_boundary_operator(domain,
                               internal_boundary_function,
                               width=1.,
                               height=1.,
                               end_points=None,
                               exchange_lines=None,
                               enquiry_points=None,
                               invert_elevation=None,
                               apron=0.0,
                               enquiry_gap=0.0,
                               use_velocity_head=False,
                               zero_outflow_momentum=False,
                               force_constant_inlet_elevations=True,
                               smoothing_timescale=0.0,
                               compute_discharge_implicitly=True,
                               description=None,
                               label=None,
                               structure_type='internal_boundary',
                               logging=False,
                               verbose=True,
                               master_proc=0,
                               procs=None,
                               inlet_master_proc=[0, 0],
                               inlet_procs=None,
                               enquiry_proc=[0, 0]):

    # If not parallel domain then allocate serial Internal boundary operator
    if isinstance(domain, Parallel_domain) is False:
        if verbose:
            print("Allocating non parallel internal_boundary operator .....")
        return anuga.structures.internal_boundary_operator.Internal_boundary_operator(
            domain=domain,
            internal_boundary_function=internal_boundary_function,
            width=width,
            height=height,
            end_points=end_points,
            exchange_lines=exchange_lines,
            enquiry_points=enquiry_points,
            invert_elevation=invert_elevation,
            apron=apron,
            enquiry_gap=enquiry_gap,
            use_velocity_head=use_velocity_head,
            zero_outflow_momentum=zero_outflow_momentum,
            force_constant_inlet_elevations=force_constant_inlet_elevations,
            smoothing_timescale=smoothing_timescale,
            compute_discharge_implicitly=compute_discharge_implicitly,
            description=description,
            label=label,
            structure_type=structure_type,
            logging=logging,
            verbose=verbose)

    from anuga.utilities import parallel_abstraction as pypar
    if procs is None:
        procs = list(range(0, pypar.size()))

    myid = pypar.rank()

    end_points = ensure_numeric(end_points)
    exchange_lines = ensure_numeric(exchange_lines)
    enquiry_points = ensure_numeric(enquiry_points)

    if height is None:
        height = width

    diameter = None

    if apron is None:
        apron = width

    # Calculate location of inlet enquiry points and exchange lines
    if myid == master_proc:
        if exchange_lines is not None:
            exchange_lines_tmp = exchange_lines
            enquiry_points_tmp = __process_skew_culvert(
                exchange_lines, end_points, enquiry_points, apron, enquiry_gap)

            for i in procs:
                if i == master_proc: continue
                pypar.send(enquiry_points_tmp, i)

        elif end_points is not None:
            exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert(
                end_points, width, enquiry_points, apron, enquiry_gap)
            for i in procs:
                if i == master_proc: continue
                pypar.send(exchange_lines_tmp, i)
                pypar.send(enquiry_points_tmp, i)
        else:
            raise Exception('Define either exchange_lines or end_points')

    else:
        if exchange_lines is not None:
            exchange_lines_tmp = exchange_lines
            enquiry_points_tmp = pypar.receive(master_proc)
        elif end_points is not None:
            exchange_lines_tmp = pypar.receive(master_proc)
            enquiry_points_tmp = pypar.receive(master_proc)

    # Determine processors associated with first inlet
    line0 = exchange_lines_tmp[0]
    enquiry_point0 = enquiry_points_tmp[0]

    alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs(
        domain,
        line0,
        enquiry_point=enquiry_point0,
        master_proc=master_proc,
        procs=procs,
        verbose=verbose)

    # Determine processors associated with second inlet
    line1 = exchange_lines_tmp[1]
    enquiry_point1 = enquiry_points_tmp[1]

    alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs(
        domain,
        line1,
        enquiry_point=enquiry_point1,
        master_proc=master_proc,
        procs=procs,
        verbose=verbose)

    structure_procs = list(set(inlet0_procs + inlet1_procs))
    inlet_master_proc = [inlet0_master_proc, inlet1_master_proc]
    inlet_procs = [inlet0_procs, inlet1_procs]
    enquiry_proc = [enquiry0_proc, enquiry1_proc]

    if myid == master_proc and verbose:
        print(
            "Parallel Internal boundary Operator ============================="
        )
        print("Structure Master Proc is P" + str(inlet0_master_proc))
        print("Structure Procs are P" + str(structure_procs))
        print("Inlet Master Procs are P" + str(inlet_master_proc))
        print("Inlet Procs are P" + str(inlet_procs[0]) + " and " +
              str(inlet_procs[1]))
        print("Inlet Enquiry Procs are P" + str(enquiry_proc))
        print("Enquiry Points are " + str(enquiry_point0) + " and " +
              str(enquiry_point1))
        print("Inlet Exchange Lines are " + str(line0) + " and " + str(line1))
        print("========================================================")

    if alloc0 or alloc1:
        return Parallel_Internal_boundary_operator(
            domain=domain,
            internal_boundary_function=internal_boundary_function,
            width=width,
            height=height,
            end_points=end_points,
            exchange_lines=exchange_lines,
            enquiry_points=enquiry_points,
            invert_elevation=invert_elevation,
            apron=apron,
            enquiry_gap=enquiry_gap,
            use_velocity_head=use_velocity_head,
            zero_outflow_momentum=zero_outflow_momentum,
            force_constant_inlet_elevations=force_constant_inlet_elevations,
            smoothing_timescale=smoothing_timescale,
            compute_discharge_implicitly=compute_discharge_implicitly,
            description=description,
            label=label,
            structure_type=structure_type,
            logging=logging,
            verbose=verbose,
            master_proc=inlet0_master_proc,
            procs=structure_procs,
            inlet_master_proc=inlet_master_proc,
            inlet_procs=inlet_procs,
            enquiry_proc=enquiry_proc)
    else:
        return None
def Weir_orifice_trapezoid_operator(
        domain,
        losses,
        width,
        blockage=0.0,
        barrels=1.0,
        z1=None,
        z2=None,
        height=None,
        end_points=None,
        exchange_lines=None,
        enquiry_points=None,
        invert_elevations=None,
        #culvert_slope=None,
        apron=0.1,
        manning=0.013,
        enquiry_gap=0.0,
        smoothing_timescale=0.0,
        use_momentum_jet=True,
        use_velocity_head=True,
        description=None,
        label=None,
        structure_type='weir_orifice_trapezoid',
        logging=False,
        verbose=False,
        master_proc=0,
        procs=None):

    # If not parallel domain then allocate serial Weir orifice trapezoid operator
    if isinstance(domain, Parallel_domain) is False:
        if verbose:
            print(
                "Allocating non parallel weir orifice trapzezoid operator ....."
            )
        return anuga.structures.weir_orifice_trapezoid_operator.Weir_orifice_trapezoid_operator(
            domain=domain,
            losses=losses,
            width=width,
            height=height,
            blockage=blockage,
            barrels=barrels,
            z1=z1,
            z2=z2,
            #culvert_slope=culvert_slope,
            end_points=end_points,
            exchange_lines=exchange_lines,
            enquiry_points=enquiry_points,
            invert_elevations=invert_elevations,
            apron=apron,
            manning=manning,
            enquiry_gap=enquiry_gap,
            smoothing_timescale=smoothing_timescale,
            use_momentum_jet=use_momentum_jet,
            use_velocity_head=use_velocity_head,
            description=description,
            label=label,
            structure_type=structure_type,
            logging=logging,
            verbose=verbose)

    from anuga.utilities import parallel_abstraction as pypar
    if procs is None:
        procs = list(range(0, pypar.size()))

    myid = pypar.rank()

    end_points = ensure_numeric(end_points)
    exchange_lines = ensure_numeric(exchange_lines)
    enquiry_points = ensure_numeric(enquiry_points)

    if height is None:
        height = width

    diameter = None

    if apron is None:
        apron = width

    # Calculate location of inlet enquiry points and exchange lines
    if myid == master_proc:
        if exchange_lines is not None:
            exchange_lines_tmp = exchange_lines
            enquiry_points_tmp = __process_skew_culvert(
                exchange_lines, end_points, enquiry_points, apron, enquiry_gap)

            for i in procs:
                if i == master_proc: continue
                pypar.send(enquiry_points_tmp, i)

        elif end_points is not None:
            exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert(
                end_points, width, enquiry_points, apron, enquiry_gap)
            for i in procs:
                if i == master_proc: continue
                pypar.send(exchange_lines_tmp, i)
                pypar.send(enquiry_points_tmp, i)
        else:
            raise Exception('Define either exchange_lines or end_points')

    else:
        if exchange_lines is not None:
            exchange_lines_tmp = exchange_lines
            enquiry_points_tmp = pypar.receive(master_proc)
        elif end_points is not None:
            exchange_lines_tmp = pypar.receive(master_proc)
            enquiry_points_tmp = pypar.receive(master_proc)

    # Determine processors associated with first inlet
    line0 = exchange_lines_tmp[0]
    enquiry_point0 = enquiry_points_tmp[0]

    alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs(
        domain,
        line0,
        enquiry_point=enquiry_point0,
        master_proc=master_proc,
        procs=procs,
        verbose=verbose)

    # Determine processors associated with second inlet
    line1 = exchange_lines_tmp[1]
    enquiry_point1 = enquiry_points_tmp[1]

    alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs(
        domain,
        line1,
        enquiry_point=enquiry_point1,
        master_proc=master_proc,
        procs=procs,
        verbose=verbose)

    structure_procs = list(set(inlet0_procs + inlet1_procs))
    inlet_master_proc = [inlet0_master_proc, inlet1_master_proc]
    inlet_procs = [inlet0_procs, inlet1_procs]
    enquiry_proc = [enquiry0_proc, enquiry1_proc]

    if myid == master_proc and verbose:
        print(
            "Parallel Weir Orifice Trapezoid Operator ============================="
        )
        print("Structure Master Proc is P" + str(inlet0_master_proc))
        print("Structure Procs are P" + str(structure_procs))
        print("Inlet Master Procs are P" + str(inlet_master_proc))
        print("Inlet Procs are P" + str(inlet_procs[0]) + " and " +
              str(inlet_procs[1]))
        print("Inlet Enquiry Procs are P" + str(enquiry_proc))
        print("Enquiry Points are " + str(enquiry_point0) + " and " +
              str(enquiry_point1))
        print("Inlet Exchange Lines are " + str(line0) + " and " + str(line1))
        print("========================================================")

    if alloc0 or alloc1:
        return Parallel_Weir_orifice_trapezoid_operator(
            domain=domain,
            losses=losses,
            width=width,
            height=height,
            blockage=blockage,
            barrels=barrels,
            z1=z1,
            z2=z2,
            #culvert_slope=culvert_slope,
            end_points=end_points,
            exchange_lines=exchange_lines,
            enquiry_points=enquiry_points,
            invert_elevations=invert_elevations,
            apron=apron,
            manning=manning,
            enquiry_gap=enquiry_gap,
            smoothing_timescale=smoothing_timescale,
            use_momentum_jet=use_momentum_jet,
            use_velocity_head=use_velocity_head,
            description=description,
            label=label,
            structure_type=structure_type,
            logging=logging,
            verbose=verbose,
            master_proc=inlet0_master_proc,
            procs=structure_procs,
            inlet_master_proc=inlet_master_proc,
            inlet_procs=inlet_procs,
            enquiry_proc=enquiry_proc)
    else:
        return None
    def __init__(self, coordinates, vertices,
                 boundary=None,
                 full_send_dict=None,
                 ghost_recv_dict=None,
                 number_of_full_nodes=None,
                 number_of_full_triangles=None,
                 geo_reference=None,
                 processor = None,
                 numproc = None,
                 number_of_global_triangles=None, ## SR added this
                 number_of_global_nodes= None, ## SR added this
                 s2p_map=None,
                 p2s_map=None, #jj added this
                 tri_l2g = None, ## SR added this
                 node_l2g = None, #): ## SR added this
                 ghost_layer_width = 2): ## SR added this



        #-----------------------------------------
        # Sometimes we want to manually
        # create instances of the parallel_domain
        # otherwise ...
        #----------------------------------------
        if processor is None:
            processor = pypar.rank()
        if numproc is None:
            numproc = pypar.size()

        Domain.__init__(self,
                        coordinates,
                        vertices,
                        boundary,
                        full_send_dict=full_send_dict,
                        ghost_recv_dict=ghost_recv_dict,
                        processor=processor,
                        numproc=numproc,
                        number_of_full_nodes=number_of_full_nodes,
                        number_of_full_triangles=number_of_full_triangles,
                        geo_reference=geo_reference, #) #jj added this
                        ghost_layer_width = ghost_layer_width)
        

        self.parallel = True

        # PETE: Find the number of full nodes and full triangles, this is a temporary fix
        # until the bug with get_number_of_full_[nodes|triangles]() is fixed.

        if number_of_full_nodes is not None:
            self.number_of_full_nodes_tmp = number_of_full_nodes
        else:
            self.number_of_full_nodes_tmp = self.get_number_of_nodes()

        if number_of_full_triangles is not None:
            self.number_of_full_triangles_tmp = number_of_full_triangles
        else:
            self.number_of_full_triangles_tmp = self.get_number_of_triangles()

        generic_comms.setup_buffers(self)

        self.global_name = 'domain'

        self.number_of_global_triangles=number_of_global_triangles
        self.number_of_global_nodes = number_of_global_nodes

        self.s2p_map = s2p_map
        self.p2s_map = p2s_map


        self.s2p_map = None
        self.p2s_map = None

        self.tri_l2g = tri_l2g
        self.node_l2g = node_l2g

        self.ghost_counter = 0
Exemple #12
0
    def dump_triangulation(self, filename="domain.png"):
        # Get vertex coordinates, partition full and ghost triangles based on self.tri_full_flag

        try:
            import matplotlib
            matplotlib.use('Agg')
            import matplotlib.pyplot as plt
            import matplotlib.tri as tri
        except:
            print(
                "Couldn't import module from matplotlib, probably you need to update matplotlib"
            )
            raise

        vertices = self.get_vertex_coordinates()
        full_mask = num.repeat(self.tri_full_flag == 1, 3)
        ghost_mask = num.repeat(self.tri_full_flag == 0, 3)

        myid = pypar.rank()
        numprocs = pypar.size()

        if myid == 0:

            fig = plt.figure()
            fx = {}
            fy = {}
            gx = {}
            gy = {}

            # Proc 0 gathers full and ghost nodes from self and other processors
            fx[0] = vertices[full_mask, 0]
            fy[0] = vertices[full_mask, 1]
            gx[0] = vertices[ghost_mask, 0]
            gy[0] = vertices[ghost_mask, 1]

            for i in range(1, numprocs):
                fx[i] = pypar.receive(i)
                fy[i] = pypar.receive(i)
                gx[i] = pypar.receive(i)
                gy[i] = pypar.receive(i)

            # Plot full triangles
            for i in range(0, numprocs):
                n = int(old_div(len(fx[i]), 3))

                triang = num.array(list(range(0, 3 * n)))
                triang.shape = (n, 3)
                plt.triplot(fx[i], fy[i], triang, 'g-', linewidth=0.5)

            # Plot ghost triangles
            for i in range(0, numprocs):
                n = int(old_div(len(gx[i]), 3))
                if n > 0:
                    triang = num.array(list(range(0, 3 * n)))
                    triang.shape = (n, 3)
                    plt.triplot(gx[i], gy[i], triang, 'b--', linewidth=0.5)

            # Save triangulation to location pointed by filename
            plt.savefig(filename, dpi=600)

        else:
            # Proc 1..numprocs send full and ghost triangles to Proc 0
            pypar.send(vertices[full_mask, 0], 0)
            pypar.send(vertices[full_mask, 1], 0)
            pypar.send(vertices[ghost_mask, 0], 0)
            pypar.send(vertices[ghost_mask, 1], 0)
Exemple #13
0
    # Mesh partitioning using Metis
    from anuga.parallel.distribute_mesh import build_submesh
    from anuga.parallel.distribute_mesh import pmesh_divide_metis_with_map

    from anuga.parallel.parallel_shallow_water import Parallel_domain
    
    

from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh

#------------------------------------------------------------------------------
# Read in processor information
#------------------------------------------------------------------------------

numprocs = size()
myid = rank()
processor_name = get_processor_name()
#print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name)



def collect_value(value):
    
    value = value

    if myid == 0:
        for i in range(numprocs):
            if i == 0: continue
            val = receive(i)
            value = value + val
def distibute_three_processors():
    """
    Do a parallel test of distributing a rectangle onto 3 processors

    """

    # FIXME: Need to update expected values on macos
    if sys.platform == 'darwin':
        return

    # FIXME: Need to update expected values on macos
    #if sys.platform == 'win32':
    #	return

    from anuga.utilities import parallel_abstraction as pypar

    myid = pypar.rank()
    numprocs = pypar.size()

    if not numprocs == 3:
        return

    try:
        import pymetis
        metis_version = 5
    except:
        metis_version = 4

    #print numprocs

    #barrier()

    if myid == 0:

        nodes_0, triangles_0, boundary_0 = rectangular_cross(2, 2)

        domain = Domain(nodes_0, triangles_0, boundary_0)

        domain.set_quantity('elevation',
                            topography)  # Use function for elevation
        domain.set_quantity('friction', 0.0)  # Constant friction
        domain.set_quantity('stage',
                            expression='elevation')  # Dry initial stage
        domain.set_quantity('xmomentum', expression='friction + 2.0')
        domain.set_quantity('ymomentum', ycoord)

        #----------------------------------------------------------------------------------
        # Test pmesh_divide_metis
        #----------------------------------------------------------------------------------
        vertices, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(
            domain, numprocs)

        if False:
            print_seq_values(vertices, triangles, triangles_per_proc)

        true_seq_values = get_true_seq_values(metis_version=metis_version)

        if False:
            print("True Seq Values = \\")
            pprint(true_seq_values)

        assert_allclose(vertices, true_seq_values['vertices'])
        assert_allclose(triangles, true_seq_values['triangles'])
        assert_allclose(triangles_per_proc,
                        true_seq_values['triangles_per_proc'])

        #----------------------------------------------------------------------------------
        # Test build_submesh
        #----------------------------------------------------------------------------------
        submesh = build_submesh(vertices, triangles, boundary, quantities,
                                triangles_per_proc)

        if False:
            print('submesh_values = \\')
            print_submesh_values(submesh)

        true_values = get_true_submesh_values(metis_version)

        assert_allclose(submesh['full_nodes'][0], true_values['full_nodes_0'])
        assert_allclose(submesh['full_nodes'][1], true_values['full_nodes_1'])
        assert_allclose(submesh['full_nodes'][2], true_values['full_nodes_2'])

        assert_allclose(submesh['ghost_nodes'][0],
                        true_values['ghost_nodes_0'])
        assert_allclose(submesh['ghost_nodes'][1],
                        true_values['ghost_nodes_1'])
        assert_allclose(submesh['ghost_nodes'][2],
                        true_values['ghost_nodes_2'])

        assert_allclose(submesh['full_triangles'][0],
                        true_values['full_triangles_0'])
        assert_allclose(submesh['full_triangles'][1],
                        true_values['full_triangles_1'])
        assert_allclose(submesh['full_triangles'][2],
                        true_values['full_triangles_2'])

        assert_allclose(submesh['ghost_triangles'][0],
                        true_values['ghost_triangles_0'])
        assert_allclose(submesh['ghost_triangles'][1],
                        true_values['ghost_triangles_1'])
        assert_allclose(submesh['ghost_triangles'][2],
                        true_values['ghost_triangles_2'])

        assert_allclose(submesh['ghost_commun'][0],
                        true_values['ghost_commun_0'])
        assert_allclose(submesh['ghost_commun'][1],
                        true_values['ghost_commun_1'])
        assert_allclose(submesh['ghost_commun'][2],
                        true_values['ghost_commun_2'])

        assert_(submesh['full_commun'] == true_values['full_commun'])

    barrier()
    #--------------------------------
    # Now do the comunnication part
    #--------------------------------

    if myid == 0:

        points, vertices, boundary, quantities, \
                    ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
                    ghost_layer_width =\
                    extract_submesh(submesh, triangles_per_proc)

        #----------------------------------------------------------------------------------
        # Test send_submesh
        #----------------------------------------------------------------------------------
        for p in range(1, numprocs):
            send_submesh(submesh, triangles_per_proc, p, verbose=False)
    else:
        #----------------------------------------------------------------------------------
        # Test rec_submesh
        #----------------------------------------------------------------------------------
        points, triangles, boundary, quantities, \
                ghost_recv_dict, full_send_dict, \
                no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
                ghost_layer_width = \
                rec_submesh(0, verbose=False)

    barrier()

    #--------------------------------
    # Now do the test
    #--------------------------------
    if myid == 0:

        if False:
            print('extract_values = \\')
            print_extract_submesh(points, triangles, ghost_recv_dict, \
                                  full_send_dict, tri_map, node_map, ghost_layer_width)

        true_values = get_true_extract_submesh(metis_version)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
        assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
        assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
        assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    if myid == 1:

        if False:
            print("rec_submesh_1 = \\")
            print_rec_submesh_1(points, triangles, ghost_recv_dict, full_send_dict, \
                         tri_map, node_map, ghost_layer_width)

        true_values = get_true_rec_submesh_1(metis_version)

        if False:
            print('true_rec_values_1 = \\')
            pprint(true_values)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
        assert_allclose(ghost_recv_dict[2], true_values['ghost_recv_dict_2'])
        assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
        assert_allclose(full_send_dict[2], true_values['full_send_dict_2'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    if myid == 2:

        if False:
            print("rec_submesh_2 = \\")
            print_rec_submesh_2(points, triangles, ghost_recv_dict, full_send_dict, \
                         tri_map, node_map, ghost_layer_width)

        true_values = get_true_rec_submesh_2(metis_version)

        if False:
            print('true_rec_values_2 = \\')
            pprint(true_values)

        assert_allclose(points, true_values['points'])
        assert_allclose(triangles, true_values['triangles'])
        assert_allclose(ghost_recv_dict[0], true_values['ghost_recv_dict_0'])
        assert_allclose(ghost_recv_dict[1], true_values['ghost_recv_dict_1'])
        assert_allclose(full_send_dict[0], true_values['full_send_dict_0'])
        assert_allclose(full_send_dict[1], true_values['full_send_dict_1'])
        assert_allclose(tri_map, true_values['tri_map'])
        assert_allclose(node_map, true_values['node_map'])
        assert_allclose(ghost_layer_width, true_values['ghost_layer_width'])

    finalize()
def plotCentroidError(domain, control_data, rthr = 1E-7, athr = 1E-12, 
                      quantity = 'stage', filename = 'centroid_error.png'):

    n_triangles = num.sum(domain.tri_full_flag)
    
    if size() > 1:
        # If parallel, translate control data to parallel indexing
        local_control_data = num.zeros(n_triangles)
        inv_tri_map = domain.get_inv_tri_map()
        
        for i in range(n_triangles):
            local_control_data[i] = control_data[inv_tri_map[(rank(), i)]]
    else:
        local_control_data = control_data

    # Evaluate absolute and relative difference between control and actual values
    stage = domain.get_quantity(quantity)
    actual_data = stage.centroid_values[:n_triangles]
    adiff = num.fabs((actual_data - local_control_data))
    rdiff = adiff/num.fabs(local_control_data)

    # Compute masks for error (err_mask) and non-error (acc_mask) vertex indices based on thresholds
    vertices = domain.get_vertex_coordinates()
    err_mask = rdiff > rthr
    err_mask[adiff <= athr] = False    
    err_mask = num.repeat(err_mask, 3)

    acc_mask = ~err_mask
    inv_tri_map = domain.get_inv_tri_map()

    # Plot error and non-error triangle
    if rank() == 0:
        fx = {}
        fy = {}
        gx = {}
        gy = {}

        fx[0] = vertices[acc_mask,0]
        fy[0] = vertices[acc_mask,1]
        gx[0] = vertices[err_mask,0]
        gy[0] = vertices[err_mask,1]

        # Receive vertex indices of non-error triangles (fx, fy) and error triangles (gx, gy)
        for i in range(1,size()):
            fx[i] = receive(i)
            fy[i] = receive(i)
            gx[i] = receive(i)
            gy[i] = receive(i)

        # Plot non-error triangles in green
        for i in range(0,size()):
            n = int(len(fx[i])/3)
            triang = num.array(range(0,3*n))
            triang.shape = (n, 3)

            if len(fx[i]) > 0:
                plt.triplot(fx[i], fy[i], triang, 'g-')

        # Plot error triangles in blue
        for i in range(0,size()):
            n = int(len(gx[i])/3)
                            
            triang = num.array(range(0,3*n))
            triang.shape = (n, 3)

            if len(gx[i]) > 0: 
                plt.triplot(gx[i], gy[i], triang, 'b--')
                
        # Save plot
        plt.savefig(filename)
        
    else:
        # Send error and non-error vertex indices to Proc 0
        send(vertices[acc_mask,0], 0)
        send(vertices[acc_mask,1], 0)
        send(vertices[err_mask,0], 0)
        send(vertices[err_mask,1], 0)      
Exemple #16
0
def plotCentroidError(domain,
                      control_data,
                      rthr=1E-7,
                      athr=1E-12,
                      quantity='stage',
                      filename='centroid_error.png'):

    n_triangles = num.sum(domain.tri_full_flag)

    if size() > 1:
        # If parallel, translate control data to parallel indexing
        local_control_data = num.zeros(n_triangles)
        inv_tri_map = domain.get_inv_tri_map()

        for i in range(n_triangles):
            local_control_data[i] = control_data[inv_tri_map[(rank(), i)]]
    else:
        local_control_data = control_data

    # Evaluate absolute and relative difference between control and actual values
    stage = domain.get_quantity(quantity)
    actual_data = stage.centroid_values[:n_triangles]
    adiff = num.fabs((actual_data - local_control_data))
    rdiff = adiff / num.fabs(local_control_data)

    # Compute masks for error (err_mask) and non-error (acc_mask) vertex indices based on thresholds
    vertices = domain.get_vertex_coordinates()
    err_mask = rdiff > rthr
    err_mask[adiff <= athr] = False
    err_mask = num.repeat(err_mask, 3)

    acc_mask = ~err_mask
    inv_tri_map = domain.get_inv_tri_map()

    # Plot error and non-error triangle
    if rank() == 0:
        fx = {}
        fy = {}
        gx = {}
        gy = {}

        fx[0] = vertices[acc_mask, 0]
        fy[0] = vertices[acc_mask, 1]
        gx[0] = vertices[err_mask, 0]
        gy[0] = vertices[err_mask, 1]

        # Receive vertex indices of non-error triangles (fx, fy) and error triangles (gx, gy)
        for i in range(1, size()):
            fx[i] = receive(i)
            fy[i] = receive(i)
            gx[i] = receive(i)
            gy[i] = receive(i)

        # Plot non-error triangles in green
        for i in range(0, size()):
            n = int(len(fx[i]) / 3)
            triang = num.array(range(0, 3 * n))
            triang.shape = (n, 3)

            if len(fx[i]) > 0:
                plt.triplot(fx[i], fy[i], triang, 'g-')

        # Plot error triangles in blue
        for i in range(0, size()):
            n = int(len(gx[i]) / 3)

            triang = num.array(range(0, 3 * n))
            triang.shape = (n, 3)

            if len(gx[i]) > 0:
                plt.triplot(gx[i], gy[i], triang, 'b--')

        # Save plot
        plt.savefig(filename)

    else:
        # Send error and non-error vertex indices to Proc 0
        send(vertices[acc_mask, 0], 0)
        send(vertices[acc_mask, 1], 0)
        send(vertices[err_mask, 0], 0)
        send(vertices[err_mask, 1], 0)
Exemple #17
0
def rec_submesh_flat(p, verbose=True):

    from anuga.utilities import parallel_abstraction as pypar

    numprocs = pypar.size()
    myid = pypar.rank()

    submesh_cell = {}

    if verbose:
        print(indent + 'P%d: Receiving submesh from P%d' % (myid, p))

    # receive the tagmap for the boundary conditions

    tagmap = pypar.receive(p)

    itagmap = {}
    for t in tagmap:
        itagmap[tagmap[t]] = t

    # receive the quantities key information
    qkeys = pypar.receive(p)

    # recieve information about the array sizes
    x = num.zeros((9, ), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    setup_array = x

    no_full_nodes = setup_array[0]
    no_ghost_nodes = setup_array[1]
    no_full_triangles = setup_array[2]
    no_ghost_triangles = setup_array[3]
    no_full_boundary = setup_array[4]
    no_ghost_boundary = setup_array[5]
    no_ghost_commun = setup_array[6]
    no_full_commun = setup_array[7]
    no_quantities = setup_array[8]

    # ghost layer width
    x = num.zeros((1, ), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    submesh_cell["ghost_layer_width"] = x[0]

    # receive the number of triangles per processor
    x = num.zeros((numprocs, ), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    triangles_per_proc = x

    # receive the full nodes
    x = num.zeros((no_full_nodes, 3), num.float)
    pypar.receive(p, buffer=x, bypass=True)
    submesh_cell["full_nodes"] = x

    # receive the ghost nodes
    x = num.zeros((no_ghost_nodes, 3), num.float)
    pypar.receive(p, buffer=x, bypass=True)
    submesh_cell["ghost_nodes"] = x

    # receive the full triangles
    x = num.zeros((no_full_triangles, 3), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    submesh_cell["full_triangles"] = x

    # receive the ghost triangles
    x = num.zeros((no_ghost_triangles, 4), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    submesh_cell["ghost_triangles"] = x

    # receive the full boundary
    x = num.zeros((no_full_boundary, 3), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    bnd_c = x

    submesh_cell["full_boundary"] = {}
    for b in bnd_c:
        submesh_cell["full_boundary"][b[0], b[1]] = itagmap[b[2]]

    # receive the ghost boundary
    x = num.zeros((no_ghost_boundary, 3), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    bnd_c = x

    submesh_cell["ghost_boundary"] = {}
    for b in bnd_c:
        submesh_cell["ghost_boundary"][b[0], b[1]] = itagmap[b[2]]

    # receive the ghost communication pattern
    x = num.zeros((no_ghost_commun, 2), num.int)

    pypar.receive(p, buffer=x, bypass=True)
    submesh_cell["ghost_commun"] = x

    # receive the full communication pattern
    x = num.zeros((no_full_commun, 2), num.int)
    pypar.receive(p, buffer=x, bypass=True)
    full_commun = x

    submesh_cell["full_commun"] = {}
    for c in full_commun:
        submesh_cell["full_commun"][c[0]] = []
    for c in full_commun:
        submesh_cell["full_commun"][c[0]].append(c[1])

    # receive the quantities

    submesh_cell["full_quan"] = {}
    for i in range(no_quantities):
        x = num.zeros((no_full_triangles, 3), num.float)
        pypar.receive(p, buffer=x, bypass=True)
        submesh_cell["full_quan"][qkeys[i]] = x

    submesh_cell["ghost_quan"] = {}
    for i in range(no_quantities):
        x = num.zeros((no_ghost_triangles, 3), num.float)
        pypar.receive(p, buffer=x, bypass=True)
        submesh_cell["ghost_quan"][qkeys[i]] = x

    return submesh_cell, triangles_per_proc,\
        no_full_nodes, no_full_triangles
    def __init__(self,
                 domain,
                 end_points,
                 exchange_lines,
                 enquiry_points,
                 invert_elevations,
                 width,
                 height,
                 diameter,
                 z1,
                 z2,
                 blockage,
                 barrels,
                 apron,
                 manning,
                 enquiry_gap,
                 use_momentum_jet,
                 zero_outflow_momentum,
                 use_old_momentum_method,
                 always_use_Q_wetdry_adjustment,
                 force_constant_inlet_elevations,
                 description,
                 label,
                 structure_type,
                 logging,
                 verbose,
                 master_proc=0,
                 procs=None,
                 inlet_master_proc=[0, 0],
                 inlet_procs=None,
                 enquiry_proc=None):

        self.myid = pypar.rank()
        self.num_procs = pypar.size()

        anuga.Operator.__init__(self, domain)

        # Allocate default processor associations if not specified in arguments
        # although we assume that such associations are provided correctly by the
        # parallel_operator_factory.

        self.master_proc = master_proc
        self.inlet_master_proc = inlet_master_proc

        if procs is None:
            self.procs = [master_proc]
        else:
            self.procs = procs

        if inlet_procs is None:
            self.inlet_procs = [[inlet_master_proc[0]], [inlet_master_proc[0]]]
        else:
            self.inlet_procs = inlet_procs

        if enquiry_proc is None:
            self.enquiry_proc = [[inlet_master_proc[0]],
                                 [inlet_master_proc[0]]]
        else:
            self.enquiry_proc = enquiry_proc

        self.end_points = ensure_numeric(end_points)
        self.exchange_lines = ensure_numeric(exchange_lines)
        self.enquiry_points = ensure_numeric(enquiry_points)
        self.invert_elevations = ensure_numeric(invert_elevations)

        assert (width is not None
                and diameter is None) or (width is None
                                          and diameter is not None)

        if width is None:
            width = diameter

        if diameter is None:
            diameter = width

        if height is None:
            height = width

        if apron is None:
            apron = width

        self.width = width
        self.height = height
        self.diameter = diameter
        self.z1 = z1
        self.z2 = z2
        self.blockage = blockage
        self.barrels = barrels
        self.apron = apron
        self.manning = manning
        self.enquiry_gap = enquiry_gap
        self.use_momentum_jet = use_momentum_jet
        self.zero_outflow_momentum = zero_outflow_momentum
        if use_momentum_jet and zero_outflow_momentum:
            msg = "Can't have use_momentum_jet and zero_outflow_momentum both True"
            raise Exception(msg)
        self.use_old_momentum_method = use_old_momentum_method
        self.always_use_Q_wetdry_adjustment = always_use_Q_wetdry_adjustment

        if description is None:
            self.description = ' '
        else:
            self.description = description

        if label is None:
            self.label = "structure_%g" % Parallel_Structure_operator.counter + "_P" + str(
                self.myid)
        else:
            self.label = label + '_%g' % Parallel_Structure_operator.counter + "_P" + str(
                self.myid)

        if structure_type is None:
            self.structure_type = 'generic structure'
        else:
            self.structure_type = structure_type

        self.verbose = verbose

        # Keep count of structures
        if self.myid == master_proc:
            Parallel_Structure_operator.counter += 1

        # Slots for recording current statistics
        self.accumulated_flow = 0.0
        self.discharge = 0.0
        self.discharge_abs_timemean = 0.0
        self.velocity = 0.0
        self.outlet_depth = 0.0
        self.delta_total_energy = 0.0
        self.driving_energy = 0.0

        if exchange_lines is not None:
            self.__process_skew_culvert()
        elif end_points is not None:
            self.__process_non_skew_culvert()
        else:
            raise Exception('Define either exchange_lines or end_points')

        self.inlets = []

        # Allocate parallel inlet enquiry, assign None if processor is not associated with particular
        # inlet.

        if self.myid in self.inlet_procs[0]:
            line0 = self.exchange_lines[0]
            if self.apron is None:
                poly0 = line0
            else:
                offset = -self.apron * self.outward_vector_0
                poly0 = num.array(
                    [line0[0], line0[1], line0[1] + offset, line0[0] + offset])

            if self.invert_elevations is None:
                invert_elevation0 = None
            else:
                invert_elevation0 = self.invert_elevations[0]

            enquiry_point0 = self.enquiry_points[0]
            outward_vector0 = self.culvert_vector

            self.inlets.append(
                parallel_inlet_enquiry.Parallel_Inlet_enquiry(
                    self.domain,
                    line0,
                    enquiry_point0,
                    invert_elevation=invert_elevation0,
                    outward_culvert_vector=outward_vector0,
                    master_proc=self.inlet_master_proc[0],
                    procs=self.inlet_procs[0],
                    enquiry_proc=self.enquiry_proc[0],
                    verbose=self.verbose))

            if force_constant_inlet_elevations:
                # Try to enforce a constant inlet elevation
                inlet_global_elevation = self.inlets[
                    -1].get_global_average_elevation()
                self.inlets[-1].set_elevations(inlet_global_elevation)

        else:
            self.inlets.append(None)

        if self.myid in self.inlet_procs[1]:
            line1 = self.exchange_lines[1]
            if self.apron is None:
                poly1 = line1
            else:
                offset = -self.apron * self.outward_vector_1
                poly1 = num.array(
                    [line1[0], line1[1], line1[1] + offset, line1[0] + offset])

            if self.invert_elevations is None:
                invert_elevation1 = None
            else:
                invert_elevation1 = self.invert_elevations[1]

            enquiry_point1 = self.enquiry_points[1]
            outward_vector1 = -self.culvert_vector

            self.inlets.append(
                parallel_inlet_enquiry.Parallel_Inlet_enquiry(
                    self.domain,
                    line1,
                    enquiry_point1,
                    invert_elevation=invert_elevation1,
                    outward_culvert_vector=outward_vector1,
                    master_proc=self.inlet_master_proc[1],
                    procs=self.inlet_procs[1],
                    enquiry_proc=self.enquiry_proc[1],
                    verbose=self.verbose))

            if force_constant_inlet_elevations:
                # Try to enforce a constant inlet elevation
                inlet_global_elevation = self.inlets[
                    -1].get_global_average_elevation()
                self.inlets[-1].set_elevations(inlet_global_elevation)

        else:
            self.inlets.append(None)

        self.inflow_index = 0
        self.outflow_index = 1

        self.set_parallel_logging(logging)
Exemple #19
0
def distribute_mesh(domain, verbose=False, debug=False, parameters=None):
    """ Distribute andsend the mesh info to all the processors.
    Should only be run from processor 0 and will send info to the other
    processors.
    There should be a corresponding  rec_submesh called from all the other
    processors
    """

    if debug:
        verbose = True

    numprocs = size()

    
    # Subdivide the mesh
    if verbose: print 'Subdivide mesh'
    new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
           s2p_map, p2s_map = \
           pmesh_divide_metis_with_map(domain, numprocs)

    #PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
    #      sp2_map (maps parallel domain triangles to domain triangles)



    # Build the mesh that should be assigned to each processor,
    # this includes ghost nodes and the communication pattern
    if verbose: print 'Build submeshes'    
    submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities, triangles_per_proc, parameters)

    if verbose:
        for p in range(numprocs):
            N = len(submesh['ghost_nodes'][p])                
            M = len(submesh['ghost_triangles'][p])
            print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
                  %(N, M, p)

    #if debug:
    #    from pprint import pprint
    #    pprint(submesh)


    # Send the mesh partition to the appropriate processor
    if verbose: print 'Distribute submeshes'        
    for p in range(1, numprocs):
        send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose)

    # Build the local mesh for processor 0
    points, vertices, boundary, quantities, \
            ghost_recv_dict, full_send_dict, \
            tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
              extract_submesh(submesh, triangles_per_proc, p2s_map, 0)


              
    # Keep track of the number full nodes and triangles.
    # This is useful later if one needs access to a ghost-free domain
    # Here, we do it for process 0. The others are done in rec_submesh.
    number_of_full_nodes = len(submesh['full_nodes'][0])
    number_of_full_triangles = len(submesh['full_triangles'][0])
        
    #print
    #for p in range(numprocs):
    #    print 'Process %d:' %(p)
    #
    #    print 'full_triangles:'
    #    print submesh['full_triangles'][p]
    #
    #    print 'full_nodes:'
    #    print submesh['full_nodes'][p]
    #
    #    print 'ghost_triangles:'
    #    print submesh['ghost_triangles'][p]#
    #
    #    print 'ghost_nodes:'
    #   print submesh['ghost_nodes'][p]                                
    #    print
    #
    #print 'Receive dict'
    #print ghost_recv_dict
    #
    #print 'Send dict'
    #print full_send_dict        


    # Return structures necessary for building the parallel domain
    return points, vertices, boundary, quantities,\
           ghost_recv_dict, full_send_dict,\
           number_of_full_nodes, number_of_full_triangles, \
           s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
           ghost_layer_width
    def dump_triangulation(self, filename="domain.png"):
        # Get vertex coordinates, partition full and ghost triangles based on self.tri_full_flag

        try:
            import matplotlib
            matplotlib.use('Agg')
            import matplotlib.pyplot as plt
            import matplotlib.tri as tri
        except:
            print "Couldn't import module from matplotlib, probably you need to update matplotlib"
            raise

        vertices = self.get_vertex_coordinates()
        full_mask = num.repeat(self.tri_full_flag == 1, 3)
        ghost_mask = num.repeat(self.tri_full_flag == 0, 3)
        
        myid = pypar.rank()
        numprocs = pypar.size()

        if myid == 0:

            fig = plt.figure()
            fx = {}
            fy = {}
            gx = {}
            gy = {}

            # Proc 0 gathers full and ghost nodes from self and other processors
            fx[0] = vertices[full_mask,0]
            fy[0] = vertices[full_mask,1]
            gx[0] = vertices[ghost_mask,0]
            gy[0] = vertices[ghost_mask,1]
            
            for i in range(1,numprocs):
                fx[i] = pypar.receive(i)
                fy[i] = pypar.receive(i)
                gx[i] = pypar.receive(i)
                gy[i] = pypar.receive(i)

            # Plot full triangles
            for i in range(0, numprocs):
                n = int(len(fx[i])/3)
                            
                triang = num.array(range(0,3*n))
                triang.shape = (n, 3)
                plt.triplot(fx[i], fy[i], triang, 'g-', linewidth = 0.5)

            # Plot ghost triangles
            for i in range(0, numprocs):
                n = int(len(gx[i])/3)
                if n > 0:
                    triang = num.array(range(0,3*n))
                    triang.shape = (n, 3)
                    plt.triplot(gx[i], gy[i], triang, 'b--', linewidth = 0.5)

            # Save triangulation to location pointed by filename
            plt.savefig(filename, dpi=600)

        else:
            # Proc 1..numprocs send full and ghost triangles to Proc 0
            pypar.send(vertices[full_mask,0], 0)
            pypar.send(vertices[full_mask,1], 0)
            pypar.send(vertices[ghost_mask,0], 0)
            pypar.send(vertices[ghost_mask,1], 0)
Exemple #21
0
    def __init__(
            self,
            coordinates,
            vertices,
            boundary=None,
            full_send_dict=None,
            ghost_recv_dict=None,
            number_of_full_nodes=None,
            number_of_full_triangles=None,
            geo_reference=None,
            processor=None,
            numproc=None,
            number_of_global_triangles=None,  ## SR added this
            number_of_global_nodes=None,  ## SR added this
            s2p_map=None,
            p2s_map=None,  #jj added this
            tri_l2g=None,  ## SR added this
            node_l2g=None,  #): ## SR added this
            ghost_layer_width=2):  ## SR added this

        #-----------------------------------------
        # Sometimes we want to manually
        # create instances of the parallel_domain
        # otherwise ...
        #----------------------------------------
        if processor is None:
            processor = pypar.rank()
        if numproc is None:
            numproc = pypar.size()

        Domain.__init__(
            self,
            coordinates,
            vertices,
            boundary,
            full_send_dict=full_send_dict,
            ghost_recv_dict=ghost_recv_dict,
            processor=processor,
            numproc=numproc,
            number_of_full_nodes=number_of_full_nodes,
            number_of_full_triangles=number_of_full_triangles,
            geo_reference=geo_reference,  #) #jj added this
            ghost_layer_width=ghost_layer_width)

        self.parallel = True

        # PETE: Find the number of full nodes and full triangles, this is a temporary fix
        # until the bug with get_number_of_full_[nodes|triangles]() is fixed.

        if number_of_full_nodes is not None:
            self.number_of_full_nodes_tmp = number_of_full_nodes
        else:
            self.number_of_full_nodes_tmp = self.get_number_of_nodes()

        if number_of_full_triangles is not None:
            self.number_of_full_triangles_tmp = number_of_full_triangles
        else:
            self.number_of_full_triangles_tmp = self.get_number_of_triangles()

        generic_comms.setup_buffers(self)

        self.global_name = 'domain'

        self.number_of_global_triangles = number_of_global_triangles
        self.number_of_global_nodes = number_of_global_nodes

        self.s2p_map = s2p_map
        self.p2s_map = p2s_map

        self.s2p_map = None
        self.p2s_map = None

        self.tri_l2g = tri_l2g
        self.node_l2g = node_l2g

        self.ghost_counter = 0
def Inlet_operator(domain,
                   poly,
                   Q,
                   velocity=None,
                   zero_velocity=False,
                   default=0.0,
                   description=None,
                   label=None,
                   logging=False,
                   master_proc=0,
                   procs=None,
                   verbose=False):

    # If not parallel domain then allocate serial Inlet operator
    if isinstance(domain, Parallel_domain) is False:
        if verbose: print("Allocating non parallel inlet operator .....")
        return anuga.structures.inlet_operator.Inlet_operator(
            domain,
            poly,
            Q,
            velocity=velocity,
            zero_velocity=zero_velocity,
            default=default,
            description=description,
            label=label,
            logging=logging,
            verbose=verbose)

    from anuga.utilities import parallel_abstraction as pypar
    if procs is None:
        procs = list(range(0, pypar.size()))

    myid = pypar.rank()

    poly = num.array(poly, dtype='d')

    alloc, inlet_master_proc, inlet_procs, enquiry_proc = allocate_inlet_procs(
        domain, poly, master_proc=master_proc, procs=procs, verbose=verbose)

    if alloc:
        if verbose and myid == inlet_master_proc:
            print("Parallel Inlet Operator =================")
            print("Poly = " + str(poly))
            print("Master Processor is P%d" % (inlet_master_proc))
            print("Processors are P%s" % (inlet_procs))
            print("=========================================")

        return Parallel_Inlet_operator(domain,
                                       poly,
                                       Q,
                                       velocity=velocity,
                                       zero_velocity=zero_velocity,
                                       default=default,
                                       description=description,
                                       label=label,
                                       logging=logging,
                                       master_proc=inlet_master_proc,
                                       procs=inlet_procs,
                                       verbose=verbose)
    else:
        return None