def get_enquiry_depths(self): # Should be called from all processors associated with operator enq0 = None enq1 = None get0 = 'self.inlets[0].get_enquiry_depth()' get1 = 'self.inlets[1].get_enquiry_depth()' if self.myid == self.master_proc: if self.myid == self.enquiry_proc[0]: enq0 = eval(get0) else: enq0 = pypar.receive(self.enquiry_proc[0]) if self.myid == self.enquiry_proc[1]: enq1 = eval(get1) else: enq1 = pypar.receive(self.enquiry_proc[1]) else: if self.myid == self.enquiry_proc[0]: enq0 = eval(get0) pypar.send(enq0, self.master_proc) if self.myid == self.enquiry_proc[1]: enq1 = eval(get1) pypar.send(enq1, self.master_proc) return [enq0, enq1]
def get_enquiry_elevations(self): enq0 = None enq1 = None get0 = 'self.inlets[0].get_enquiry_elevation()' get1 = 'self.inlets[1].get_enquiry_elevation()' if self.myid == self.master_proc: if self.myid == self.enquiry_proc[0]: enq0 = eval(get0) else: enq0 = pypar.receive(self.enquiry_proc[0]) if self.myid == self.enquiry_proc[1]: enq1 = eval(get1) else: enq1 = pypar.receive(self.enquiry_proc[1]) else: if self.myid == self.enquiry_proc[0]: enq0 = eval(get0) pypar.send(enq0, self.master_proc) if self.myid == self.enquiry_proc[1]: enq1 = eval(get1) pypar.send(enq1, self.master_proc) return [enq0, enq1]
def get_global_average_elevation(self): # GLOBAL: Master processor gathers elevations from all child processors, and returns average # WARNING: requires synchronization, must be called by all procs associated # with this inlet from anuga.utilities import parallel_abstraction as pypar local_elevation = num.sum(self.get_elevations()*self.get_areas()) global_area = self.get_global_area() global_elevation = local_elevation if self.myid == self.master_proc: for i in self.procs: if i == self.master_proc: continue val = pypar.receive(i) global_elevation = global_elevation + val else: pypar.send(local_elevation, self.master_proc) if global_area > 0.0: return old_div(global_elevation,global_area) else: return 0.0
def __call__(self): from anuga.utilities import parallel_abstraction as pypar volume = 0 # Need to run global command on all processors current_volume = self.inlet.get_global_total_water_volume() total_area = self.inlet.get_global_area() # Only the master proc calculates the update if self.myid == self.master_proc: timestep = self.domain.get_timestep() t = self.domain.get_time() Q1 = self.update_Q(t) Q2 = self.update_Q(t + timestep) volume = 0.5 * (Q1 + Q2) * timestep assert current_volume >= 0.0, 'Volume of watrer in inlet negative!' for i in self.procs: if i == self.master_proc: continue pypar.send((volume, current_volume, total_area, timestep), i) else: volume, current_volume, total_area, timestep = pypar.receive( self.master_proc) #print self.myid, volume, current_volume, total_area, timestep self.applied_Q = old_div(volume, timestep) # Distribute positive volume so as to obtain flat surface otherwise # just pull water off to have a uniform depth. if volume >= 0.0: self.inlet.set_stages_evenly(volume) self.domain.fractional_step_volume_integral += volume if self.velocity is not None: # This is done locally without communication depths = self.inlet.get_depths() self.inlet.set_xmoms(self.inlet.get_xmoms() + depths * self.velocity[0]) self.inlet.set_ymoms(self.inlet.get_ymoms() + depths * self.velocity[1]) elif current_volume + volume >= 0.0: depth = old_div((current_volume + volume), total_area) self.inlet.set_depths(depth) self.domain.fractional_step_volume_integral += volume else: #extracting too much water! self.inlet.set_depths(0.0) self.applied_Q = -old_div(current_volume, timestep) self.domain.fractional_step_volume_integral -= current_volume
def collect_value(value): value = value if myid == 0: for i in range(numprocs): if i == 0: continue val = receive(i) value = value + val else: send(value, 0) return value
def get_global_total_water_volume(self): # GLOBAL: master proc gathers total water volumes from each proc and returns average # WARNING: requires synchronization, must be called by all procs associated # with this inlet from anuga.utilities import parallel_abstraction as pypar local_volume = num.sum(self.get_depths()*self.get_areas()) volume = local_volume if self.myid == self.master_proc: for i in self.procs: if i == self.master_proc: continue val = pypar.receive(i) volume = volume + val else: pypar.send(volume, self.master_proc) return volume
def print_l2_stats(full_edge): numprocs = pypar.size() myid = pypar.rank() tri_norm = zeros(3, Float) recv_norm = zeros(3, Float) tri_norm[0] = pow(l2_norm(full_edge[:, 0]), 2) tri_norm[1] = pow(l2_norm(full_edge[:, 1]), 2) tri_norm[2] = pow(l2_norm(full_edge[:, 2]), 2) if myid == 0: for p in range(numprocs - 1): pypar.receive(p + 1, recv_norm) tri_norm[0] = tri_norm[0] + recv_norm[0] tri_norm[1] = tri_norm[1] + recv_norm[1] tri_norm[2] = tri_norm[2] + recv_norm[2] print('l2_norm along each axis : [', pow(tri_norm[0], 0.5),', ', pow(tri_norm[1], 0.5), \ ', ', pow(tri_norm[2], 0.5), ']') else: pypar.send(tri_norm, 0)
def print_linf_stats(full_edge): numprocs = pypar.size() myid = pypar.rank() tri_norm = zeros(3, Float) recv_norm = zeros(3, Float) tri_norm[0] = linf_norm(full_edge[:, 0]) tri_norm[1] = linf_norm(full_edge[:, 1]) tri_norm[2] = linf_norm(full_edge[:, 2]) if myid == 0: for p in range(numprocs - 1): pypar.receive(p + 1, recv_norm) tri_norm[0] = max(tri_norm[0], recv_norm[0]) tri_norm[1] = max(tri_norm[1], recv_norm[1]) tri_norm[2] = max(tri_norm[2], recv_norm[2]) print('linf_norm along each axis : [', tri_norm[0], ', ', tri_norm[1], ', ', tri_norm[2], ']') else: pypar.send(tri_norm, 0)
def get_global_area(self): # GLOBAL: Master processor gathers area from all child processors, and returns value # WARNING: requires synchronization, must be called by all procs associated # with this inlet from anuga.utilities import parallel_abstraction as pypar local_area = self.area area = local_area if self.myid == self.master_proc: for i in self.procs: if i == self.master_proc: continue val = pypar.receive(i) area = area + val else: pypar.send(area, self.master_proc) return area
def collect_value(value): value = value if myid == 0: for i in range(numprocs): if i == 0: continue val = receive(i) value = value + val else: send(value, 0) if myid == 0: for i in range(1,numprocs): send(value,i) else: value = receive(0) return value
def get_global_average_ymom(self): # GLOBAL: master proc gathers all ymom values and returns average # WARNING: requires synchronization, must be called by all procs associated # with this inlet from anuga.utilities import parallel_abstraction as pypar global_area = self.get_global_area() local_ymoms = num.sum(self.get_ymoms() * self.get_areas()) global_ymoms = local_ymoms if self.myid == self.master_proc: for i in self.procs: if i == self.master_proc: continue val = pypar.receive(i) global_ymoms = global_ymoms + val else: pypar.send(local_ymoms, self.master_proc) if global_area > 0.0: return old_div(global_ymoms, global_area) else: return 0.0
def distribute(domain, verbose=False, debug=False, parameters=None): """ Distribute the domain to all processes parameters allows user to change size of ghost layer """ if not pypar_available or numprocs == 1: return domain # Bypass if myid == 0: from sequential_distribute import Sequential_distribute partition = Sequential_distribute(domain, verbose, debug, parameters) partition.distribute(numprocs) kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = partition.extract_submesh(0) for p in range(1, numprocs): tostore = partition.extract_submesh(p) send(tostore, p) else: kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = receive(0) #--------------------------------------------------------------------------- # Now Create parallel domain #--------------------------------------------------------------------------- parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs) #------------------------------------------------------------------------ # Copy in quantity data #------------------------------------------------------------------------ for q in quantities: try: parallel_domain.set_quantity(q, quantities[q]) except KeyError: #print 'Try to create quantity %s'% q from anuga import Quantity Q = Quantity(parallel_domain, name=q, register=True) parallel_domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary parallel_domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ parallel_domain.set_flow_algorithm(domain_flow_algorithm) parallel_domain.set_name(domain_name) parallel_domain.set_datadir(domain_dir) parallel_domain.set_store(domain_store) parallel_domain.set_store_centroids(domain_store_centroids) parallel_domain.set_minimum_storable_height(domain_minimum_storable_height) parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height) parallel_domain.geo_reference = domain_georef parallel_domain.set_quantities_to_be_stored(domain_quantities_to_be_stored) parallel_domain.smooth = domain_smooth return parallel_domain
if __name__ == "__main__": test_points = [] if myid == 0: for i in range(samples): x = random.randrange(0, 1000) / 1000.0 * length y = random.randrange(0, 1000) / 1000.0 * width point = [x, y] test_points.append(point) for i in range(1, numprocs): pypar.send(test_points, i) else: test_points = pypar.receive(0) print("Test Points::") print(test_points) if myid == 0: control_data = run_simulation(parallel=False, test_points=test_points, verbose=True) for proc in range(1, numprocs): pypar.send(control_data, proc) else: control_data = pypar.receive(0)
def discharge_routine_explicit(self): from anuga.utilities import parallel_abstraction as pypar local_debug = False # If the structure has been closed, then no water gets through if self.height <= 0.0: if self.myid == self.master_proc: Q = 0.0 barrel_velocity = 0.0 outlet_culvert_depth = 0.0 self.case = "Structure is blocked" self.inflow = self.inlets[0] self.outflow = self.inlets[1] return Q, barrel_velocity, outlet_culvert_depth else: return None, None, None #Send attributes of both enquiry points to the master proc if self.myid == self.master_proc: if self.myid == self.enquiry_proc[0]: enq_total_energy0 = self.inlets[0].get_enquiry_total_energy() enq_stage0 = self.inlets[0].get_enquiry_stage() else: enq_total_energy0 = pypar.receive(self.enquiry_proc[0]) enq_stage0 = pypar.receive(self.enquiry_proc[0]) if self.myid == self.enquiry_proc[1]: enq_total_energy1 = self.inlets[1].get_enquiry_total_energy() enq_stage1 = self.inlets[1].get_enquiry_stage() else: enq_total_energy1 = pypar.receive(self.enquiry_proc[1]) enq_stage1 = pypar.receive(self.enquiry_proc[1]) else: if self.myid == self.enquiry_proc[0]: pypar.send(self.inlets[0].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[0].get_enquiry_stage(), self.master_proc) if self.myid == self.enquiry_proc[1]: pypar.send(self.inlets[1].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[1].get_enquiry_stage(), self.master_proc) # Determine the direction of the flow if self.myid == self.master_proc: # Variables required by anuga's structure operator which are not # used barrel_velocity = numpy.nan outlet_culvert_depth = numpy.nan flow_area = numpy.nan case = '' # 'Timescale' for smoothed discharge and energy ts = old_div( self.domain.timestep, max(self.domain.timestep, self.smoothing_timescale, 1.0e-30)) # Energy or stage as head if self.use_velocity_head: E0 = enq_total_energy0 E1 = enq_total_energy1 else: E0 = enq_stage0 E1 = enq_stage1 self.delta_total_energy = E0 - E1 self.driving_energy = max(E0, E1) # Compute 'smoothed' versions of key variables self.smooth_delta_total_energy += ts * ( self.delta_total_energy - self.smooth_delta_total_energy) if numpy.sign(self.smooth_delta_total_energy) != numpy.sign( self.delta_total_energy): self.smooth_delta_total_energy = 0. # Compute the 'tailwater' energy from the 'headwater' energy # and the smooth_delta_total_energy. Note if ts = 1 (no # smoothing), then the raw inlet energies are used if E0 >= E1: inlet0_energy = 1.0 * E0 inlet1_energy = inlet0_energy - self.smooth_delta_total_energy else: inlet1_energy = 1.0 * E1 inlet0_energy = inlet1_energy + self.smooth_delta_total_energy # Compute discharge Q = self.internal_boundary_function(inlet0_energy, inlet1_energy) self.smooth_Q = self.smooth_Q + ts * (Q - self.smooth_Q) if numpy.sign(self.smooth_Q) != numpy.sign(Q): # The flow direction of the 'instantaneous Q' based on the # 'smoothed delta_total_energy' is not the same as the # direction of smooth_Q. To prevent 'jumping around', let's # set Q to zero Q = 0. else: # Make Q positive (for anuga's structure operator) Q = min(abs(self.smooth_Q), abs(Q)) else: self.delta_total_energy = numpy.nan self.driving_energy = numpy.nan self.inflow_index = 0 self.outflow_index = 1 # master proc orders reversal if applicable if self.myid == self.master_proc: # Reverse the inflow and outflow direction? if self.smooth_Q < 0.: self.inflow_index = 1 self.outflow_index = 0 for i in self.procs: if i == self.master_proc: continue pypar.send(True, i) else: for i in self.procs: if i == self.master_proc: continue pypar.send(False, i) else: reverse = pypar.receive(self.master_proc) if reverse: self.inflow_index = 1 self.outflow_index = 0 # Master proc computes return values if self.myid == self.master_proc: return Q, barrel_velocity, outlet_culvert_depth else: return None, None, None
def plotCentroidError(domain, control_data, rthr=1E-7, athr=1E-12, quantity='stage', filename='centroid_error.png'): n_triangles = num.sum(domain.tri_full_flag) if size() > 1: # If parallel, translate control data to parallel indexing local_control_data = num.zeros(n_triangles) inv_tri_map = domain.get_inv_tri_map() for i in range(n_triangles): local_control_data[i] = control_data[inv_tri_map[(rank(), i)]] else: local_control_data = control_data # Evaluate absolute and relative difference between control and actual values stage = domain.get_quantity(quantity) actual_data = stage.centroid_values[:n_triangles] adiff = num.fabs((actual_data - local_control_data)) rdiff = adiff / num.fabs(local_control_data) # Compute masks for error (err_mask) and non-error (acc_mask) vertex indices based on thresholds vertices = domain.get_vertex_coordinates() err_mask = rdiff > rthr err_mask[adiff <= athr] = False err_mask = num.repeat(err_mask, 3) acc_mask = ~err_mask inv_tri_map = domain.get_inv_tri_map() # Plot error and non-error triangle if rank() == 0: fx = {} fy = {} gx = {} gy = {} fx[0] = vertices[acc_mask, 0] fy[0] = vertices[acc_mask, 1] gx[0] = vertices[err_mask, 0] gy[0] = vertices[err_mask, 1] # Receive vertex indices of non-error triangles (fx, fy) and error triangles (gx, gy) for i in range(1, size()): fx[i] = receive(i) fy[i] = receive(i) gx[i] = receive(i) gy[i] = receive(i) # Plot non-error triangles in green for i in range(0, size()): n = int(len(fx[i]) / 3) triang = num.array(range(0, 3 * n)) triang.shape = (n, 3) if len(fx[i]) > 0: plt.triplot(fx[i], fy[i], triang, 'g-') # Plot error triangles in blue for i in range(0, size()): n = int(len(gx[i]) / 3) triang = num.array(range(0, 3 * n)) triang.shape = (n, 3) if len(gx[i]) > 0: plt.triplot(gx[i], gy[i], triang, 'b--') # Save plot plt.savefig(filename) else: # Send error and non-error vertex indices to Proc 0 send(vertices[acc_mask, 0], 0) send(vertices[acc_mask, 1], 0) send(vertices[err_mask, 0], 0) send(vertices[err_mask, 1], 0)
def statistics(self): # WARNING: requires synchronization, must be called by all procs associated # with this inlet from anuga.utilities import parallel_abstraction as pypar message = '' tri_indices = {} if self.myid == self.master_proc: tri_indices[self.myid] = self.triangle_indices for proc in self.procs: if proc == self.master_proc: continue tri_indices[proc] = pypar.receive(proc) else: pypar.send(self.triangle_indices, self.master_proc) if self.myid == self.master_proc: message += '=====================================\n' message += 'Inlet\n' message += '=====================================\n' for proc in self.procs: message += '======> inlet triangle indices and centres and elevation at P%d\n' %(proc) message += '%s' % tri_indices[proc] message += '\n' message += '%s' % self.domain.get_centroid_coordinates()[tri_indices[proc]] message += '\n' elev = self.domain.quantities['elevation'].centroid_values[tri_indices[proc]] message += '%s' % elev message += '\n' try: elevation_difference = elev.max() - elev.min() except ValueError: elevation_difference = 0.0 if not num.allclose(elevation_difference, 0.): message += 'Elevation range of ' + str(elevation_difference) message += 'Warning: Non-constant inlet elevation can lead to well-balancing problems' try: # If the inlet does not have an enquiry point this will # fail gracefully message += '\n' message += 'Enquiry point:' message += '%s' % self.domain.get_centroid_coordinates()[self.enquiry_index] message += '\n' message += 'Enquiry Index:' message += '%s' % self.enquiry_index message += '\n' except: pass message += 'line\n' message += '%s' % self.line message += '\n' return message
def send_submesh(submesh, triangles_per_proc, p, verbose=True): from anuga.utilities import parallel_abstraction as pypar myid = pypar.rank() nprocs = pypar.size() if verbose: print('P%d: Sending submesh to P%d' % (myid, p)) # build and send the tagmap for the boundary conditions tagmap = {} counter = 1 for b in submesh["full_boundary"][p]: bkey = submesh["full_boundary"][p][b] if bkey not in tagmap: tagmap[bkey] = counter counter = counter + 1 for b in submesh["ghost_boundary"][p]: bkey = submesh["ghost_boundary"][p][b] if bkey not in tagmap: tagmap[bkey] = counter counter = counter + 1 # send boundary tags pypar.send(tagmap, p) # send the quantities key information pypar.send(list(submesh["full_quan"].keys()), p) # compress full_commun flat_full_commun = [] for c in submesh["full_commun"][p]: for i in range(len(submesh["full_commun"][p][c])): flat_full_commun.append([c, submesh["full_commun"][p][c][i]]) # send the array sizes so memory can be allocated setup_array = num.zeros((9, ), num.int) setup_array[0] = len(submesh["full_nodes"][p]) setup_array[1] = len(submesh["ghost_nodes"][p]) setup_array[2] = len(submesh["full_triangles"][p]) setup_array[3] = len(submesh["ghost_triangles"][p]) setup_array[4] = len(submesh["full_boundary"][p]) setup_array[5] = len(submesh["ghost_boundary"][p]) setup_array[6] = len(submesh["ghost_commun"][p]) setup_array[7] = len(flat_full_commun) setup_array[8] = len(submesh["full_quan"]) x = num.array(setup_array, num.int) pypar.send(x, p, bypass=True) # ghost layer width x = num.array(submesh["ghost_layer_width"][p], num.int) pypar.send(x, p, bypass=True) # send the number of triangles per processor x = num.array(triangles_per_proc, num.int) pypar.send(x, p, bypass=True) # send the nodes x = num.array(submesh["full_nodes"][p], num.float) pypar.send(x, p, bypass=True) x = num.array(submesh["ghost_nodes"][p], num.float) pypar.send(x, p, bypass=True) # send the triangles x = num.array(submesh["full_triangles"][p], num.int) pypar.send(x, p, bypass=True) # send ghost triangles x = num.array(submesh["ghost_triangles"][p], num.int) pypar.send(x, p, bypass=True) # send the boundary bc = [] for b in submesh["full_boundary"][p]: bc.append([b[0], b[1], tagmap[submesh["full_boundary"][p][b]]]) x = num.array(bc, num.int) pypar.send(x, p, bypass=True) bc = [] for b in submesh["ghost_boundary"][p]: bc.append([b[0], b[1], tagmap[submesh["ghost_boundary"][p][b]]]) x = num.array(bc, num.int) pypar.send(x, p, bypass=True) # send the communication pattern x = num.array(submesh["ghost_commun"][p], num.int) pypar.send(x, p, bypass=True) x = num.array(flat_full_commun, num.int) pypar.send(x, p, bypass=True) # send the quantities for k in submesh["full_quan"]: x = num.array(submesh["full_quan"][k][p], num.float) pypar.send(x, p, bypass=True) for k in submesh["ghost_quan"]: x = num.array(submesh["ghost_quan"][k][p], num.float) pypar.send(x, p, bypass=True)
def dump_triangulation(self, filename="domain.png"): # Get vertex coordinates, partition full and ghost triangles based on self.tri_full_flag try: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import matplotlib.tri as tri except: print "Couldn't import module from matplotlib, probably you need to update matplotlib" raise vertices = self.get_vertex_coordinates() full_mask = num.repeat(self.tri_full_flag == 1, 3) ghost_mask = num.repeat(self.tri_full_flag == 0, 3) myid = pypar.rank() numprocs = pypar.size() if myid == 0: fig = plt.figure() fx = {} fy = {} gx = {} gy = {} # Proc 0 gathers full and ghost nodes from self and other processors fx[0] = vertices[full_mask,0] fy[0] = vertices[full_mask,1] gx[0] = vertices[ghost_mask,0] gy[0] = vertices[ghost_mask,1] for i in range(1,numprocs): fx[i] = pypar.receive(i) fy[i] = pypar.receive(i) gx[i] = pypar.receive(i) gy[i] = pypar.receive(i) # Plot full triangles for i in range(0, numprocs): n = int(len(fx[i])/3) triang = num.array(range(0,3*n)) triang.shape = (n, 3) plt.triplot(fx[i], fy[i], triang, 'g-', linewidth = 0.5) # Plot ghost triangles for i in range(0, numprocs): n = int(len(gx[i])/3) if n > 0: triang = num.array(range(0,3*n)) triang.shape = (n, 3) plt.triplot(gx[i], gy[i], triang, 'b--', linewidth = 0.5) # Save triangulation to location pointed by filename plt.savefig(filename, dpi=600) else: # Proc 1..numprocs send full and ghost triangles to Proc 0 pypar.send(vertices[full_mask,0], 0) pypar.send(vertices[full_mask,1], 0) pypar.send(vertices[ghost_mask,0], 0) pypar.send(vertices[ghost_mask,1], 0)
def run_simulation(parallel=False): domain = create_domain_from_file(mesh_filename) domain.set_quantity('stage', Set_Stage(756000.0, 756500.0, 2.0)) #-------------------------------------------------------------------------- # Create parallel domain if requested #-------------------------------------------------------------------------- if parallel: if myid == 0 and verbose: print('DISTRIBUTING PARALLEL DOMAIN') domain = distribute(domain) #------------------------------------------------------------------------------ # Setup boundary conditions # This must currently happen *after* domain has been distributed #------------------------------------------------------------------------------ domain.store = False Br = Reflective_boundary(domain) # Solid reflective wall domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br}) #------------------------------------------------------------------------------ # Setup diagnostic arrays #------------------------------------------------------------------------------ l1list = [] l2list = [] linflist = [] l1norm = num.zeros(3, num.float) l2norm = num.zeros(3, num.float) linfnorm = num.zeros(3, num.float) recv_norm = num.zeros(3, num.float) #------------------------------------------------------------------------------ # Evolution #------------------------------------------------------------------------------ if parallel: if myid == 0 and verbose: print('PARALLEL EVOLVE') else: if verbose: print('SEQUENTIAL EVOLVE') for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime): edges = domain.quantities[quantity].edge_values.take(num.flatnonzero(domain.tri_full_flag),axis=0) l1norm[0] = l1_norm(edges[:,0]) l1norm[1] = l1_norm(edges[:,1]) l1norm[2] = l1_norm(edges[:,2]) l2norm[0] = l2_norm(edges[:,0]) l2norm[1] = l2_norm(edges[:,1]) l2norm[2] = l2_norm(edges[:,2]) linfnorm[0] = linf_norm(edges[:,0]) linfnorm[1] = linf_norm(edges[:,1]) linfnorm[2] = linf_norm(edges[:,2]) if parallel: l2norm[0] = pow(l2norm[0], 2) l2norm[1] = pow(l2norm[1], 2) l2norm[2] = pow(l2norm[2], 2) if myid == 0: #domain.write_time() #print edges[:,1] for p in range(1, numprocs): recv_norm = pypar.receive(p) l1norm += recv_norm recv_norm = pypar.receive(p) l2norm += recv_norm recv_norm = pypar.receive(p) linfnorm[0] = max(linfnorm[0], recv_norm[0]) linfnorm[1] = max(linfnorm[1], recv_norm[1]) linfnorm[2] = max(linfnorm[2], recv_norm[2]) l2norm[0] = pow(l2norm[0], 0.5) l2norm[1] = pow(l2norm[1], 0.5) l2norm[2] = pow(l2norm[2], 0.5) l1list.append(l1norm) l2list.append(l2norm) linflist.append(linfnorm) else: pypar.send(l1norm, 0) pypar.send(l2norm, 0) pypar.send(linfnorm, 0) else: #domain.write_time() l1list.append(l1norm) l2list.append(l2norm) linflist.append(linfnorm) return (l1list, l2list, linflist)
def discharge_routine(self): """ Get info from inlets and then call sequential function """ from anuga.utilities import parallel_abstraction as pypar local_debug = False # If the cuvert has been closed, then no water gets through if self.culvert_height <= 0.0: Q = 0.0 barrel_velocity = 0.0 outlet_culvert_depth = 0.0 self.case = "Culvert blocked" self.inflow = self.inlets[0] self.outflow = self.inlets[1] return Q, barrel_velocity, outlet_culvert_depth #Send attributes of both enquiry points to the master proc if self.myid == self.master_proc: if self.myid == self.enquiry_proc[0]: enq_total_energy0 = self.inlets[0].get_enquiry_total_energy() enq_stage0 = self.inlets[0].get_enquiry_stage() else: enq_total_energy0 = pypar.receive(self.enquiry_proc[0]) enq_stage0 = pypar.receive(self.enquiry_proc[0]) if self.myid == self.enquiry_proc[1]: enq_total_energy1 = self.inlets[1].get_enquiry_total_energy() enq_stage1 = self.inlets[1].get_enquiry_stage() else: enq_total_energy1 = pypar.receive(self.enquiry_proc[1]) enq_stage1 = pypar.receive(self.enquiry_proc[1]) else: if self.myid == self.enquiry_proc[0]: pypar.send(self.inlets[0].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[0].get_enquiry_stage(), self.master_proc) if self.myid == self.enquiry_proc[1]: pypar.send(self.inlets[1].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[1].get_enquiry_stage(), self.master_proc) # Determine the direction of the flow if self.myid == self.master_proc: if self.use_velocity_head: self.delta_total_energy = enq_total_energy0 - enq_total_energy1 else: self.delta_total_energy = enq_stage0 - enq_stage1 self.inflow_index = 0 self.outflow_index = 1 # master proc orders reversal if applicable if self.myid == self.master_proc: forward_Euler_smooth = True self.smooth_delta_total_energy, ts = total_energy( self.smooth_delta_total_energy, self.delta_total_energy, self.domain.timestep, self.smoothing_timescale, forward_Euler_smooth) # Reverse the inflow and outflow direction? if self.smooth_delta_total_energy < 0: self.inflow_index = 1 self.outflow_index = 0 #self.delta_total_energy = -self.delta_total_energy self.delta_total_energy = -self.smooth_delta_total_energy for i in self.procs: if i == self.master_proc: continue pypar.send(True, i) else: self.delta_total_energy = self.smooth_delta_total_energy for i in self.procs: if i == self.master_proc: continue pypar.send(False, i) #print "ZZZZ: Delta total energy = %f" %(self.delta_total_energy) else: reverse = pypar.receive(self.master_proc) if reverse: self.inflow_index = 1 self.outflow_index = 0 # Get attribute from inflow enquiry point if self.myid == self.master_proc: if self.myid == self.enquiry_proc[self.inflow_index]: inflow_enq_depth = self.inlets[ self.inflow_index].get_enquiry_depth() inflow_enq_specific_energy = self.inlets[ self.inflow_index].get_enquiry_specific_energy() else: inflow_enq_depth = pypar.receive( self.enquiry_proc[self.inflow_index]) inflow_enq_specific_energy = pypar.receive( self.enquiry_proc[self.inflow_index]) else: if self.myid == self.enquiry_proc[self.inflow_index]: pypar.send(self.inlets[self.inflow_index].get_enquiry_depth(), self.master_proc) pypar.send( self.inlets[ self.inflow_index].get_enquiry_specific_energy(), self.master_proc) # Get attribute from outflow enquiry point if self.myid == self.master_proc: if self.myid == self.enquiry_proc[self.outflow_index]: outflow_enq_depth = self.inlets[ self.outflow_index].get_enquiry_depth() else: outflow_enq_depth = pypar.receive( self.enquiry_proc[self.outflow_index]) #print('receive',outflow_enq_depth) #print "ZZZZZ: outflow_enq_depth = %f" %(outflow_enq_depth) else: if self.myid == self.enquiry_proc[self.outflow_index]: #print('send',self.inlets[self.outflow_index].get_enquiry_depth()) pypar.send(self.inlets[self.outflow_index].get_enquiry_depth(), self.master_proc) # Master proc computes return values if self.myid == self.master_proc: #inflow_enq_specific_energy if inflow_enq_depth > 0.01: #this value was 0.01: if local_debug: anuga.log.critical('Specific E & Deltat Tot E = %s, %s' % (str(inflow_enq_specific_energy), str(self.delta_total_energy))) anuga.log.critical('culvert type = %s' % str(culvert_type)) # Water has risen above inlet msg = 'Specific energy at inlet is negative' assert inflow_enq_specific_energy >= 0.0, msg if self.use_velocity_head: self.driving_energy = inflow_enq_specific_energy else: self.driving_energy = inflow_enq_depth Q, barrel_velocity, outlet_culvert_depth, flow_area, case = \ boyd_box_function(depth =self.culvert_height, width =self.culvert_width, flow_width =self.culvert_width, length =self.culvert_length, blockage =self.culvert_blockage, barrels =self.culvert_barrels, driving_energy =self.driving_energy, delta_total_energy =self.delta_total_energy, outlet_enquiry_depth=outflow_enq_depth, sum_loss =self.sum_loss, manning =self.manning) self.smooth_Q, Q, barrel_velocity = smooth_discharge( self.smooth_delta_total_energy, self.smooth_Q, Q, flow_area, ts, forward_Euler_smooth) else: # self.inflow.get_enquiry_depth() < 0.01: Q = barrel_velocity = outlet_culvert_depth = 0.0 case = 'Inlet dry' self.case = case # Temporary flow limit if barrel_velocity > self.max_velocity: barrel_velocity = self.max_velocity Q = flow_area * barrel_velocity return Q, barrel_velocity, outlet_culvert_depth else: return None, None, None
def old_distribute(domain, verbose=False, debug=False, parameters = None): """ Distribute the domain to all processes parameters values """ if debug: verbose = True barrier() # FIXME: Dummy assignment (until boundaries are refactored to # be independent of domains until they are applied) if myid == 0: bdmap = {} for tag in domain.get_boundary_tags(): bdmap[tag] = None domain.set_boundary(bdmap) if not pypar_available or numprocs == 1 : return domain # Bypass # For some obscure reason this communication must happen prior to # the more complex mesh distribution - Oh Well! if myid == 0: domain_name = domain.get_name() domain_dir = domain.get_datadir() domain_store = domain.get_store() domain_store_centroids = domain.get_store_centroids() domain_smooth = domain.smooth domain_reduction = domain.reduction domain_minimum_storable_height = domain.minimum_storable_height domain_flow_algorithm = domain.get_flow_algorithm() domain_minimum_allowed_height = domain.get_minimum_allowed_height() georef = domain.geo_reference number_of_global_triangles = domain.number_of_triangles number_of_global_nodes = domain.number_of_nodes # FIXME - what other attributes need to be transferred? for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send((domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, number_of_global_nodes), p) else: if verbose: print 'P%d: Receiving domain attributes' %(myid) domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, \ number_of_global_nodes = receive(0) # Distribute boundary conditions # FIXME: This cannot handle e.g. Time_boundaries due to # difficulties pickling functions if myid == 0: boundary_map = domain.boundary_map for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send(boundary_map, p) else: if verbose: print 'P%d: Receiving boundary map' %(myid) boundary_map = receive(0) send_s2p = False if myid == 0: # Partition and distribute mesh. # Structures returned is in the # correct form for the ANUGA data structure points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles,\ s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width =\ distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] if debug: print 'P%d' %myid print 'tri_map ',tri_map print 'node_map',node_map print 'tri_l2g', tri_l2g print 'node_l2g', node_l2g print 's2p_map', s2p_map print 'p2s_map', p2s_map def protocol(x): vanilla=False import pypar control_info, x = pypar.create_control_info(x, vanilla, return_object=True) print 'protocol', control_info[0] # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs if send_s2p : n = len(s2p_map) s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(),num.int), (n,1) ) s2p_map_values_flat = num.array(s2p_map.values(),num.int) s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1 ) n = len(p2s_map) p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(),num.int), (n,2) ) p2s_map_values_flat = num.reshape(num.array(p2s_map.values(),num.int) , (n,1)) p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1 ) for p in range(1, numprocs): # FIXME SR: Creates cPickle dump send(s2p_map_flat, p) # FIXME SR: Creates cPickle dump #print p2s_map send(p2s_map_flat, p) else: if verbose: print 'Not sending s2p_map and p2s_map' s2p_map = None p2s_map = None if verbose: print 'Communication done' else: # Read in the mesh partition that belongs to this # processor if verbose: print 'P%d: Receiving submeshes' %(myid) points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles, \ tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\ rec_submesh(0, verbose) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] # Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping if send_s2p : s2p_map_flat = receive(0) s2p_map = dict.fromkeys(s2p_map_flat[:,0], s2p_map_flat[:,1:2]) p2s_map_flat = receive(0) p2s_map_keys = [tuple(x) for x in p2s_map_flat[:,0:1]] p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:,2]) else: s2p_map = None p2s_map = None #------------------------------------------------------------------------ # Build the domain for this processor using partion structures #------------------------------------------------------------------------ if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (myid, number_of_full_nodes, number_of_full_triangles) domain = Parallel_domain(points, vertices, boundary, full_send_dict=full_send_dict, ghost_recv_dict=ghost_recv_dict, number_of_full_nodes=number_of_full_nodes, number_of_full_triangles=number_of_full_triangles, geo_reference=georef, number_of_global_triangles = number_of_global_triangles, number_of_global_nodes = number_of_global_nodes, s2p_map = s2p_map, p2s_map = p2s_map, ## jj added this tri_l2g = tri_l2g, ## SR added this node_l2g = node_l2g, ghost_layer_width = ghost_layer_width) #------------------------------------------------------------------------ # Transfer initial conditions to each subdomain #------------------------------------------------------------------------ for q in quantities: domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ domain.set_name(domain_name) domain.set_datadir(domain_dir) domain.set_store(domain_store) domain.set_store_centroids(domain_store_centroids) domain.set_store_vertices_smoothly(domain_smooth,domain_reduction) domain.set_minimum_storable_height(domain_minimum_storable_height) domain.set_minimum_allowed_height(domain_minimum_allowed_height) domain.set_flow_algorithm(domain_flow_algorithm) domain.geo_reference = georef #------------------------------------------------------------------------ # Return parallel domain to all nodes #------------------------------------------------------------------------ return domain
def discharge_routine(self): """ Get info from inlets and then call sequential function """ from anuga.utilities import parallel_abstraction as pypar local_debug = False #Send attributes of both enquiry points to the master proc if self.myid == self.master_proc: if self.myid == self.enquiry_proc[0]: enq_total_energy0 = self.inlets[0].get_enquiry_total_energy() enq_stage0 = self.inlets[0].get_enquiry_stage() else: enq_total_energy0 = pypar.receive(self.enquiry_proc[0]) enq_stage0 = pypar.receive(self.enquiry_proc[0]) if self.myid == self.enquiry_proc[1]: enq_total_energy1 = self.inlets[1].get_enquiry_total_energy() enq_stage1 = self.inlets[1].get_enquiry_stage() else: enq_total_energy1 = pypar.receive(self.enquiry_proc[1]) enq_stage1 = pypar.receive(self.enquiry_proc[1]) else: if self.myid == self.enquiry_proc[0]: pypar.send(self.inlets[0].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[0].get_enquiry_stage(), self.master_proc) if self.myid == self.enquiry_proc[1]: pypar.send(self.inlets[1].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[1].get_enquiry_stage(), self.master_proc) # Determine the direction of the flow if self.myid == self.master_proc: if self.use_velocity_head: self.delta_total_energy = enq_total_energy0 - enq_total_energy1 else: self.delta_total_energy = enq_stage0 - enq_stage1 self.inflow_index = 0 self.outflow_index = 1 # master proc orders reversal if applicable if self.myid == self.master_proc: # May/June 2014 -- change the driving forces gradually, with forward euler timestepping # forward_Euler_smooth = True if (forward_Euler_smooth): # To avoid 'overshoot' we ensure ts<1. if (self.domain.timestep > 0.): ts = old_div( self.domain.timestep, max(self.domain.timestep, self.smoothing_timescale, 1.0e-06)) else: # This case is included in the serial version, which ensures the unit tests pass # even when domain.timestep=0.0. # Note though the discontinuous behaviour as domain.timestep-->0. from above ts = 1.0 self.smooth_delta_total_energy=self.smooth_delta_total_energy+\ ts*(self.delta_total_energy-self.smooth_delta_total_energy) else: # Use backward euler -- the 'sensible' ts limitation is different in this case # ts --> Inf is reasonable and corresponds to the 'nosmoothing' case ts = old_div(self.domain.timestep, max(self.smoothing_timescale, 1.0e-06)) self.smooth_delta_total_energy = old_div( (self.smooth_delta_total_energy + ts * (self.delta_total_energy)), (1. + ts)) # Reverse the inflow and outflow direction? if self.smooth_delta_total_energy < 0: self.inflow_index = 1 self.outflow_index = 0 #self.delta_total_energy = -self.delta_total_energy self.delta_total_energy = -self.smooth_delta_total_energy for i in self.procs: if i == self.master_proc: continue pypar.send(True, i) else: self.delta_total_energy = self.smooth_delta_total_energy for i in self.procs: if i == self.master_proc: continue pypar.send(False, i) #print "ZZZZ: Delta total energy = %f" %(self.delta_total_energy) else: reverse = pypar.receive(self.master_proc) if reverse: self.inflow_index = 1 self.outflow_index = 0 # Get attribute from inflow enquiry point if self.myid == self.master_proc: if self.myid == self.enquiry_proc[self.inflow_index]: inflow_enq_depth = self.inlets[ self.inflow_index].get_enquiry_depth() inflow_enq_specific_energy = self.inlets[ self.inflow_index].get_enquiry_specific_energy() else: inflow_enq_depth = pypar.receive( self.enquiry_proc[self.inflow_index]) inflow_enq_specific_energy = pypar.receive( self.enquiry_proc[self.inflow_index]) else: if self.myid == self.enquiry_proc[self.inflow_index]: pypar.send(self.inlets[self.inflow_index].get_enquiry_depth(), self.master_proc) pypar.send( self.inlets[ self.inflow_index].get_enquiry_specific_energy(), self.master_proc) # Get attribute from outflow enquiry point if self.myid == self.master_proc: if self.myid == self.enquiry_proc[self.outflow_index]: outflow_enq_depth = self.inlets[ self.outflow_index].get_enquiry_depth() else: outflow_enq_depth = pypar.receive( self.enquiry_proc[self.outflow_index]) #print "ZZZZZ: outflow_enq_depth = %f" %(outflow_enq_depth) else: if self.myid == self.enquiry_proc[self.outflow_index]: pypar.send(self.inlets[self.outflow_index].get_enquiry_depth(), self.master_proc) # Master proc computes return values if self.myid == self.master_proc: #inflow_enq_specific_energy if inflow_enq_depth > 0.01: #this value was 0.01: if local_debug: anuga.log.critical('Specific E & Deltat Tot E = %s, %s' % (str(inflow_enq_specific_energy), str(self.delta_total_energy))) anuga.log.critical('culvert type = %s' % str(self.structure_type)) # Water has risen above inlet msg = 'Specific energy at inlet is negative' assert inflow_enq_specific_energy >= 0.0, msg if self.use_velocity_head: self.driving_energy = inflow_enq_specific_energy else: self.driving_energy = inflow_enq_depth Q, barrel_velocity, outlet_culvert_depth, flow_area, case = \ boyd_pipe_function(depth =inflow_enq_depth, diameter =self.culvert_diameter, length =self.culvert_length, blockage =self.culvert_blockage, barrels =self.culvert_barrels, driving_energy =self.driving_energy, delta_total_energy =self.delta_total_energy, outlet_enquiry_depth=outflow_enq_depth, sum_loss =self.sum_loss, manning =self.manning) ################################################ # Smooth discharge. This can reduce oscillations # # NOTE: The sign of smooth_Q assumes that # self.inflow_index=0 and self.outflow_index=1 # , whereas the sign of Q is always positive Qsign = (self.outflow_index - self.inflow_index ) # To adjust sign of Q if (forward_Euler_smooth): self.smooth_Q = self.smooth_Q + ts * (Q * Qsign - self.smooth_Q) else: # Try implicit euler method self.smooth_Q = old_div((self.smooth_Q + ts * (Q * Qsign)), (1. + ts)) if numpy.sign(self.smooth_Q) != Qsign: # The flow direction of the 'instantaneous Q' based on the # 'smoothed delta_total_energy' is not the same as the # direction of smooth_Q. To prevent 'jumping around', let's # set Q to zero Q = 0. else: Q = min(abs(self.smooth_Q), Q) #abs(self.smooth_Q) barrel_velocity = old_div(Q, flow_area) # END CODE BLOCK for DEPTH > Required depth for CULVERT Flow else: # self.inflow.get_enquiry_depth() < 0.01: Q = barrel_velocity = outlet_culvert_depth = 0.0 case = 'Inlet dry' self.case = case # Temporary flow limit if barrel_velocity > self.max_velocity: barrel_velocity = self.max_velocity Q = flow_area * barrel_velocity return Q, barrel_velocity, outlet_culvert_depth else: return None, None, None
def communicate_ghosts_blocking(domain): # We must send the information from the full cells and # receive the information for the ghost cells # We have a dictionary of lists with ghosts expecting updates from # the separate processors import numpy as num import time t0 = time.time() # update of non-local ghost cells for iproc in range(domain.numproc): if iproc == domain.processor: #Send data from iproc processor to other processors for send_proc in domain.full_send_dict: if send_proc != iproc: Idf = domain.full_send_dict[send_proc][0] Xout = domain.full_send_dict[send_proc][2] for i, q in enumerate(domain.conserved_quantities): #print 'Send',i,q Q_cv = domain.quantities[q].centroid_values Xout[:, i] = num.take(Q_cv, Idf) pypar.send(Xout, int(send_proc), use_buffer=True, bypass=True) else: #Receive data from the iproc processor if iproc in domain.ghost_recv_dict: Idg = domain.ghost_recv_dict[iproc][0] X = domain.ghost_recv_dict[iproc][2] X = pypar.receive(int(iproc), buffer=X, bypass=True) for i, q in enumerate(domain.conserved_quantities): #print 'Receive',i,q Q_cv = domain.quantities[q].centroid_values num.put(Q_cv, Idg, X[:, i]) #local update of ghost cells iproc = domain.processor if iproc in domain.full_send_dict: # LINDA: # now store full as local id, global id, value Idf = domain.full_send_dict[iproc][0] # LINDA: # now store ghost as local id, global id, value Idg = domain.ghost_recv_dict[iproc][0] for i, q in enumerate(domain.conserved_quantities): #print 'LOCAL SEND RECEIVE',i,q Q_cv = domain.quantities[q].centroid_values num.put(Q_cv, Idg, num.take(Q_cv, Idf)) domain.communication_time += time.time() - t0
def communicate_ghosts_blocking(domain): # We must send the information from the full cells and # receive the information for the ghost cells # We have a dictionary of lists with ghosts expecting updates from # the separate processors import numpy as num import time t0 = time.time() # update of non-local ghost cells for iproc in range(domain.numproc): if iproc == domain.processor: #Send data from iproc processor to other processors for send_proc in domain.full_send_dict: if send_proc != iproc: Idf = domain.full_send_dict[send_proc][0] Xout = domain.full_send_dict[send_proc][2] for i, q in enumerate(domain.conserved_quantities): #print 'Send',i,q Q_cv = domain.quantities[q].centroid_values Xout[:,i] = num.take(Q_cv, Idf) pypar.send(Xout, int(send_proc), use_buffer=True, bypass=True) else: #Receive data from the iproc processor if domain.ghost_recv_dict.has_key(iproc): Idg = domain.ghost_recv_dict[iproc][0] X = domain.ghost_recv_dict[iproc][2] X = pypar.receive(int(iproc), buffer=X, bypass=True) for i, q in enumerate(domain.conserved_quantities): #print 'Receive',i,q Q_cv = domain.quantities[q].centroid_values num.put(Q_cv, Idg, X[:,i]) #local update of ghost cells iproc = domain.processor if domain.full_send_dict.has_key(iproc): # LINDA: # now store full as local id, global id, value Idf = domain.full_send_dict[iproc][0] # LINDA: # now store ghost as local id, global id, value Idg = domain.ghost_recv_dict[iproc][0] for i, q in enumerate(domain.conserved_quantities): #print 'LOCAL SEND RECEIVE',i,q Q_cv = domain.quantities[q].centroid_values num.put(Q_cv, Idg, num.take(Q_cv, Idf)) domain.communication_time += time.time()-t0
def statistics(self): # Warning: requires synchronization, must be called by all procs associated # with this structure message = ' ' if self.myid == self.master_proc: message = '===============================================\n' message += 'Parallel Structure Operator: %s\n' % self.label message += '===============================================\n' message += 'Structure Type: %s\n' % self.structure_type message += 'Description\n' message += '%s' % self.description message += '\n' #add the culvert dimensions, blockage factor here if self.structure_type == 'boyd_pipe': message += 'Culvert Diameter: %s\n' % self.diameter message += 'Culvert Blockage: %s\n' % self.blockage message += 'No. of barrels: %s\n' % self.barrels elif self.structure_type == 'boyd_box': message += 'Culvert Height: %s\n' % self.height message += 'Culvert Width: %s\n' % self.width message += 'Culvert Blockage: %s\n' % self.blockage message += 'No. of barrels: %s\n' % self.barrels else: message += 'Culvert Height : %s\n' % self.height message += 'Culvert Width : %s\n' % self.width message += 'Batter Slope 1 : %s\n' % self.z1 message += 'Batter Slope 2 : %s\n' % self.z2 message += 'Culvert Blockage: %s\n' % self.blockage message += 'No. of barrels: %s\n' % self.barrels #print "Structure Myids ",self.myid, self.label for i, inlet in enumerate(self.inlets): if self.myid == self.master_proc: message += '-------------------------------------\n' message += 'Inlet %i\n' % (i) message += '-------------------------------------\n' #print "*****",inlet, i,self.myid if inlet is not None: stats = inlet.statistics() if self.myid == self.master_proc: if self.myid != self.inlet_master_proc[i]: stats = pypar.receive(self.inlet_master_proc[i]) elif self.myid == self.inlet_master_proc[i]: pypar.send(stats, self.master_proc) if self.myid == self.master_proc: message += stats if self.myid == self.master_proc: message += '=====================================\n' return message
def distribute(domain, verbose=False, debug=False, parameters = None): """ Distribute the domain to all processes parameters allows user to change size of ghost layer """ if not pypar_available or numprocs == 1 : return domain # Bypass if myid == 0: from sequential_distribute import Sequential_distribute partition = Sequential_distribute(domain, verbose, debug, parameters) partition.distribute(numprocs) kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = partition.extract_submesh(0) for p in range(1, numprocs): tostore = partition.extract_submesh(p) send(tostore,p) else: kwargs, points, vertices, boundary, quantities, boundary_map, \ domain_name, domain_dir, domain_store, domain_store_centroids, \ domain_minimum_storable_height, domain_minimum_allowed_height, \ domain_flow_algorithm, domain_georef, \ domain_quantities_to_be_stored, domain_smooth \ = receive(0) #--------------------------------------------------------------------------- # Now Create parallel domain #--------------------------------------------------------------------------- parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs) #------------------------------------------------------------------------ # Copy in quantity data #------------------------------------------------------------------------ for q in quantities: try: parallel_domain.set_quantity(q, quantities[q]) except KeyError: #print 'Try to create quantity %s'% q from anuga import Quantity Q = Quantity(parallel_domain, name=q, register=True) parallel_domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary parallel_domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ parallel_domain.set_flow_algorithm(domain_flow_algorithm) parallel_domain.set_name(domain_name) parallel_domain.set_datadir(domain_dir) parallel_domain.set_store(domain_store) parallel_domain.set_store_centroids(domain_store_centroids) parallel_domain.set_minimum_storable_height(domain_minimum_storable_height) parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height) parallel_domain.geo_reference = domain_georef parallel_domain.set_quantities_to_be_stored(domain_quantities_to_be_stored) parallel_domain.smooth = domain_smooth return parallel_domain
def __call__(self): timestep = self.domain.get_timestep() Q, barrel_speed, outlet_depth = self.discharge_routine() # Get attributes of Inflow inlet, all procs associated with inlet must call if self.myid in self.inlet_procs[self.inflow_index]: old_inflow_depth = self.inlets[ self.inflow_index].get_global_average_depth() old_inflow_stage = self.inlets[ self.inflow_index].get_global_average_stage() old_inflow_xmom = self.inlets[ self.inflow_index].get_global_average_xmom() old_inflow_ymom = self.inlets[ self.inflow_index].get_global_average_ymom() inflow_area = self.inlets[self.inflow_index].get_global_area() # Master proc of inflow inlet sends attributes to master proc of structure if self.myid == self.master_proc: if self.myid != self.inlet_master_proc[self.inflow_index]: old_inflow_depth = pypar.receive( self.inlet_master_proc[self.inflow_index]) old_inflow_stage = pypar.receive( self.inlet_master_proc[self.inflow_index]) old_inflow_xmom = pypar.receive( self.inlet_master_proc[self.inflow_index]) old_inflow_ymom = pypar.receive( self.inlet_master_proc[self.inflow_index]) inflow_area = pypar.receive( self.inlet_master_proc[self.inflow_index]) elif self.myid == self.inlet_master_proc[self.inflow_index]: pypar.send(old_inflow_depth, self.master_proc) pypar.send(old_inflow_stage, self.master_proc) pypar.send(old_inflow_xmom, self.master_proc) pypar.send(old_inflow_ymom, self.master_proc) pypar.send(inflow_area, self.master_proc) # Implement the update of flow over a timestep by # using a semi-implict update. This ensures that # the update does not create a negative depth # Master proc of structure only if self.myid == self.master_proc: if old_inflow_depth > 0.0: dt_Q_on_d = old_div(timestep * Q, old_inflow_depth) else: dt_Q_on_d = 0.0 # Check whether we should use the wet-dry Q adjustment (where Q is # multiplied by new_inflow_depth/old_inflow_depth) always_use_Q_wetdry_adjustment = self.always_use_Q_wetdry_adjustment # Always use it if we are near wet-dry use_Q_wetdry_adjustment = ((always_use_Q_wetdry_adjustment) |\ (old_inflow_depth*inflow_area <= Q*timestep)) factor = 1.0 / (1.0 + old_div(dt_Q_on_d, inflow_area)) if use_Q_wetdry_adjustment: new_inflow_depth = old_inflow_depth * factor if old_inflow_depth > 0.: timestep_star = old_div(timestep * new_inflow_depth, old_inflow_depth) else: timestep_star = 0. else: new_inflow_depth = old_inflow_depth - old_div( timestep * Q, inflow_area) timestep_star = timestep #new_inflow_xmom = old_inflow_xmom*factor #new_inflow_ymom = old_inflow_ymom*factor if (self.use_old_momentum_method): # This method is here for consistency with the old version of the # routine new_inflow_xmom = old_inflow_xmom * factor new_inflow_ymom = old_inflow_ymom * factor else: # For the momentum balance, note that Q also transports the velocity, # which has an average value of new_inflow_mom/depth (or old_inflow_mom/depth). # # new_inflow_xmom*inflow_area = # old_inflow_xmom*inflow_area - # timestep*Q*(new_inflow_xmom/old_inflow_depth) # and: # new_inflow_ymom*inflow_area = # old_inflow_ymom*inflow_area - # timestep*Q*(new_inflow_ymom/old_inflow_depth) # # The choice of new_inflow_mom in the final term might be # replaced with old_inflow_mom. # # The units balance: (m^2/s)*(m^2) = (m^2/s)*(m^2) - s*(m^3/s)*(m^2/s)*(m^(-1)) # if old_inflow_depth > 0.: if use_Q_wetdry_adjustment: factor2 = 1.0 / ( 1.0 + old_div(dt_Q_on_d * new_inflow_depth, (old_inflow_depth * inflow_area))) else: factor2 = 1.0 / ( 1.0 + old_div(timestep * Q, (old_inflow_depth * inflow_area))) else: factor2 = 0. new_inflow_xmom = old_inflow_xmom * factor2 new_inflow_ymom = old_inflow_ymom * factor2 # Master proc of structure sends new inflow attributes to all inflow inlet processors if self.myid == self.master_proc: for i in self.inlet_procs[self.inflow_index]: if i == self.master_proc: continue pypar.send(new_inflow_depth, i) pypar.send(new_inflow_xmom, i) pypar.send(new_inflow_ymom, i) elif self.myid in self.inlet_procs[self.inflow_index]: new_inflow_depth = pypar.receive(self.master_proc) new_inflow_xmom = pypar.receive(self.master_proc) new_inflow_ymom = pypar.receive(self.master_proc) # Inflow inlet procs sets new attributes if self.myid in self.inlet_procs[self.inflow_index]: self.inlets[self.inflow_index].set_depths(new_inflow_depth) self.inlets[self.inflow_index].set_xmoms(new_inflow_xmom) self.inlets[self.inflow_index].set_ymoms(new_inflow_ymom) # Get outflow inlet attributes, all processors associated with outflow inlet must call if self.myid in self.inlet_procs[self.outflow_index]: outflow_area = self.inlets[self.outflow_index].get_global_area() outflow_average_depth = self.inlets[ self.outflow_index].get_global_average_depth() outflow_outward_culvert_vector = self.inlets[ self.outflow_index].outward_culvert_vector outflow_average_xmom = self.inlets[ self.outflow_index].get_global_average_xmom() outflow_average_ymom = self.inlets[ self.outflow_index].get_global_average_ymom() # Master proc of outflow inlet sends attribute to master proc of structure if self.myid == self.master_proc: if self.myid != self.inlet_master_proc[self.outflow_index]: outflow_area = pypar.receive( self.inlet_master_proc[self.outflow_index]) outflow_average_depth = pypar.receive( self.inlet_master_proc[self.outflow_index]) outflow_outward_culvert_vector = pypar.receive( self.inlet_master_proc[self.outflow_index]) outflow_average_xmom = pypar.receive( self.inlet_master_proc[self.outflow_index]) outflow_average_ymom = pypar.receive( self.inlet_master_proc[self.outflow_index]) elif self.myid == self.inlet_master_proc[self.outflow_index]: pypar.send(outflow_area, self.master_proc) pypar.send(outflow_average_depth, self.master_proc) pypar.send(outflow_outward_culvert_vector, self.master_proc) pypar.send(outflow_average_xmom, self.master_proc) pypar.send(outflow_average_ymom, self.master_proc) # Master proc of structure computes new outflow attributes if self.myid == self.master_proc: loss = (old_inflow_depth - new_inflow_depth) * inflow_area xmom_loss = (old_inflow_xmom - new_inflow_xmom) * inflow_area ymom_loss = (old_inflow_ymom - new_inflow_ymom) * inflow_area # set outflow outflow_extra_depth = old_div(Q * timestep_star, outflow_area) outflow_direction = -outflow_outward_culvert_vector #outflow_extra_momentum = outflow_extra_depth*barrel_speed*outflow_direction gain = outflow_extra_depth * outflow_area # Update Stats self.discharge = old_div( Q * timestep_star, timestep ) #outflow_extra_depth*self.outflow.get_area()/timestep self.discharge_abs_timemean += old_div(Q * timestep_star, self.domain.yieldstep) self.velocity = barrel_speed #self.discharge/outlet_depth/self.width new_outflow_depth = outflow_average_depth + outflow_extra_depth self.outlet_depth = new_outflow_depth #if self.use_momentum_jet : # # FIXME (SR) Review momentum to account for possible hydraulic jumps at outlet # #new_outflow_xmom = outflow.get_average_xmom() + outflow_extra_momentum[0] # #new_outflow_ymom = outflow.get_average_ymom() + outflow_extra_momentum[1] # new_outflow_xmom = barrel_speed*new_outflow_depth*outflow_direction[0] # new_outflow_ymom = barrel_speed*new_outflow_depth*outflow_direction[1] #else: # #new_outflow_xmom = outflow.get_average_xmom() # #new_outflow_ymom = outflow.get_average_ymom() # new_outflow_xmom = 0.0 # new_outflow_ymom = 0.0 if self.use_momentum_jet: # FIXME (SR) Review momentum to account for possible hydraulic jumps at outlet # FIXME (GD) Depending on barrel speed I think this will be either # a source or sink of momentum (considering the momentum losses # above). Might not always be reasonable. #new_outflow_xmom = self.outflow.get_average_xmom() + outflow_extra_momentum[0] #new_outflow_ymom = self.outflow.get_average_ymom() + outflow_extra_momentum[1] new_outflow_xmom = barrel_speed * new_outflow_depth * outflow_direction[ 0] new_outflow_ymom = barrel_speed * new_outflow_depth * outflow_direction[ 1] elif self.zero_outflow_momentum: new_outflow_xmom = 0.0 new_outflow_ymom = 0.0 #new_outflow_xmom = outflow.get_average_xmom() #new_outflow_ymom = outflow.get_average_ymom() else: # Add the momentum lost from the inflow to the outflow. For # structures where barrel_speed is unknown + direction doesn't # change from inflow to outflow new_outflow_xmom = outflow_average_xmom + old_div( xmom_loss, outflow_area) new_outflow_ymom = outflow_average_ymom + old_div( ymom_loss, outflow_area) # master proc of structure sends outflow attributes to all outflow procs for i in self.inlet_procs[self.outflow_index]: if i == self.myid: continue pypar.send(new_outflow_depth, i) pypar.send(new_outflow_xmom, i) pypar.send(new_outflow_ymom, i) # outflow inlet procs receives new outflow attributes elif self.myid in self.inlet_procs[self.outflow_index]: new_outflow_depth = pypar.receive(self.master_proc) new_outflow_xmom = pypar.receive(self.master_proc) new_outflow_ymom = pypar.receive(self.master_proc) # outflow inlet procs sets new outflow attributes if self.myid in self.inlet_procs[self.outflow_index]: self.inlets[self.outflow_index].set_depths(new_outflow_depth) self.inlets[self.outflow_index].set_xmoms(new_outflow_xmom) self.inlets[self.outflow_index].set_ymoms(new_outflow_ymom)
def Weir_orifice_trapezoid_operator( domain, losses, width, blockage=0.0, barrels=1.0, z1=None, z2=None, height=None, end_points=None, exchange_lines=None, enquiry_points=None, invert_elevations=None, #culvert_slope=None, apron=0.1, manning=0.013, enquiry_gap=0.0, smoothing_timescale=0.0, use_momentum_jet=True, use_velocity_head=True, description=None, label=None, structure_type='weir_orifice_trapezoid', logging=False, verbose=False, master_proc=0, procs=None): # If not parallel domain then allocate serial Weir orifice trapezoid operator if isinstance(domain, Parallel_domain) is False: if verbose: print( "Allocating non parallel weir orifice trapzezoid operator ....." ) return anuga.structures.weir_orifice_trapezoid_operator.Weir_orifice_trapezoid_operator( domain=domain, losses=losses, width=width, height=height, blockage=blockage, barrels=barrels, z1=z1, z2=z2, #culvert_slope=culvert_slope, end_points=end_points, exchange_lines=exchange_lines, enquiry_points=enquiry_points, invert_elevations=invert_elevations, apron=apron, manning=manning, enquiry_gap=enquiry_gap, smoothing_timescale=smoothing_timescale, use_momentum_jet=use_momentum_jet, use_velocity_head=use_velocity_head, description=description, label=label, structure_type=structure_type, logging=logging, verbose=verbose) from anuga.utilities import parallel_abstraction as pypar if procs is None: procs = list(range(0, pypar.size())) myid = pypar.rank() end_points = ensure_numeric(end_points) exchange_lines = ensure_numeric(exchange_lines) enquiry_points = ensure_numeric(enquiry_points) if height is None: height = width diameter = None if apron is None: apron = width # Calculate location of inlet enquiry points and exchange lines if myid == master_proc: if exchange_lines is not None: exchange_lines_tmp = exchange_lines enquiry_points_tmp = __process_skew_culvert( exchange_lines, end_points, enquiry_points, apron, enquiry_gap) for i in procs: if i == master_proc: continue pypar.send(enquiry_points_tmp, i) elif end_points is not None: exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert( end_points, width, enquiry_points, apron, enquiry_gap) for i in procs: if i == master_proc: continue pypar.send(exchange_lines_tmp, i) pypar.send(enquiry_points_tmp, i) else: raise Exception('Define either exchange_lines or end_points') else: if exchange_lines is not None: exchange_lines_tmp = exchange_lines enquiry_points_tmp = pypar.receive(master_proc) elif end_points is not None: exchange_lines_tmp = pypar.receive(master_proc) enquiry_points_tmp = pypar.receive(master_proc) # Determine processors associated with first inlet line0 = exchange_lines_tmp[0] enquiry_point0 = enquiry_points_tmp[0] alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs( domain, line0, enquiry_point=enquiry_point0, master_proc=master_proc, procs=procs, verbose=verbose) # Determine processors associated with second inlet line1 = exchange_lines_tmp[1] enquiry_point1 = enquiry_points_tmp[1] alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs( domain, line1, enquiry_point=enquiry_point1, master_proc=master_proc, procs=procs, verbose=verbose) structure_procs = list(set(inlet0_procs + inlet1_procs)) inlet_master_proc = [inlet0_master_proc, inlet1_master_proc] inlet_procs = [inlet0_procs, inlet1_procs] enquiry_proc = [enquiry0_proc, enquiry1_proc] if myid == master_proc and verbose: print( "Parallel Weir Orifice Trapezoid Operator =============================" ) print("Structure Master Proc is P" + str(inlet0_master_proc)) print("Structure Procs are P" + str(structure_procs)) print("Inlet Master Procs are P" + str(inlet_master_proc)) print("Inlet Procs are P" + str(inlet_procs[0]) + " and " + str(inlet_procs[1])) print("Inlet Enquiry Procs are P" + str(enquiry_proc)) print("Enquiry Points are " + str(enquiry_point0) + " and " + str(enquiry_point1)) print("Inlet Exchange Lines are " + str(line0) + " and " + str(line1)) print("========================================================") if alloc0 or alloc1: return Parallel_Weir_orifice_trapezoid_operator( domain=domain, losses=losses, width=width, height=height, blockage=blockage, barrels=barrels, z1=z1, z2=z2, #culvert_slope=culvert_slope, end_points=end_points, exchange_lines=exchange_lines, enquiry_points=enquiry_points, invert_elevations=invert_elevations, apron=apron, manning=manning, enquiry_gap=enquiry_gap, smoothing_timescale=smoothing_timescale, use_momentum_jet=use_momentum_jet, use_velocity_head=use_velocity_head, description=description, label=label, structure_type=structure_type, logging=logging, verbose=verbose, master_proc=inlet0_master_proc, procs=structure_procs, inlet_master_proc=inlet_master_proc, inlet_procs=inlet_procs, enquiry_proc=enquiry_proc) else: return None
def set_stages_evenly(self,volume): """ Distribute volume of water over inlet exchange region so that stage is level """ # WARNING: requires synchronization, must be called by all procs associated # with this inlet from anuga.utilities import parallel_abstraction as pypar centroid_coordinates = self.domain.get_full_centroid_coordinates(absolute=True) areas = self.get_areas() stages = self.get_stages() stages_order = stages.argsort() # PETE: send stages and areas, apply merging procedure s_areas = {} s_stages = {} s_stages_order = {} total_stages = len(stages) if self.myid == self.master_proc: s_areas[self.myid] = areas s_stages[self.myid] = stages s_stages_order[self.myid] = stages_order # Recieve areas, stages, and stages order for i in self.procs: if i != self.master_proc: s_areas[i] = pypar.receive(i) s_stages[i] = pypar.receive(i) s_stages_order[i] = pypar.receive(i) total_stages = total_stages + len(s_stages[i]) else: # Send areas, stages, and stages order to master proc of inlet pypar.send(areas, self.master_proc) pypar.send(stages, self.master_proc) pypar.send(stages_order, self.master_proc) # merge sorted stage order if self.myid == self.master_proc: pos = {} summed_volume = 0. summed_areas = 0. prev_stage = 0. num_stages = 0. first = True for i in self.procs: pos[i] = 0 while num_stages < total_stages: # Determine current minimum stage of all the processors in s_stages num_stages = num_stages + 1 current_stage = num.finfo(num.float32).max index = -1 for i in self.procs: if pos[i] >= len(s_stages[i]): continue if s_stages[i][s_stages_order[i][pos[i]]] < current_stage: current_stage = s_stages[i][s_stages_order[i][pos[i]]] index = i # If first iteration, then only update summed_areas, position, and prev|current stage if first: first = False summed_areas = s_areas[index][s_stages_order[index][pos[index]]] pos[index] = pos[index] + 1 prev_stage = current_stage continue assert index >= 0, "Index out of bounds" # Update summed volume and summed areas tmp_volume = summed_volume + (summed_areas * (current_stage - prev_stage)) # Terminate if volume exceeded if tmp_volume >= volume: break summed_areas = summed_areas + s_areas[index][s_stages_order[index][pos[index]]] pos[index] = pos[index] + 1 summed_volume = tmp_volume # Update position of index processor and current stage prev_stage = current_stage # Calculate new stage new_stage = prev_stage + old_div((volume - summed_volume), summed_areas) # Send postion and new stage to all processors for i in self.procs: if i != self.master_proc: pypar.send(pos[i], i) pypar.send(new_stage, i) # Update own depth stages[stages_order[0:pos[self.myid]]] = new_stage else: pos = pypar.receive(self.master_proc) new_stage = pypar.receive(self.master_proc) stages[stages_order[0:pos]] = new_stage self.set_stages(stages) stages = self.get_stages() stages_order = stages.argsort()
def Internal_boundary_operator(domain, internal_boundary_function, width=1., height=1., end_points=None, exchange_lines=None, enquiry_points=None, invert_elevation=None, apron=0.0, enquiry_gap=0.0, use_velocity_head=False, zero_outflow_momentum=False, force_constant_inlet_elevations=True, smoothing_timescale=0.0, compute_discharge_implicitly=True, description=None, label=None, structure_type='internal_boundary', logging=False, verbose=True, master_proc=0, procs=None, inlet_master_proc=[0, 0], inlet_procs=None, enquiry_proc=[0, 0]): # If not parallel domain then allocate serial Internal boundary operator if isinstance(domain, Parallel_domain) is False: if verbose: print("Allocating non parallel internal_boundary operator .....") return anuga.structures.internal_boundary_operator.Internal_boundary_operator( domain=domain, internal_boundary_function=internal_boundary_function, width=width, height=height, end_points=end_points, exchange_lines=exchange_lines, enquiry_points=enquiry_points, invert_elevation=invert_elevation, apron=apron, enquiry_gap=enquiry_gap, use_velocity_head=use_velocity_head, zero_outflow_momentum=zero_outflow_momentum, force_constant_inlet_elevations=force_constant_inlet_elevations, smoothing_timescale=smoothing_timescale, compute_discharge_implicitly=compute_discharge_implicitly, description=description, label=label, structure_type=structure_type, logging=logging, verbose=verbose) from anuga.utilities import parallel_abstraction as pypar if procs is None: procs = list(range(0, pypar.size())) myid = pypar.rank() end_points = ensure_numeric(end_points) exchange_lines = ensure_numeric(exchange_lines) enquiry_points = ensure_numeric(enquiry_points) if height is None: height = width diameter = None if apron is None: apron = width # Calculate location of inlet enquiry points and exchange lines if myid == master_proc: if exchange_lines is not None: exchange_lines_tmp = exchange_lines enquiry_points_tmp = __process_skew_culvert( exchange_lines, end_points, enquiry_points, apron, enquiry_gap) for i in procs: if i == master_proc: continue pypar.send(enquiry_points_tmp, i) elif end_points is not None: exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert( end_points, width, enquiry_points, apron, enquiry_gap) for i in procs: if i == master_proc: continue pypar.send(exchange_lines_tmp, i) pypar.send(enquiry_points_tmp, i) else: raise Exception('Define either exchange_lines or end_points') else: if exchange_lines is not None: exchange_lines_tmp = exchange_lines enquiry_points_tmp = pypar.receive(master_proc) elif end_points is not None: exchange_lines_tmp = pypar.receive(master_proc) enquiry_points_tmp = pypar.receive(master_proc) # Determine processors associated with first inlet line0 = exchange_lines_tmp[0] enquiry_point0 = enquiry_points_tmp[0] alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs( domain, line0, enquiry_point=enquiry_point0, master_proc=master_proc, procs=procs, verbose=verbose) # Determine processors associated with second inlet line1 = exchange_lines_tmp[1] enquiry_point1 = enquiry_points_tmp[1] alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs( domain, line1, enquiry_point=enquiry_point1, master_proc=master_proc, procs=procs, verbose=verbose) structure_procs = list(set(inlet0_procs + inlet1_procs)) inlet_master_proc = [inlet0_master_proc, inlet1_master_proc] inlet_procs = [inlet0_procs, inlet1_procs] enquiry_proc = [enquiry0_proc, enquiry1_proc] if myid == master_proc and verbose: print( "Parallel Internal boundary Operator =============================" ) print("Structure Master Proc is P" + str(inlet0_master_proc)) print("Structure Procs are P" + str(structure_procs)) print("Inlet Master Procs are P" + str(inlet_master_proc)) print("Inlet Procs are P" + str(inlet_procs[0]) + " and " + str(inlet_procs[1])) print("Inlet Enquiry Procs are P" + str(enquiry_proc)) print("Enquiry Points are " + str(enquiry_point0) + " and " + str(enquiry_point1)) print("Inlet Exchange Lines are " + str(line0) + " and " + str(line1)) print("========================================================") if alloc0 or alloc1: return Parallel_Internal_boundary_operator( domain=domain, internal_boundary_function=internal_boundary_function, width=width, height=height, end_points=end_points, exchange_lines=exchange_lines, enquiry_points=enquiry_points, invert_elevation=invert_elevation, apron=apron, enquiry_gap=enquiry_gap, use_velocity_head=use_velocity_head, zero_outflow_momentum=zero_outflow_momentum, force_constant_inlet_elevations=force_constant_inlet_elevations, smoothing_timescale=smoothing_timescale, compute_discharge_implicitly=compute_discharge_implicitly, description=description, label=label, structure_type=structure_type, logging=logging, verbose=verbose, master_proc=inlet0_master_proc, procs=structure_procs, inlet_master_proc=inlet_master_proc, inlet_procs=inlet_procs, enquiry_proc=enquiry_proc) else: return None
import atexit atexit.register(finalize) pypar.barrier() test_points = [] if myid == 0: if verbose: print('PARALLEL START') for i in range(samples): x = random.randrange(0, 1000) / 1000.0 * length y = random.randrange(0, 1000) / 1000.0 * width point = [x, y] test_points.append(point) for i in range(1, numprocs): pypar.send(test_points, i) else: test_points = pypar.receive(0) if myid == 0: control_data, success = run_simulation(parallel=False, test_points=test_points, verbose=verbose) for proc in range(1, numprocs): pypar.send(control_data, proc) else: control_data = pypar.receive(0) pypar.barrier() _, success = run_simulation(parallel=True,
def allocate_inlet_procs(domain, poly, enquiry_point=None, master_proc=0, procs=None, verbose=False): from anuga.utilities import parallel_abstraction as pypar if procs is None: procs = list(range(0, pypar.size())) myid = pypar.rank() vertex_coordinates = domain.get_full_vertex_coordinates(absolute=True) domain_centroids = domain.centroid_coordinates size = 0 has_enq_point = False numprocs = pypar.size() inlet_procs = [] max_size = -1 inlet_master_proc = -1 inlet_enq_proc = -1 # Calculate the number of points of the line inside full polygon #tri_id = line_intersect(vertex_coordinates, poly) if len(poly) == 2: # poly is a line if verbose: print("======================") tri_id = line_intersect(vertex_coordinates, poly) else: # poly is a polygon if verbose: print("+++++++++++++++++++++++") tris_0 = line_intersect(vertex_coordinates, [poly[0], poly[1]]) tris_1 = inside_polygon(domain_centroids, poly) tri_id = num.union1d(tris_0, tris_1) if verbose: print("P%d has %d triangles in poly %s" % (myid, len(tri_id), poly)) size = len(tri_id) if enquiry_point is not None: try: k = domain.get_triangle_containing_point(enquiry_point) if domain.tri_full_flag[k] == 1: size = size + 1 has_enq_point = True if verbose: print("P%d has enq point %s" % (myid, enquiry_point)) else: if verbose: print("P%d contains ghost copy of enq point %s" % (myid, enquiry_point)) has_enq_point = False except: if verbose: print("P%d does not contain enq point %s" % (myid, enquiry_point)) has_enq_point = False if myid == master_proc: # Recieve size of overlap from each processor # Initialize line_master_proc and inlet_procs if size > 0: inlet_procs = [master_proc] max_size = size inlet_master_proc = master_proc if has_enq_point: inlet_enq_proc = master_proc # Recieve size of overlap for i in procs: if i == master_proc: continue x = pypar.receive(i) y = pypar.receive(i) if x > 0: inlet_procs.append(i) # Choose inlet_master_proc as the one with the most overlap if x > max_size: max_size = x inlet_master_proc = i if y is True: assert inlet_enq_proc == -1, "Enquiry point correspond to more than one proc" inlet_enq_proc = i assert len(inlet_procs) > 0, "Line does not intersect any domain" assert inlet_master_proc >= 0, "No master processor assigned" if enquiry_point is not None: msg = "Enquiry point %s doesn't intersect mesh, maybe inside a building, try reducing enquiry_gap" % str( enquiry_point) if inlet_enq_proc < 0: raise Exception(msg) # Send inlet_master_proc and inlet_procs to all processors in inlet_procs for i in procs: if i != master_proc: pypar.send(inlet_master_proc, i) pypar.send(inlet_procs, i) pypar.send(inlet_enq_proc, i) else: pypar.send(size, master_proc) pypar.send(has_enq_point, master_proc) inlet_master_proc = pypar.receive(master_proc) inlet_procs = pypar.receive(master_proc) inlet_enq_proc = pypar.receive(master_proc) if has_enq_point: assert inlet_enq_proc == myid, "Enquiry found in proc, but not declared globally" if size > 0: return True, inlet_master_proc, inlet_procs, inlet_enq_proc else: return False, inlet_master_proc, inlet_procs, inlet_enq_proc
def dump_triangulation(self, filename="domain.png"): # Get vertex coordinates, partition full and ghost triangles based on self.tri_full_flag try: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import matplotlib.tri as tri except: print( "Couldn't import module from matplotlib, probably you need to update matplotlib" ) raise vertices = self.get_vertex_coordinates() full_mask = num.repeat(self.tri_full_flag == 1, 3) ghost_mask = num.repeat(self.tri_full_flag == 0, 3) myid = pypar.rank() numprocs = pypar.size() if myid == 0: fig = plt.figure() fx = {} fy = {} gx = {} gy = {} # Proc 0 gathers full and ghost nodes from self and other processors fx[0] = vertices[full_mask, 0] fy[0] = vertices[full_mask, 1] gx[0] = vertices[ghost_mask, 0] gy[0] = vertices[ghost_mask, 1] for i in range(1, numprocs): fx[i] = pypar.receive(i) fy[i] = pypar.receive(i) gx[i] = pypar.receive(i) gy[i] = pypar.receive(i) # Plot full triangles for i in range(0, numprocs): n = int(old_div(len(fx[i]), 3)) triang = num.array(list(range(0, 3 * n))) triang.shape = (n, 3) plt.triplot(fx[i], fy[i], triang, 'g-', linewidth=0.5) # Plot ghost triangles for i in range(0, numprocs): n = int(old_div(len(gx[i]), 3)) if n > 0: triang = num.array(list(range(0, 3 * n))) triang.shape = (n, 3) plt.triplot(gx[i], gy[i], triang, 'b--', linewidth=0.5) # Save triangulation to location pointed by filename plt.savefig(filename, dpi=600) else: # Proc 1..numprocs send full and ghost triangles to Proc 0 pypar.send(vertices[full_mask, 0], 0) pypar.send(vertices[full_mask, 1], 0) pypar.send(vertices[ghost_mask, 0], 0) pypar.send(vertices[ghost_mask, 1], 0)
def old_distribute(domain, verbose=False, debug=False, parameters=None): """ Distribute the domain to all processes parameters values """ if debug: verbose = True barrier() # FIXME: Dummy assignment (until boundaries are refactored to # be independent of domains until they are applied) if myid == 0: bdmap = {} for tag in domain.get_boundary_tags(): bdmap[tag] = None domain.set_boundary(bdmap) if not pypar_available or numprocs == 1: return domain # Bypass # For some obscure reason this communication must happen prior to # the more complex mesh distribution - Oh Well! if myid == 0: domain_name = domain.get_name() domain_dir = domain.get_datadir() domain_store = domain.get_store() domain_store_centroids = domain.get_store_centroids() domain_smooth = domain.smooth domain_reduction = domain.reduction domain_minimum_storable_height = domain.minimum_storable_height domain_flow_algorithm = domain.get_flow_algorithm() domain_minimum_allowed_height = domain.get_minimum_allowed_height() georef = domain.geo_reference number_of_global_triangles = domain.number_of_triangles number_of_global_nodes = domain.number_of_nodes # FIXME - what other attributes need to be transferred? for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send((domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, number_of_global_nodes), p) else: if verbose: print 'P%d: Receiving domain attributes' % (myid) domain_name, domain_dir, domain_store, \ domain_store_centroids, domain_smooth, domain_reduction, \ domain_minimum_storable_height, domain_flow_algorithm, \ domain_minimum_allowed_height, georef, \ number_of_global_triangles, \ number_of_global_nodes = receive(0) # Distribute boundary conditions # FIXME: This cannot handle e.g. Time_boundaries due to # difficulties pickling functions if myid == 0: boundary_map = domain.boundary_map for p in xrange(1, numprocs): # FIXME SR: Creates cPickle dump send(boundary_map, p) else: if verbose: print 'P%d: Receiving boundary map' % (myid) boundary_map = receive(0) send_s2p = False if myid == 0: # Partition and distribute mesh. # Structures returned is in the # correct form for the ANUGA data structure points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles,\ s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \ ghost_layer_width =\ distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] if debug: print 'P%d' % myid print 'tri_map ', tri_map print 'node_map', node_map print 'tri_l2g', tri_l2g print 'node_l2g', node_l2g print 's2p_map', s2p_map print 'p2s_map', p2s_map def protocol(x): vanilla = False import pypar control_info, x = pypar.create_control_info(x, vanilla, return_object=True) print 'protocol', control_info[0] # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs if send_s2p: n = len(s2p_map) s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(), num.int), (n, 1)) s2p_map_values_flat = num.array(s2p_map.values(), num.int) s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1) n = len(p2s_map) p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(), num.int), (n, 2)) p2s_map_values_flat = num.reshape( num.array(p2s_map.values(), num.int), (n, 1)) p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1) for p in range(1, numprocs): # FIXME SR: Creates cPickle dump send(s2p_map_flat, p) # FIXME SR: Creates cPickle dump #print p2s_map send(p2s_map_flat, p) else: if verbose: print 'Not sending s2p_map and p2s_map' s2p_map = None p2s_map = None if verbose: print 'Communication done' else: # Read in the mesh partition that belongs to this # processor if verbose: print 'P%d: Receiving submeshes' % (myid) points, vertices, boundary, quantities,\ ghost_recv_dict, full_send_dict,\ number_of_full_nodes, number_of_full_triangles, \ tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\ rec_submesh(0, verbose) # Extract l2g maps #tri_l2g = extract_l2g_map(tri_map) #node_l2g = extract_l2g_map(node_map) #tri_l2g = p2s_map[tri_l2g] # Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping if send_s2p: s2p_map_flat = receive(0) s2p_map = dict.fromkeys(s2p_map_flat[:, 0], s2p_map_flat[:, 1:2]) p2s_map_flat = receive(0) p2s_map_keys = [tuple(x) for x in p2s_map_flat[:, 0:1]] p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:, 2]) else: s2p_map = None p2s_map = None #------------------------------------------------------------------------ # Build the domain for this processor using partion structures #------------------------------------------------------------------------ if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % ( myid, number_of_full_nodes, number_of_full_triangles) domain = Parallel_domain( points, vertices, boundary, full_send_dict=full_send_dict, ghost_recv_dict=ghost_recv_dict, number_of_full_nodes=number_of_full_nodes, number_of_full_triangles=number_of_full_triangles, geo_reference=georef, number_of_global_triangles=number_of_global_triangles, number_of_global_nodes=number_of_global_nodes, s2p_map=s2p_map, p2s_map=p2s_map, ## jj added this tri_l2g=tri_l2g, ## SR added this node_l2g=node_l2g, ghost_layer_width=ghost_layer_width) #------------------------------------------------------------------------ # Transfer initial conditions to each subdomain #------------------------------------------------------------------------ for q in quantities: domain.set_quantity(q, quantities[q]) #------------------------------------------------------------------------ # Transfer boundary conditions to each subdomain #------------------------------------------------------------------------ boundary_map['ghost'] = None # Add binding to ghost boundary domain.set_boundary(boundary_map) #------------------------------------------------------------------------ # Transfer other attributes to each subdomain #------------------------------------------------------------------------ domain.set_name(domain_name) domain.set_datadir(domain_dir) domain.set_store(domain_store) domain.set_store_centroids(domain_store_centroids) domain.set_store_vertices_smoothly(domain_smooth, domain_reduction) domain.set_minimum_storable_height(domain_minimum_storable_height) domain.set_minimum_allowed_height(domain_minimum_allowed_height) domain.set_flow_algorithm(domain_flow_algorithm) domain.geo_reference = georef #------------------------------------------------------------------------ # Return parallel domain to all nodes #------------------------------------------------------------------------ return domain
def discharge_routine_implicit(self): """ Uses semi-implicit discharge estimation: Discharge = (1-theta)*Q(H0, T0) + theta*Q(H0 + delta_H, T0+delta_T)) where H0 = headwater stage, T0 = tailwater stage, delta_H = change in headwater stage over a timestep, delta_T = change in tailwater stage over a timestep, and Q is the discharge function, and theta is a constant in [0,1] determining the degree of implicitness (currently hardcoded). Note this is effectively assuming: 1) Q is a function of stage, not energy (so we can relate mass change directly to delta_H, delta_T). We could generalise it to the energy case ok. 2) The stage is computed on the exchange line (or the change in stage at the enquiry point is effectively the same as that on the exchange line) """ from anuga.utilities import parallel_abstraction as pypar local_debug = False # If the structure has been closed, then no water gets through if self.height <= 0.0: if self.myid == self.master_proc: Q = 0.0 barrel_velocity = 0.0 outlet_culvert_depth = 0.0 self.case = "Structure is blocked" self.inflow = self.inlets[0] self.outflow = self.inlets[1] return Q, barrel_velocity, outlet_culvert_depth else: return None, None, None #Send attributes of both enquiry points to the master proc if self.myid == self.master_proc: if self.myid == self.enquiry_proc[0]: enq_total_energy0 = self.inlets[0].get_enquiry_total_energy() enq_stage0 = self.inlets[0].get_enquiry_stage() else: enq_total_energy0 = pypar.receive(self.enquiry_proc[0]) enq_stage0 = pypar.receive(self.enquiry_proc[0]) if self.myid == self.enquiry_proc[1]: enq_total_energy1 = self.inlets[1].get_enquiry_total_energy() enq_stage1 = self.inlets[1].get_enquiry_stage() else: enq_total_energy1 = pypar.receive(self.enquiry_proc[1]) enq_stage1 = pypar.receive(self.enquiry_proc[1]) else: if self.myid == self.enquiry_proc[0]: pypar.send(self.inlets[0].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[0].get_enquiry_stage(), self.master_proc) if self.myid == self.enquiry_proc[1]: pypar.send(self.inlets[1].get_enquiry_total_energy(), self.master_proc) pypar.send(self.inlets[1].get_enquiry_stage(), self.master_proc) # Send inlet areas to the master proc. FIXME: Inlet areas don't change # -- perhaps we could just do this once? # area0 if self.myid in self.inlet_procs[0]: area0 = self.inlets[0].get_global_area() if self.myid == self.master_proc: if self.myid != self.inlet_master_proc[0]: area0 = pypar.receive(self.inlet_master_proc[0]) elif self.myid == self.inlet_master_proc[0]: pypar.send(area0, self.master_proc) # area1 if self.myid in self.inlet_procs[1]: area1 = self.inlets[1].get_global_area() if self.myid == self.master_proc: if self.myid != self.inlet_master_proc[1]: area1 = pypar.receive(self.inlet_master_proc[1]) elif self.myid == self.inlet_master_proc[1]: pypar.send(area1, self.master_proc) # Compute discharge if self.myid == self.master_proc: # Energy or stage as head if self.use_velocity_head: E0 = enq_total_energy0 E1 = enq_total_energy1 else: E0 = enq_stage0 E1 = enq_stage1 # Variables for anuga's structure operator self.delta_total_energy = E0 - E1 self.driving_energy = max(E0, E1) Q0 = self.internal_boundary_function(E0, E1) dt = self.domain.get_timestep() if dt > 0.: # Key constants for iterative solution theta = 1.0 sol = numpy.array([0., 0.]) # estimate of (delta_H, delta_T) areas = numpy.array([area0, area1]) # Use scipy root finding def F_to_solve(sol): Q1 = self.internal_boundary_function( E0 + sol[0], E1 + sol[1]) discharge = (1 - theta) * Q0 + theta * Q1 output = sol * areas - discharge * dt * numpy.array( [-1., 1.]) return (output) final_sol = sco.root(F_to_solve, sol, method='lm').x Q1 = self.internal_boundary_function(E0 + final_sol[0], E1 + final_sol[1]) Q = (1.0 - theta) * Q0 + theta * Q1 else: Q = Q0 # Smooth discharge if dt > 0.: ts = old_div(dt, max(dt, self.smoothing_timescale, 1.0e-30)) else: # No smoothing ts = 1.0 self.smooth_Q = self.smooth_Q + ts * (Q - self.smooth_Q) else: self.delta_total_energy = numpy.nan self.driving_energy = numpy.nan self.inflow_index = 0 self.outflow_index = 1 # master proc orders reversal if applicable if self.myid == self.master_proc: # Reverse the inflow and outflow direction? if Q < 0.: self.inflow_index = 1 self.outflow_index = 0 for i in self.procs: if i == self.master_proc: continue pypar.send(True, i) else: for i in self.procs: if i == self.master_proc: continue pypar.send(False, i) else: reverse = pypar.receive(self.master_proc) if reverse: self.inflow_index = 1 self.outflow_index = 0 # Master proc computes return values if self.myid == self.master_proc: # Zero Q if sign's of smooth_Q and Q differ if numpy.sign(self.smooth_Q) != numpy.sign(Q): Q = 0. self.smooth_Q = 0. else: # Make Q positive (for anuga's structure operator) Q = min(abs(self.smooth_Q), abs(Q)) # Variables required by anuga's structure operator which are # not used barrel_velocity = numpy.nan outlet_culvert_depth = numpy.nan return Q, barrel_velocity, outlet_culvert_depth else: return None, None, None
def plotCentroidError(domain, control_data, rthr = 1E-7, athr = 1E-12, quantity = 'stage', filename = 'centroid_error.png'): n_triangles = num.sum(domain.tri_full_flag) if size() > 1: # If parallel, translate control data to parallel indexing local_control_data = num.zeros(n_triangles) inv_tri_map = domain.get_inv_tri_map() for i in range(n_triangles): local_control_data[i] = control_data[inv_tri_map[(rank(), i)]] else: local_control_data = control_data # Evaluate absolute and relative difference between control and actual values stage = domain.get_quantity(quantity) actual_data = stage.centroid_values[:n_triangles] adiff = num.fabs((actual_data - local_control_data)) rdiff = adiff/num.fabs(local_control_data) # Compute masks for error (err_mask) and non-error (acc_mask) vertex indices based on thresholds vertices = domain.get_vertex_coordinates() err_mask = rdiff > rthr err_mask[adiff <= athr] = False err_mask = num.repeat(err_mask, 3) acc_mask = ~err_mask inv_tri_map = domain.get_inv_tri_map() # Plot error and non-error triangle if rank() == 0: fx = {} fy = {} gx = {} gy = {} fx[0] = vertices[acc_mask,0] fy[0] = vertices[acc_mask,1] gx[0] = vertices[err_mask,0] gy[0] = vertices[err_mask,1] # Receive vertex indices of non-error triangles (fx, fy) and error triangles (gx, gy) for i in range(1,size()): fx[i] = receive(i) fy[i] = receive(i) gx[i] = receive(i) gy[i] = receive(i) # Plot non-error triangles in green for i in range(0,size()): n = int(len(fx[i])/3) triang = num.array(range(0,3*n)) triang.shape = (n, 3) if len(fx[i]) > 0: plt.triplot(fx[i], fy[i], triang, 'g-') # Plot error triangles in blue for i in range(0,size()): n = int(len(gx[i])/3) triang = num.array(range(0,3*n)) triang.shape = (n, 3) if len(gx[i]) > 0: plt.triplot(gx[i], gy[i], triang, 'b--') # Save plot plt.savefig(filename) else: # Send error and non-error vertex indices to Proc 0 send(vertices[acc_mask,0], 0) send(vertices[acc_mask,1], 0) send(vertices[err_mask,0], 0) send(vertices[err_mask,1], 0)