def scatter( vec ): if root(): for i in xrange(p.size()-1): p.send(vec[i+1],i+1) return vec[0] else: return p.receive(0)
def get_enquiry_velocitys(self):
enq0 = None
enq1 = None
get0 = "self.inlets[0].get_enquiry_velocity()"
get1 = "self.inlets[1].get_enquiry_velocity()"
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[0]:
enq0 = eval(get0)
else:
enq0 = pypar.receive(self.enquiry_proc[0])
if self.myid == self.enquiry_proc[1]:
enq1 = eval(get1)
else:
enq1 = pypar.receive(self.enquiry_proc[1])
else:
if self.myid == self.enquiry_proc[0]:
enq0 = eval(get0)
pypar.send(enq0, self.master_proc)
if self.myid == self.enquiry_proc[1]:
enq1 = eval(get1)
pypar.send(enq1, self.master_proc)
return [enq0, enq1]
def splitArray(self, array): '''Splits array between all the processes in the environment. Each process will be returned a different section of the array to work on''' if Environment.isParallel: import pypar #Split the array into sections and return the section for this processor divisions = [] if self.isRoot(): #Root does the splitting - we send each processor the start and end index #NOTE: pypar broadcast won't work even when setting vanilla #It always returns message trucated error. divisions = self._divideArray(array) for i in range(1,pypar.size()): pypar.send(divisions[i], i) start = divisions[0][0] end = divisions[0][1] else: indexes = pypar.receive(0) start = indexes[0] end = indexes[1] return array[start:end] else: return array
def get_enquiry_depths(self):
# Should be called from all processors associated with operator
enq0 = None
enq1 = None
get0 = "self.inlets[0].get_enquiry_depth()"
get1 = "self.inlets[1].get_enquiry_depth()"
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[0]:
enq0 = eval(get0)
else:
enq0 = pypar.receive(self.enquiry_proc[0])
if self.myid == self.enquiry_proc[1]:
enq1 = eval(get1)
else:
enq1 = pypar.receive(self.enquiry_proc[1])
else:
if self.myid == self.enquiry_proc[0]:
enq0 = eval(get0)
pypar.send(enq0, self.master_proc)
if self.myid == self.enquiry_proc[1]:
enq1 = eval(get1)
pypar.send(enq1, self.master_proc)
return [enq0, enq1]
def get_global_average_elevation(self):
# GLOBAL: Master processor gathers elevations from all child processors, and returns average
# WARNING: requires synchronization, must be called by all procs associated
# with this inlet
import pypar
local_elevation = num.sum(self.get_elevations()*self.get_areas())
global_area = self.get_global_area()
global_elevation = local_elevation
if self.myid == self.master_proc:
for i in self.procs:
if i == self.master_proc: continue
val = pypar.receive(i)
global_elevation = global_elevation + val
else:
pypar.send(local_elevation, self.master_proc)
if global_area > 0.0:
return global_elevation/global_area
else:
return 0.0
def test_mpi_recvsend(target_array, message_range, target_rank, tag_mark):
send_range = message_range[0]
recv_range = message_range[1]
target_array[recv_range] = mpi.receive(target_rank,tag=tag_mark)
# print 'I`m', myrank, 'Recv : ', target_rank, 'range : ', recv_range
mpi.send(target_array[send_range].copy(), target_rank, tag=tag_mark)
def scatter_dict(whole):
"""
Broadcast and recieve a dictionary where the values are 1d arrays
and the arrays are chunked for the workers.
Only rank 0 needs the whole dictionary.
:param whole: The dictionary of 1d arrays to subdict.
:returns: (chunk of dictionary of 1d arrays, indexes of whole array)
"""
if not STATE.is_parallel:
array_len = len(whole[whole.keys()[0]])
return whole, numpy.array(range(0, array_len))
else:
import pypar # pylint: disable=W0404
if STATE.rank == 0:
array_len = len(whole[whole.keys()[0]])
for pro in range(0, STATE.size):
temp_indexes = numpy.array(range(pro, array_len, STATE.size))
temp_subdict = {}
for key in whole.keys():
temp_subdict[key] = whole[key][temp_indexes]
if pro is 0:
indexes = temp_indexes
subdict = temp_subdict
else:
pypar.send(temp_indexes, pro)
pypar.send(temp_subdict, pro)
else:
indexes = pypar.receive(0)
subdict = pypar.receive(0)
return subdict, indexes
def send(self, *args, **kwargs):
"""
Wrapper for pypar.send
"""
if self.is_parallel is True:
import pypar
pypar.send(*args, **kwargs)
def distributed_generator(iterable):
"""
Distribute the values from a generator across workers.
"""
RUN, DIE = range(2)
P = pp.size()
if P == 1:
for el in iterable:
yield el
else:
if pp.rank() == 0:
it = iter(iterable)
while True:
try:
first = next(it)
for p in range(1, P):
pp.send(next(it), p, tag=RUN)
yield first
except StopIteration:
for p in range(1, P):
pp.send(666, p, tag=DIE)
break
else:
while True:
el, status = pp.receive(0, tag=pp.any_tag, return_status=True)
if status.tag == DIE:
break
yield el
def balanceArrays(self, arrayFragment):
'''Redistributes the elements in a set of arrays equally across the nodes'''
if Environment.isParallel:
import pypar
if self.isRoot():
completeArray = arrayFragment
for i in range(1, pypar.size()):
fragment = pypar.receive(i)
completeArray.extend(fragment)
#Divide it up
divisions = self._divideArray(completeArray)
#Send the fragments
for i in range(1, pypar.size()):
start, end = divisions[i]
pypar.send(completeArray[start:end], i)
self.output('[ENV] Rebalanced array divisions %s' % divisions)
#Assign root fragment
start, end = divisions[0]
arrayFragment = completeArray[start:end]
else:
#Send the fragment
pypar.send(arrayFragment, 0)
#Retrieve the array
arrayFragment = pypar.receive(0)
else:
completeArray = arrayFragment
return arrayFragment
def broadcast( obj ): if root(): for i in xrange(p.size()-1): p.send(obj,i+1) return obj else: return p.receive(0)
def get_enquiry_water_depths(self):
enq0 = None
enq1 = None
get0 = 'self.inlets[0].get_enquiry_water_depth()'
get1 = 'self.inlets[1].get_enquiry_water_depth()'
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[0]:
enq0 = eval(get0)
else:
enq0 = pypar.receive(self.enquiry_proc[0])
if self.myid == self.enquiry_proc[1]:
enq1 = eval(get1)
else:
enq1 = pypar.receive(self.enquiry_proc[1])
else:
if self.myid == self.enquiry_proc[0]:
enq0 = eval(get0)
pypar.send(enq0, self.master_proc)
if self.myid == self.enquiry_proc[1]:
enq1 = eval(get1)
pypar.send(enq1, self.master_proc)
return [enq0, enq1]
def broadcast_vec( vec, i ): myid = p.rank() if myid == i: for j in xrange(p.size()): if j != myid: p.send(vec[i],j) else: vec[i] = p.receive(i)
def gather( obj ): if root(): result = [ None for i in xrange(p.size()) ] result[0] = obj for i in xrange(p.size()-1): result[i+1] = p.receive(i+1) return result else: p.send(obj,0)
def Calculate(self, varsByCalc, params = None):
"""
Calculate model predictions for everything in varsByCalc.
varsByCalc is a dictionary of the form:
dict[calc name][dep var] = ind var
The return dictionary is of the form:
dictionary[calc name][dep var][ind var] = result
"""
if params is not None:
self.params.update(params)
results = {}
calcs_to_do = varsByCalc.keys()
# Record which calculation each node is doing
calc_assigned = {}
while calcs_to_do:
# The number of calculations to do this round. We want to use
# all the processors if possible.
len_this_block = min(SloppyCell.num_procs, len(calcs_to_do))
for worker in range(1, len_this_block):
calc = calcs_to_do.pop()
calc_assigned[worker] = calc
logger.debug('Assigning calculation %s to worker %i.'
% (calc, worker))
command = 'Network.calculate(net, vars, params)'
args = {'net': self.get(calc), 'vars': varsByCalc[calc],
'params': self.params}
pypar.send((command, args), worker)
# The master does his share here
calc = calcs_to_do.pop()
# We use the finally statement because we want to ensure that we
# *always* wait for replies from the workers, even if the master
# encounters an exception in his evaluation.
try:
results[calc] = self.get(calc).calculate(varsByCalc[calc],
self.params)
finally:
# Collect results from the workers
for worker in range(1, len_this_block):
logger.debug('Receiving result from worker %i.' % worker)
results[calc_assigned[worker]] = pypar.receive(worker)
# If the master encounts an exception, we'll break out of the
# function ***here***
# Check the results we received. If any is a SloppyCellException,
# reraise it.
for val in results.values():
if isinstance(val, Utility.SloppyCellException):
raise val
return results
def __call__(self):
import pypar
volume = 0
# Need to run global command on all processors
current_volume = self.inlet.get_global_total_water_volume()
total_area = self.inlet.get_global_area()
# Only the master proc calculates the update
if self.myid == self.master_proc:
timestep = self.domain.get_timestep()
t = self.domain.get_time()
Q1 = self.update_Q(t)
Q2 = self.update_Q(t + timestep)
volume = 0.5*(Q1+Q2)*timestep
assert current_volume >= 0.0 , 'Volume of watrer in inlet negative!'
for i in self.procs:
if i == self.master_proc: continue
pypar.send((volume, current_volume, total_area, timestep), i)
else:
volume, current_volume, total_area, timestep = pypar.receive(self.master_proc)
#print self.myid, volume, current_volume, total_area, timestep
self.applied_Q = volume/timestep
# Distribute positive volume so as to obtain flat surface otherwise
# just pull water off to have a uniform depth.
if volume >= 0.0 :
self.inlet.set_stages_evenly(volume)
self.domain.fractional_step_volume_integral+=volume
if self.velocity is not None:
# This is done locally without communication
depths = self.inlet.get_depths()
self.inlet.set_xmoms(self.inlet.get_xmoms()+depths*self.velocity[0])
self.inlet.set_ymoms(self.inlet.get_ymoms()+depths*self.velocity[1])
elif current_volume + volume >= 0.0 :
depth = (current_volume + volume)/total_area
self.inlet.set_depths(depth)
self.domain.fractional_step_volume_integral+=volume
else: #extracting too much water!
self.inlet.set_depths(0.0)
self.applied_Q = current_volume/timestep
self.domain.fractional_step_volume_integral-=current_volume
def process(self):
# loop through all the rays that go through the window
for row in xrange(self.Rows):
for col in xrange(self.WindowCols):
# loop through all the light sources
for Light in self.LightSources:
Energy = 0.0
# move the ray to the corresponding pixel on the window. dx and dy changes
MoveUp = self.WindowHeight/2.0 - (row+self.RowStart - 0.5)*\
self.WindowHeight/self.WindowRows # starts from top most pixel
MoveRight = -self.WindowWidth/2.0 + (col + 0.5)*\
self.WindowWidth/self.WindowCols # starts from left most pixel
# define the ray and rotate (in look direction)
# then translate (window distance)
Vec = np.array([self.WindowDistance, -MoveRight, MoveUp])
Vec = self.__RotVec(Vec, self.__Lon, self.__Lat, self.Yaw)
Photon = Ray.Ray(self.LookPos,Vec,1.0)
Cont = True
# get energy after first contact
Res = self.__getEn(Photon, Light)
Energy += Res[0]
UnScatRay = Res[1]
Cont = Res[2]
#pypar.barrier();
for k in range(1):
# get energy after N contacts
if Cont:
Res = self.__getEn(UnScatRay, Light)
Energy += Res[0]
UnScatRay = Res[1]
Cont = Res[2]
self.Window[row, col] += Energy # set the energy to the corresponding window
# stitch together the images from each process
if self.myid != 0:
pypar.send(self.Window, 0)
else:
self.Visual = np.zeros(shape = (self.WindowRows, self.WindowCols))
RowEnd = self.WindowRows/self.numproc
self.Visual[:RowEnd, :] = self.Window.copy()
for Pr in range(1,self.numproc):
RowStart = self.WindowRows/self.numproc * Pr
RowEnd = self.WindowRows/self.numproc * (Pr+1)
self.Visual[RowStart:RowEnd,:] = pypar.receive(Pr)
print "Elapsed time for process %d is %f" % (self.myid, time.time() - self.start)
def grad_func(x0, *grad_args):
sys.stdout.flush()
x0 = numpy.asarray(x0)
# Generate a list of parameter sets to evaluate the function at
x_todo = [x0]
for ii in range(len(x0)):
eps = numpy.zeros(len(x0), numpy.float_)
eps[ii] = epsilon
x_todo.append(x0 + eps)
# Break that list apart into work for each node
x_by_node = []
for node_ii in range(num_procs):
x_by_node.append(x_todo[node_ii::num_procs])
# Master sends orders to all other nodes.
command = 'eval_func_over_list(func_str, x_todo, *grad_args)'
for node_ii in range(1, num_procs):
x_this_node = x_by_node[node_ii]
arguments = {'func_str': func_str,
'x_todo': x_this_node}
if send_grad_args:
arguments['grad_args'] = grad_args
pypar.send((command, arguments), node_ii)
sys.stdout.flush()
# This will hold the function evaluations done by each node.
vals_by_node = []
# The master node does its share of the work now
vals_by_node.append(eval_func_over_list(func_str, x_by_node[0],
*grad_args))
# Now receive the work done by each of the other nodes
for node_ii in range(1, num_procs):
vals_by_node.append(pypar.receive(node_ii))
# Reform the function value list that's broken apart by node.
func_evals = numpy.zeros(len(x0)+1, numpy.float_)
for node_ii,vals_this_node in enumerate(vals_by_node):
func_evals[node_ii::num_procs] = vals_this_node
# Now calculate the gradient
grad = numpy.zeros(len(x0), numpy.float_)
f0 = func_evals[0]
for ii,func_val in enumerate(func_evals[1:]):
grad[ii] = (func_val - f0)/epsilon
return grad
def broadcast(self, data, process): '''Broadcasts data from process to all other nodes''' if Environment.isParallel: import pypar if self.rank() == process: #NOTE: pypar broadcast won't work even when setting vanilla #It always returns message trucated error. for i in range(pypar.size()): if i != self.rank(): pypar.send(data, i) else: data = pypar.receive(process) return data
def collect_arr(arr):
"""
A useful collection routine for pypar.
If you are using pypar to parallelize the set of nested loops and fill
the resulting array, you usually need to combine the resulting array
from several mpi threads. In that case you just can execute
res=collect_arr(res)
And it will add the arrays from all the threads and store them in
the thread number 0
"""
if pypar.rank() > 0:
pypar.send(arr, 0)
else:
for i in range(1, pypar.size()):
arr = arr + pypar.receive(i)
return arr
def test_longint_array(self):
myid, ncpu = pp.rank(), pp.size()
N = 17 # Number of elements
if myid == 0:
A = np.array(range(N)).astype('l')
B = np.zeros(N).astype('l')
pp.send(A, 1, use_buffer=True)
B = pp.receive(ncpu - 1, buffer=B)
self.assertTrue(np.allclose(A, B))
else:
X = np.zeros(N).astype('l')
X = pp.receive(myid - 1, buffer=X)
pp.send(X, (myid + 1) % ncpu, use_buffer=True)
def slave(self):
if self.debug: print '[SLAVE %d]: I am processor %d of %d on node %s' % (self.myid, self.myid, self.numprocs, self.node)
if self.debug: print '[SLAVE %d]: Entering work loop' % (self.myid,)
while True:
result, status = pypar.receive(source=0, tag=pypar.any_tag, return_status=True)
print '[SLAVE %d]: received work with tag %d from node %d'\
%(self.myid, status.tag, status.source)
if (status.tag == PYPAR_DIETAG):
print '[SLAVE %d]: received termination from node %d' % (self.myid, 0)
return
else:
worknum = result
if self.debug: print '[SLAVE %d]: work number is %s' % (self.myid, worknum)
myresult = self.work.calcWorkResult(worknum)
pypar.send(myresult, destination=0)
if self.debug: print '[SLAVE %d]: sent result to node %d' % (self.myid, 0)
def calc_num_blocks(self):
"""
pre-req: calc_lo_hi has been calculated - and only calculated once!
"""
if self.is_parallel is True:
import pypar
# print "synchronise self.rank", self.rank
if self.rank == 0:
calc_num_blocks = self._make_block_file
for source in range(1, self.size):
# print "waiting.."
received = pypar.receive(source)
# print "received", received
calc_num_blocks += received
return calc_num_blocks
else:
# print "sending from ", self.rank
pypar.send(self._make_block_file, 0)
def slave(self):
logging.debug('[SLAVE %d]: started processor %d of %d on node %s'%(self.MPI_myid, self.MPI_myid, self.MPI_numproc, self.MPI_node))
while True:
inputMsg, status = pypar.receive(source=0,
tag=pypar.any_tag,
return_status=True)
logging.debug('[SLAVE %d]: received work "%s" with tag %d from node %d' %(self.MPI_myid, inputMsg, status.tag, status.source))
if (status.tag == self.DIETAG):
logging.debug('[SLAVE %d]: received termination from node %d'%(self.MPI_myid, 0))
return
else:
logging.debug('[SLAVE %d]: received work "%s" to map' %(self.MPI_myid, inputMsg))
resultMsg = self.mapFunction(inputMsg)
pypar.send(resultMsg, destination=0, tag=self.WORKTAG)
logging.debug('[SLAVE %d]: sent result "%s" to node %d'%(self.MPI_myid, resultMsg, 0))
def collected(iterable):
"""
Collect iterables back to the master.
"""
P = pp.size()
if P == 1:
for el in iterable:
yield el
else:
if pp.rank() == 0:
results = list(iterable)
for p in range(1, P):
pres = pp.receive(p)
results.extend(pres)
for el in results:
yield el
else:
pp.send(list(iterable), 0)
def ensemble_trajs(net, times, ensemble):
"""
Return a list of trajectories evaluated at times for all parameter sets
in ensemble.
"""
traj_set = []
elems_assigned = [ensemble[node::num_procs] for node in range(num_procs)]
for worker in range(1, num_procs):
command = 'Ensembles.few_ensemble_trajs(net, times, elements)'
args = {'net': net, 'times': times, 'elements': elems_assigned[worker]}
pypar.send((command, args), worker)
traj_set = few_ensemble_trajs(net, times, elems_assigned[0])
for worker in range(1, num_procs):
traj_set.extend(pypar.receive(worker))
return traj_set
def statistics(self):
# Warning: requires synchronization, must be called by all procs associated
# with this structure
message = " "
if self.myid == self.master_proc:
message = "===============================================\n"
message += "Parallel Structure Operator: %s\n" % self.label
message += "===============================================\n"
message += "Structure Type: %s\n" % self.structure_type
message += "Description\n"
message += "%s" % self.description
message += "\n"
# print "Structure Myids ",self.myid, self.label
for i, inlet in enumerate(self.inlets):
if self.myid == self.master_proc:
message += "-------------------------------------\n"
message += "Inlet %i\n" % (i)
message += "-------------------------------------\n"
# print "*****",inlet, i,self.myid
if inlet is not None:
stats = inlet.statistics()
if self.myid == self.master_proc:
if self.myid != self.inlet_master_proc[i]:
stats = pypar.receive(self.inlet_master_proc[i])
elif self.myid == self.inlet_master_proc[i]:
pypar.send(stats, self.master_proc)
if self.myid == self.master_proc:
message += stats
if self.myid == self.master_proc:
message += "=====================================\n"
return message
def print_l1_stats(full_edge):
numprocs = pypar.size()
myid = pypar.rank()
tri_norm = zeros(3, Float)
recv_norm = zeros(3, Float)
tri_norm[0] = l1_norm(full_edge[:, 0])
tri_norm[1] = l1_norm(full_edge[:, 1])
tri_norm[2] = l1_norm(full_edge[:, 2])
if myid == 0:
for p in range(numprocs-1):
pypar.receive(p+1, recv_norm)
tri_norm[0] = tri_norm[0]+recv_norm[0]
tri_norm[1] = tri_norm[1]+recv_norm[1]
tri_norm[2] = tri_norm[2]+recv_norm[2]
print 'l1_norm along each axis : [', tri_norm[0],', ', tri_norm[1], ', ', tri_norm[2], ']'
else:
pypar.send(tri_norm, 0)
def get_global_total_water_volume(self):
# GLOBAL: master proc gathers total water volumes from each proc and returns average
# WARNING: requires synchronization, must be called by all procs associated
# with this inlet
import pypar
local_volume = num.sum(self.get_depths()*self.get_areas())
volume = local_volume
if self.myid == self.master_proc:
for i in self.procs:
if i == self.master_proc: continue
val = pypar.receive(i)
volume = volume + val
else:
pypar.send(volume, self.master_proc)
return volume
def slave():
print '[SLAVE %d]: I am processor %d of %d on node %s'\
%(MPI_myid, MPI_myid, MPI_numproc, MPI_node)
while True:
result, status = pypar.receive(source=0,
tag=pypar.any_tag,
return_status=True)
print '[SLAVE %d]: received work "%s" with tag %d from node %d'\
%(MPI_myid, result, status.tag, status.source)
if (status.tag == DIETAG):
print '[SLAVE %d]: received termination from node %d'\
%(MPI_myid, 0)
return
else:
result = 'X'+result
pypar.send(result, destination=0, tag=WORKTAG)
print '[SLAVE %d]: sent result "%s" to node %d'\
%(MPI_myid, result, 0)
def Weir_orifice_trapezoid_operator(
domain,
losses,
width,
blockage=0.0,
barrels=1.0,
z1=None,
z2=None,
height=None,
end_points=None,
exchange_lines=None,
enquiry_points=None,
invert_elevations=None,
#culvert_slope=None,
apron=0.1,
manning=0.013,
enquiry_gap=0.0,
smoothing_timescale=0.0,
use_momentum_jet=True,
use_velocity_head=True,
description=None,
label=None,
structure_type='weir_orifice_trapezoid',
logging=False,
verbose=False,
master_proc=0,
procs=None):
# If not parallel domain then allocate serial Weir orifice trapezoid operator
if isinstance(domain, Parallel_domain) is False:
if verbose:
print "Allocating non parallel weir orifice trapzezoid operator ....."
return anuga.structures.weir_orifice_trapezoid_operator.Weir_orifice_trapezoid_operator(
domain=domain,
losses=losses,
width=width,
height=height,
blockage=blockage,
barrels=barrels,
z1=z1,
z2=z2,
#culvert_slope=culvert_slope,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevations=invert_elevations,
apron=apron,
manning=manning,
enquiry_gap=enquiry_gap,
smoothing_timescale=smoothing_timescale,
use_momentum_jet=use_momentum_jet,
use_velocity_head=use_velocity_head,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose)
import pypar
if procs is None:
procs = range(0, pypar.size())
myid = pypar.rank()
end_points = ensure_numeric(end_points)
exchange_lines = ensure_numeric(exchange_lines)
enquiry_points = ensure_numeric(enquiry_points)
if height is None:
height = width
diameter = None
if apron is None:
apron = width
# Calculate location of inlet enquiry points and exchange lines
if myid == master_proc:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = __process_skew_culvert(
exchange_lines, end_points, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(enquiry_points_tmp, i)
elif end_points is not None:
exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert(
end_points, width, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(exchange_lines_tmp, i)
pypar.send(enquiry_points_tmp, i)
else:
raise Exception, 'Define either exchange_lines or end_points'
else:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = pypar.receive(master_proc)
elif end_points is not None:
exchange_lines_tmp = pypar.receive(master_proc)
enquiry_points_tmp = pypar.receive(master_proc)
# Determine processors associated with first inlet
line0 = exchange_lines_tmp[0]
enquiry_point0 = enquiry_points_tmp[0]
alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs(
domain,
line0,
enquiry_point=enquiry_point0,
master_proc=master_proc,
procs=procs,
verbose=verbose)
# Determine processors associated with second inlet
line1 = exchange_lines_tmp[1]
enquiry_point1 = enquiry_points_tmp[1]
alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs(
domain,
line1,
enquiry_point=enquiry_point1,
master_proc=master_proc,
procs=procs,
verbose=verbose)
structure_procs = list(set(inlet0_procs + inlet1_procs))
inlet_master_proc = [inlet0_master_proc, inlet1_master_proc]
inlet_procs = [inlet0_procs, inlet1_procs]
enquiry_proc = [enquiry0_proc, enquiry1_proc]
if myid == master_proc and verbose:
print "Parallel Weir Orifice Trapezoid Operator ============================="
print "Structure Master Proc is P" + str(inlet0_master_proc)
print "Structure Procs are P" + str(structure_procs)
print "Inlet Master Procs are P" + str(inlet_master_proc)
print "Inlet Procs are P" + str(inlet_procs[0]) + " and " + str(
inlet_procs[1])
print "Inlet Enquiry Procs are P" + str(enquiry_proc)
print "Enquiry Points are " + str(enquiry_point0) + " and " + str(
enquiry_point1)
print "Inlet Exchange Lines are " + str(line0) + " and " + str(line1)
print "========================================================"
if alloc0 or alloc1:
return Parallel_Weir_orifice_trapezoid_operator(
domain=domain,
losses=losses,
width=width,
height=height,
blockage=blockage,
barrels=barrels,
z1=z1,
z2=z2,
#culvert_slope=culvert_slope,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevations=invert_elevations,
apron=apron,
manning=manning,
enquiry_gap=enquiry_gap,
smoothing_timescale=smoothing_timescale,
use_momentum_jet=use_momentum_jet,
use_velocity_head=use_velocity_head,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose,
master_proc=inlet0_master_proc,
procs=structure_procs,
inlet_master_proc=inlet_master_proc,
inlet_procs=inlet_procs,
enquiry_proc=enquiry_proc)
else:
return None
def allocate_inlet_procs(domain,
poly,
enquiry_point=None,
master_proc=0,
procs=None,
verbose=False):
import pypar
if procs is None:
procs = range(0, pypar.size())
myid = pypar.rank()
vertex_coordinates = domain.get_full_vertex_coordinates(absolute=True)
domain_centroids = domain.centroid_coordinates
size = 0
has_enq_point = False
numprocs = pypar.size()
inlet_procs = []
max_size = -1
inlet_master_proc = -1
inlet_enq_proc = -1
# Calculate the number of points of the line inside full polygon
#tri_id = line_intersect(vertex_coordinates, poly)
if len(poly) == 2: # poly is a line
if verbose: print "======================"
tri_id = line_intersect(vertex_coordinates, poly)
else: # poly is a polygon
if verbose: print "+++++++++++++++++++++++"
tris_0 = line_intersect(vertex_coordinates, [poly[0], poly[1]])
tris_1 = inside_polygon(domain_centroids, poly)
tri_id = num.union1d(tris_0, tris_1)
if verbose:
print "P%d has %d triangles in poly %s" % (myid, len(tri_id), poly)
size = len(tri_id)
if enquiry_point is not None:
try:
k = domain.get_triangle_containing_point(enquiry_point)
if domain.tri_full_flag[k] == 1:
size = size + 1
has_enq_point = True
if verbose:
print "P%d has enq point %s" % (myid, enquiry_point)
else:
if verbose:
print "P%d contains ghost copy of enq point %s" % (
myid, enquiry_point)
has_enq_point = False
except:
if verbose:
print "P%d does not contain enq point %s" % (myid,
enquiry_point)
has_enq_point = False
if myid == master_proc:
# Recieve size of overlap from each processor
# Initialize line_master_proc and inlet_procs
if size > 0:
inlet_procs = [master_proc]
max_size = size
inlet_master_proc = master_proc
if has_enq_point:
inlet_enq_proc = master_proc
# Recieve size of overlap
for i in procs:
if i == master_proc: continue
x = pypar.receive(i)
y = pypar.receive(i)
if x > 0:
inlet_procs.append(i)
# Choose inlet_master_proc as the one with the most overlap
if x > max_size:
max_size = x
inlet_master_proc = i
if y is True:
assert inlet_enq_proc == -1, "Enquiry point correspond to more than one proc"
inlet_enq_proc = i
assert len(inlet_procs) > 0, "Line does not intersect any domain"
assert inlet_master_proc >= 0, "No master processor assigned"
if enquiry_point is not None:
msg = "Enquiry point %s doesn't intersect mesh, maybe inside a building, try reducing enquiry_gap" % str(
enquiry_point)
if inlet_enq_proc < 0:
raise Exception(msg)
# Send inlet_master_proc and inlet_procs to all processors in inlet_procs
for i in procs:
if i != master_proc:
pypar.send(inlet_master_proc, i)
pypar.send(inlet_procs, i)
pypar.send(inlet_enq_proc, i)
else:
pypar.send(size, master_proc)
pypar.send(has_enq_point, master_proc)
inlet_master_proc = pypar.receive(master_proc)
inlet_procs = pypar.receive(master_proc)
inlet_enq_proc = pypar.receive(master_proc)
if has_enq_point:
assert inlet_enq_proc == myid, "Enquiry found in proc, but not declared globally"
if size > 0:
return True, inlet_master_proc, inlet_procs, inlet_enq_proc
else:
return False, inlet_master_proc, inlet_procs, inlet_enq_proc
def Internal_boundary_operator(domain,
internal_boundary_function,
width=1.,
height=1.,
end_points=None,
exchange_lines=None,
enquiry_points=None,
invert_elevation=None,
apron=0.0,
enquiry_gap=0.0,
use_velocity_head=False,
zero_outflow_momentum=False,
force_constant_inlet_elevations=True,
smoothing_timescale=0.0,
compute_discharge_implicitly=True,
description=None,
label=None,
structure_type='internal_boundary',
logging=False,
verbose=True,
master_proc=0,
procs=None,
inlet_master_proc=[0, 0],
inlet_procs=None,
enquiry_proc=[0, 0]):
# If not parallel domain then allocate serial Internal boundary operator
if isinstance(domain, Parallel_domain) is False:
if verbose:
print "Allocating non parallel internal_boundary operator ....."
return anuga.structures.internal_boundary_operator.Internal_boundary_operator(
domain=domain,
internal_boundary_function=internal_boundary_function,
width=width,
height=height,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevation=invert_elevation,
apron=apron,
enquiry_gap=enquiry_gap,
use_velocity_head=use_velocity_head,
zero_outflow_momentum=zero_outflow_momentum,
force_constant_inlet_elevations=force_constant_inlet_elevations,
smoothing_timescale=smoothing_timescale,
compute_discharge_implicitly=compute_discharge_implicitly,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose)
import pypar
if procs is None:
procs = range(0, pypar.size())
myid = pypar.rank()
end_points = ensure_numeric(end_points)
exchange_lines = ensure_numeric(exchange_lines)
enquiry_points = ensure_numeric(enquiry_points)
if height is None:
height = width
diameter = None
if apron is None:
apron = width
# Calculate location of inlet enquiry points and exchange lines
if myid == master_proc:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = __process_skew_culvert(
exchange_lines, end_points, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(enquiry_points_tmp, i)
elif end_points is not None:
exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert(
end_points, width, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(exchange_lines_tmp, i)
pypar.send(enquiry_points_tmp, i)
else:
raise Exception, 'Define either exchange_lines or end_points'
else:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = pypar.receive(master_proc)
elif end_points is not None:
exchange_lines_tmp = pypar.receive(master_proc)
enquiry_points_tmp = pypar.receive(master_proc)
# Determine processors associated with first inlet
line0 = exchange_lines_tmp[0]
enquiry_point0 = enquiry_points_tmp[0]
alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs(
domain,
line0,
enquiry_point=enquiry_point0,
master_proc=master_proc,
procs=procs,
verbose=verbose)
# Determine processors associated with second inlet
line1 = exchange_lines_tmp[1]
enquiry_point1 = enquiry_points_tmp[1]
alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs(
domain,
line1,
enquiry_point=enquiry_point1,
master_proc=master_proc,
procs=procs,
verbose=verbose)
structure_procs = list(set(inlet0_procs + inlet1_procs))
inlet_master_proc = [inlet0_master_proc, inlet1_master_proc]
inlet_procs = [inlet0_procs, inlet1_procs]
enquiry_proc = [enquiry0_proc, enquiry1_proc]
if myid == master_proc and verbose:
print "Parallel Internal boundary Operator ============================="
print "Structure Master Proc is P" + str(inlet0_master_proc)
print "Structure Procs are P" + str(structure_procs)
print "Inlet Master Procs are P" + str(inlet_master_proc)
print "Inlet Procs are P" + str(inlet_procs[0]) + " and " + str(
inlet_procs[1])
print "Inlet Enquiry Procs are P" + str(enquiry_proc)
print "Enquiry Points are " + str(enquiry_point0) + " and " + str(
enquiry_point1)
print "Inlet Exchange Lines are " + str(line0) + " and " + str(line1)
print "========================================================"
if alloc0 or alloc1:
return Parallel_Internal_boundary_operator(
domain=domain,
internal_boundary_function=internal_boundary_function,
width=width,
height=height,
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevation=invert_elevation,
apron=apron,
enquiry_gap=enquiry_gap,
use_velocity_head=use_velocity_head,
zero_outflow_momentum=zero_outflow_momentum,
force_constant_inlet_elevations=force_constant_inlet_elevations,
smoothing_timescale=smoothing_timescale,
compute_discharge_implicitly=compute_discharge_implicitly,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose,
master_proc=inlet0_master_proc,
procs=structure_procs,
inlet_master_proc=inlet_master_proc,
inlet_procs=inlet_procs,
enquiry_proc=enquiry_proc)
else:
return None
def __call__(self):
timestep = self.domain.get_timestep()
Q, barrel_speed, outlet_depth = self.discharge_routine()
# Get attributes of Inflow inlet, all procs associated with inlet must call
if self.myid in self.inlet_procs[self.inflow_index]:
old_inflow_depth = self.inlets[self.inflow_index].get_global_average_depth()
old_inflow_stage = self.inlets[self.inflow_index].get_global_average_stage()
old_inflow_xmom = self.inlets[self.inflow_index].get_global_average_xmom()
old_inflow_ymom = self.inlets[self.inflow_index].get_global_average_ymom()
inflow_area = self.inlets[self.inflow_index].get_global_area()
# Master proc of inflow inlet sends attributes to master proc of structure
if self.myid == self.master_proc:
if self.myid != self.inlet_master_proc[self.inflow_index]:
old_inflow_depth = pypar.receive(self.inlet_master_proc[self.inflow_index])
old_inflow_stage = pypar.receive(self.inlet_master_proc[self.inflow_index])
old_inflow_xmom = pypar.receive(self.inlet_master_proc[self.inflow_index])
old_inflow_ymom = pypar.receive(self.inlet_master_proc[self.inflow_index])
inflow_area = pypar.receive(self.inlet_master_proc[self.inflow_index])
elif self.myid == self.inlet_master_proc[self.inflow_index]:
pypar.send(old_inflow_depth, self.master_proc)
pypar.send(old_inflow_stage, self.master_proc)
pypar.send(old_inflow_xmom, self.master_proc)
pypar.send(old_inflow_ymom, self.master_proc)
pypar.send(inflow_area, self.master_proc)
# Implement the update of flow over a timestep by
# using a semi-implict update. This ensures that
# the update does not create a negative depth
# Master proc of structure only
if self.myid == self.master_proc:
if old_inflow_depth > 0.0 :
dt_Q_on_d = timestep*Q/old_inflow_depth
else:
dt_Q_on_d = 0.0
# Check whether we should use the wet-dry Q adjustment (where Q is
# multiplied by new_inflow_depth/old_inflow_depth)
always_use_Q_wetdry_adjustment = self.always_use_Q_wetdry_adjustment
# Always use it if we are near wet-dry
use_Q_wetdry_adjustment = ((always_use_Q_wetdry_adjustment) |\
(old_inflow_depth*inflow_area <= Q*timestep))
factor = 1.0/(1.0 + dt_Q_on_d/inflow_area)
if use_Q_wetdry_adjustment:
new_inflow_depth = old_inflow_depth*factor
if old_inflow_depth > 0.:
timestep_star = timestep*new_inflow_depth/old_inflow_depth
else:
timestep_star = 0.
else:
new_inflow_depth = old_inflow_depth - timestep*Q/inflow_area
timestep_star = timestep
#new_inflow_xmom = old_inflow_xmom*factor
#new_inflow_ymom = old_inflow_ymom*factor
if(self.use_old_momentum_method):
# This method is here for consistency with the old version of the
# routine
new_inflow_xmom = old_inflow_xmom*factor
new_inflow_ymom = old_inflow_ymom*factor
else:
# For the momentum balance, note that Q also transports the velocity,
# which has an average value of new_inflow_mom/depth (or old_inflow_mom/depth).
#
# new_inflow_xmom*inflow_area =
# old_inflow_xmom*inflow_area -
# timestep*Q*(new_inflow_xmom/old_inflow_depth)
# and:
# new_inflow_ymom*inflow_area =
# old_inflow_ymom*inflow_area -
# timestep*Q*(new_inflow_ymom/old_inflow_depth)
#
# The choice of new_inflow_mom in the final term might be
# replaced with old_inflow_mom.
#
# The units balance: (m^2/s)*(m^2) = (m^2/s)*(m^2) - s*(m^3/s)*(m^2/s)*(m^(-1))
#
if old_inflow_depth > 0.:
if use_Q_wetdry_adjustment:
factor2 = 1.0/(1.0 + dt_Q_on_d*new_inflow_depth/(old_inflow_depth*inflow_area))
else:
factor2 = 1.0/(1.0 + timestep*Q/(old_inflow_depth*inflow_area))
else:
factor2 = 0.
new_inflow_xmom = old_inflow_xmom*factor2
new_inflow_ymom = old_inflow_ymom*factor2
# Master proc of structure sends new inflow attributes to all inflow inlet processors
if self.myid == self.master_proc:
for i in self.inlet_procs[self.inflow_index]:
if i == self.master_proc: continue
pypar.send(new_inflow_depth, i)
pypar.send(new_inflow_xmom, i)
pypar.send(new_inflow_ymom, i)
elif self.myid in self.inlet_procs[self.inflow_index]:
new_inflow_depth = pypar.receive(self.master_proc)
new_inflow_xmom = pypar.receive(self.master_proc)
new_inflow_ymom = pypar.receive(self.master_proc)
# Inflow inlet procs sets new attributes
if self.myid in self.inlet_procs[self.inflow_index]:
self.inlets[self.inflow_index].set_depths(new_inflow_depth)
self.inlets[self.inflow_index].set_xmoms(new_inflow_xmom)
self.inlets[self.inflow_index].set_ymoms(new_inflow_ymom)
# Get outflow inlet attributes, all processors associated with outflow inlet must call
if self.myid in self.inlet_procs[self.outflow_index]:
outflow_area = self.inlets[self.outflow_index].get_global_area()
outflow_average_depth = self.inlets[self.outflow_index].get_global_average_depth()
outflow_outward_culvert_vector = self.inlets[self.outflow_index].outward_culvert_vector
outflow_average_xmom = self.inlets[self.outflow_index].get_global_average_xmom()
outflow_average_ymom = self.inlets[self.outflow_index].get_global_average_ymom()
# Master proc of outflow inlet sends attribute to master proc of structure
if self.myid == self.master_proc:
if self.myid != self.inlet_master_proc[self.outflow_index]:
outflow_area = pypar.receive(self.inlet_master_proc[self.outflow_index])
outflow_average_depth = pypar.receive(self.inlet_master_proc[self.outflow_index])
outflow_outward_culvert_vector = pypar.receive(self.inlet_master_proc[self.outflow_index])
outflow_average_xmom = pypar.receive(self.inlet_master_proc[self.outflow_index])
outflow_average_ymom = pypar.receive(self.inlet_master_proc[self.outflow_index])
elif self.myid == self.inlet_master_proc[self.outflow_index]:
pypar.send(outflow_area, self.master_proc)
pypar.send(outflow_average_depth, self.master_proc)
pypar.send(outflow_outward_culvert_vector, self.master_proc)
pypar.send(outflow_average_xmom, self.master_proc)
pypar.send(outflow_average_ymom, self.master_proc)
# Master proc of structure computes new outflow attributes
if self.myid == self.master_proc:
loss = (old_inflow_depth - new_inflow_depth)*inflow_area
xmom_loss = (old_inflow_xmom - new_inflow_xmom)*inflow_area
ymom_loss = (old_inflow_ymom - new_inflow_ymom)*inflow_area
# set outflow
outflow_extra_depth = Q*timestep_star/outflow_area
outflow_direction = - outflow_outward_culvert_vector
#outflow_extra_momentum = outflow_extra_depth*barrel_speed*outflow_direction
gain = outflow_extra_depth*outflow_area
# Update Stats
self.discharge = Q*timestep_star/timestep #outflow_extra_depth*self.outflow.get_area()/timestep
self.discharge_abs_timemean += Q*timestep_star/self.domain.yieldstep
self.velocity = barrel_speed #self.discharge/outlet_depth/self.width
new_outflow_depth = outflow_average_depth + outflow_extra_depth
self.outlet_depth = new_outflow_depth
#if self.use_momentum_jet :
# # FIXME (SR) Review momentum to account for possible hydraulic jumps at outlet
# #new_outflow_xmom = outflow.get_average_xmom() + outflow_extra_momentum[0]
# #new_outflow_ymom = outflow.get_average_ymom() + outflow_extra_momentum[1]
# new_outflow_xmom = barrel_speed*new_outflow_depth*outflow_direction[0]
# new_outflow_ymom = barrel_speed*new_outflow_depth*outflow_direction[1]
#else:
# #new_outflow_xmom = outflow.get_average_xmom()
# #new_outflow_ymom = outflow.get_average_ymom()
# new_outflow_xmom = 0.0
# new_outflow_ymom = 0.0
if self.use_momentum_jet:
# FIXME (SR) Review momentum to account for possible hydraulic jumps at outlet
# FIXME (GD) Depending on barrel speed I think this will be either
# a source or sink of momentum (considering the momentum losses
# above). Might not always be reasonable.
#new_outflow_xmom = self.outflow.get_average_xmom() + outflow_extra_momentum[0]
#new_outflow_ymom = self.outflow.get_average_ymom() + outflow_extra_momentum[1]
new_outflow_xmom = barrel_speed*new_outflow_depth*outflow_direction[0]
new_outflow_ymom = barrel_speed*new_outflow_depth*outflow_direction[1]
elif self.zero_outflow_momentum:
new_outflow_xmom = 0.0
new_outflow_ymom = 0.0
#new_outflow_xmom = outflow.get_average_xmom()
#new_outflow_ymom = outflow.get_average_ymom()
else:
# Add the momentum lost from the inflow to the outflow. For
# structures where barrel_speed is unknown + direction doesn't
# change from inflow to outflow
new_outflow_xmom = outflow_average_xmom + xmom_loss/outflow_area
new_outflow_ymom = outflow_average_ymom + ymom_loss/outflow_area
# master proc of structure sends outflow attributes to all outflow procs
for i in self.inlet_procs[self.outflow_index]:
if i == self.myid: continue
pypar.send(new_outflow_depth, i)
pypar.send(new_outflow_xmom, i)
pypar.send(new_outflow_ymom, i)
# outflow inlet procs receives new outflow attributes
elif self.myid in self.inlet_procs[self.outflow_index]:
new_outflow_depth = pypar.receive(self.master_proc)
new_outflow_xmom = pypar.receive(self.master_proc)
new_outflow_ymom = pypar.receive(self.master_proc)
# outflow inlet procs sets new outflow attributes
if self.myid in self.inlet_procs[self.outflow_index]:
self.inlets[self.outflow_index].set_depths(new_outflow_depth)
self.inlets[self.outflow_index].set_xmoms(new_outflow_xmom)
self.inlets[self.outflow_index].set_ymoms(new_outflow_ymom)
def send(data, destination, tag=0):
pypar.send(data, destination=destination, tag=tag)
#
# Communication
#
if numprocs > 1:
#
# Processor 0 gathers all results and merge them
#
if myid == 0:
for id in range(1,numprocs):
print "P%d receving from P%d" %(0, id)
x = x + pypar.receive(id) #Add up (would be more complex in general)
# All other processors send their results back to processor 0
#
else:
print "P%d sending to P%d" %(myid, 0)
pypar.send(x, 0)
print "Proc %d after communication" %myid
#
# Compute overall average and report
#
if myid == 0:
avg = x/len(data)
print "Global average is %.4f" %avg
pypar.finalize()
def set_stages_evenly(self, volume):
""" Distribute volume of water over
inlet exchange region so that stage is level
"""
# WARNING: requires synchronization, must be called by all procs associated
# with this inlet
import pypar
centroid_coordinates = self.domain.get_full_centroid_coordinates(
absolute=True)
areas = self.get_areas()
stages = self.get_stages()
stages_order = stages.argsort()
# PETE: send stages and areas, apply merging procedure
s_areas = {}
s_stages = {}
s_stages_order = {}
total_stages = len(stages)
if self.myid == self.master_proc:
s_areas[self.myid] = areas
s_stages[self.myid] = stages
s_stages_order[self.myid] = stages_order
# Recieve areas, stages, and stages order
for i in self.procs:
if i != self.master_proc:
s_areas[i] = pypar.receive(i)
s_stages[i] = pypar.receive(i)
s_stages_order[i] = pypar.receive(i)
total_stages = total_stages + len(s_stages[i])
else:
# Send areas, stages, and stages order to master proc of inlet
pypar.send(areas, self.master_proc)
pypar.send(stages, self.master_proc)
pypar.send(stages_order, self.master_proc)
# merge sorted stage order
if self.myid == self.master_proc:
pos = {}
summed_volume = 0.
summed_areas = 0.
prev_stage = 0.
num_stages = 0.
first = True
for i in self.procs:
pos[i] = 0
while num_stages < total_stages:
# Determine current minimum stage of all the processors in s_stages
num_stages = num_stages + 1
current_stage = num.finfo(num.float32).max
index = -1
for i in self.procs:
if pos[i] >= len(s_stages[i]):
continue
if s_stages[i][s_stages_order[i][pos[i]]] < current_stage:
current_stage = s_stages[i][s_stages_order[i][pos[i]]]
index = i
# If first iteration, then only update summed_areas, position, and prev|current stage
if first:
first = False
summed_areas = s_areas[index][s_stages_order[index][
pos[index]]]
pos[index] = pos[index] + 1
prev_stage = current_stage
continue
assert index >= 0, "Index out of bounds"
# Update summed volume and summed areas
tmp_volume = summed_volume + (summed_areas *
(current_stage - prev_stage))
# Terminate if volume exceeded
if tmp_volume >= volume:
break
summed_areas = summed_areas + s_areas[index][
s_stages_order[index][pos[index]]]
pos[index] = pos[index] + 1
summed_volume = tmp_volume
# Update position of index processor and current stage
prev_stage = current_stage
# Calculate new stage
new_stage = prev_stage + (volume - summed_volume) / summed_areas
# Send postion and new stage to all processors
for i in self.procs:
if i != self.master_proc:
pypar.send(pos[i], i)
pypar.send(new_stage, i)
# Update own depth
stages[stages_order[0:pos[self.myid]]] = new_stage
else:
pos = pypar.receive(self.master_proc)
new_stage = pypar.receive(self.master_proc)
stages[stages_order[0:pos]] = new_stage
self.set_stages(stages)
stages = self.get_stages()
stages_order = stages.argsort()
def send( s ): if s.participant: mpi.send( s.get_data(), 0 )
for i in range(MAXI):
m = BLOCK * i + 1
noelem[i] = m
pypar.barrier() # Synchronize
if myid == 0:
#
# Main process
#
t1 = pypar.time()
if method == 0:
pypar.send(A[:m], destination=1, tag=msgid, vanilla=vanilla)
C = pypar.receive(numprocs - 1, tag=msgid, vanilla=vanilla)
elif method == 1:
pypar.send(A[:m],
use_buffer=True,
destination=1,
tag=msgid,
vanilla=vanilla)
C = pypar.receive(numprocs - 1,
buffer=A[:m],
tag=msgid,
vanilla=vanilla)
elif method == 2:
pypar.send(A[:m],
use_buffer=True,
destination=1,
atexit.register(finalize)
pypar.barrier()
test_points = []
if myid == 0:
if verbose: print 'PARALLEL START'
random.seed(1001)
for i in range(samples):
x = random.randrange(0,1000)/1000.0 * length
y = random.randrange(0,1000)/1000.0 * width
point = [x, y]
test_points.append(point)
for i in range(1,numprocs):
pypar.send(test_points, i)
else:
test_points = pypar.receive(0)
if myid == 0:
control_data, success = run_simulation(parallel=False, test_points = test_points, verbose = verbose)
for proc in range(1,numprocs):
pypar.send(control_data, proc)
else:
control_data = pypar.receive(0)
pypar.barrier()
_, success = run_simulation(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
numproc = pypar.size()
myid = pypar.rank()
node = pypar.Get_processor_name()
print "I am proc %d of %d on node %s" % (myid, numproc, node)
if numproc < 2:
print "Demo must run on at least 2 processors to continue"
pypar.abort()
if myid == 0:
msg = Numeric.array([0], typ)
print 'Processor 0 sending message "%s" to processor %d' % (str(msg), 1)
pypar.send(msg, 1)
msg, status = pypar.receive(numproc - 1, return_status=True)
print 'Processor 0 received message "%s" from processor %d' % (str(msg),
numproc - 1)
print 'Size of msg was %d bytes' % (status.bytes())
else:
source = myid - 1
destination = (myid + 1) % numproc
msg, status = pypar.receive(source, return_status=True)
print 'Processor %d received message "%s" from processor %d'\
%(myid, str(msg), source)
print 'Size of msg was %d bytes' % (status.bytes())
def discharge_routine(self):
"""
Get info from inlets and then call sequential function
"""
import pypar
local_debug = False
#Send attributes of both enquiry points to the master proc
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[0]:
enq_total_energy0 = self.inlets[0].get_enquiry_total_energy()
enq_stage0 = self.inlets[0].get_enquiry_stage()
else:
enq_total_energy0 = pypar.receive(self.enquiry_proc[0])
enq_stage0 = pypar.receive(self.enquiry_proc[0])
if self.myid == self.enquiry_proc[1]:
enq_total_energy1 = self.inlets[1].get_enquiry_total_energy()
enq_stage1 = self.inlets[1].get_enquiry_stage()
else:
enq_total_energy1 = pypar.receive(self.enquiry_proc[1])
enq_stage1 = pypar.receive(self.enquiry_proc[1])
else:
if self.myid == self.enquiry_proc[0]:
pypar.send(self.inlets[0].get_enquiry_total_energy(), self.master_proc)
pypar.send(self.inlets[0].get_enquiry_stage(), self.master_proc)
if self.myid == self.enquiry_proc[1]:
pypar.send(self.inlets[1].get_enquiry_total_energy(), self.master_proc)
pypar.send(self.inlets[1].get_enquiry_stage(), self.master_proc)
# Determine the direction of the flow
if self.myid == self.master_proc:
if self.use_velocity_head:
self.delta_total_energy = enq_total_energy0 - enq_total_energy1
else:
self.delta_total_energy = enq_stage0 - enq_stage1
self.inflow_index = 0
self.outflow_index = 1
# master proc orders reversal if applicable
if self.myid == self.master_proc:
# May/June 2014 -- change the driving forces gradually, with forward euler timestepping
#
forward_Euler_smooth=True
if(forward_Euler_smooth):
# To avoid 'overshoot' we ensure ts<1.
if(self.domain.timestep>0.):
ts=self.domain.timestep/max(self.domain.timestep, self.smoothing_timescale,1.0e-06)
else:
# This case is included in the serial version, which ensures the unit tests pass
# even when domain.timestep=0.0.
# Note though the discontinuous behaviour as domain.timestep-->0. from above
ts=1.0
self.smooth_delta_total_energy=self.smooth_delta_total_energy+\
ts*(self.delta_total_energy-self.smooth_delta_total_energy)
else:
# Use backward euler -- the 'sensible' ts limitation is different in this case
# ts --> Inf is reasonable and corresponds to the 'nosmoothing' case
ts=self.domain.timestep/max(self.smoothing_timescale, 1.0e-06)
self.smooth_delta_total_energy = (self.smooth_delta_total_energy+ts*(self.delta_total_energy))/(1.+ts)
# Reverse the inflow and outflow direction?
if self.smooth_delta_total_energy < 0:
self.inflow_index = 1
self.outflow_index = 0
#self.delta_total_energy = -self.delta_total_energy
self.delta_total_energy = -self.smooth_delta_total_energy
for i in self.procs:
if i == self.master_proc: continue
pypar.send(True, i)
else:
self.delta_total_energy = self.smooth_delta_total_energy
for i in self.procs:
if i == self.master_proc: continue
pypar.send(False, i)
#print "ZZZZ: Delta total energy = %f" %(self.delta_total_energy)
else:
reverse = pypar.receive(self.master_proc)
if reverse:
self.inflow_index = 1
self.outflow_index = 0
# Get attribute from inflow enquiry point
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[self.inflow_index]:
inflow_enq_depth = self.inlets[self.inflow_index].get_enquiry_depth()
inflow_enq_specific_energy = self.inlets[self.inflow_index].get_enquiry_specific_energy()
else:
inflow_enq_depth = pypar.receive(self.enquiry_proc[self.inflow_index])
inflow_enq_specific_energy = pypar.receive(self.enquiry_proc[self.inflow_index])
else:
if self.myid == self.enquiry_proc[self.inflow_index]:
pypar.send(self.inlets[self.inflow_index].get_enquiry_depth(), self.master_proc)
pypar.send(self.inlets[self.inflow_index].get_enquiry_specific_energy(), self.master_proc)
# Get attribute from outflow enquiry point
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[self.outflow_index]:
outflow_enq_depth = self.inlets[self.outflow_index].get_enquiry_depth()
else:
outflow_enq_depth = pypar.receive(self.enquiry_proc[self.outflow_index])
#print "ZZZZZ: outflow_enq_depth = %f" %(outflow_enq_depth)
else:
if self.myid == self.enquiry_proc[self.outflow_index]:
pypar.send(self.inlets[self.outflow_index].get_enquiry_depth(), self.master_proc)
# Master proc computes return values
if self.myid == self.master_proc:
#inflow_enq_specific_energy
if inflow_enq_depth > 0.01: #this value was 0.01:
if local_debug:
anuga.log.critical('Specific E & Deltat Tot E = %s, %s'
% (str(inflow_enq_specific_energy),
str(self.delta_total_energy)))
anuga.log.critical('culvert type = %s' % str(culvert_type))
# Water has risen above inlet
msg = 'Specific energy at inlet is negative'
assert inflow_enq_specific_energy >= 0.0, msg
if self.use_velocity_head :
self.driving_energy = inflow_enq_specific_energy
else:
self.driving_energy = inflow_enq_depth
Q, barrel_velocity, outlet_culvert_depth, flow_area, case = \
boyd_box_function(depth =self.culvert_height,
width =self.culvert_width,
flow_width =self.culvert_width,
length =self.culvert_length,
blockage =self.culvert_blockage,
barrels =self.culvert_barrels,
driving_energy =self.driving_energy,
delta_total_energy =self.delta_total_energy,
outlet_enquiry_depth=outflow_enq_depth,
sum_loss =self.sum_loss,
manning =self.manning)
################################################
# Smooth discharge. This can reduce oscillations
#
# NOTE: The sign of smooth_Q assumes that
# self.inflow_index=0 and self.outflow_index=1
# , whereas the sign of Q is always positive
Qsign=(self.outflow_index-self.inflow_index) # To adjust sign of Q
if(forward_Euler_smooth):
self.smooth_Q = self.smooth_Q +ts*(Q*Qsign-self.smooth_Q)
else:
# Try implicit euler method
self.smooth_Q = (self.smooth_Q+ts*(Q*Qsign))/(1.+ts)
if numpy.sign(self.smooth_Q)!=Qsign:
# The flow direction of the 'instantaneous Q' based on the
# 'smoothed delta_total_energy' is not the same as the
# direction of smooth_Q. To prevent 'jumping around', let's
# set Q to zero
Q=0.
else:
Q = min(abs(self.smooth_Q), Q) #abs(self.smooth_Q)
barrel_velocity=Q/flow_area
# END CODE BLOCK for DEPTH > Required depth for CULVERT Flow
else: # self.inflow.get_enquiry_depth() < 0.01:
Q = barrel_velocity = outlet_culvert_depth = 0.0
case = 'Inlet dry'
self.case = case
# Temporary flow limit
if barrel_velocity > self.max_velocity:
barrel_velocity = self.max_velocity
Q = flow_area * barrel_velocity
return Q, barrel_velocity, outlet_culvert_depth
else:
return None, None, None
def discharge_routine_explicit(self):
import pypar
local_debug = False
# If the structure has been closed, then no water gets through
if self.height <= 0.0:
if self.myid == self.master_proc:
Q = 0.0
barrel_velocity = 0.0
outlet_culvert_depth = 0.0
self.case = "Structure is blocked"
self.inflow = self.inlets[0]
self.outflow = self.inlets[1]
return Q, barrel_velocity, outlet_culvert_depth
else:
return None, None, None
#Send attributes of both enquiry points to the master proc
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[0]:
enq_total_energy0 = self.inlets[0].get_enquiry_total_energy()
enq_stage0 = self.inlets[0].get_enquiry_stage()
else:
enq_total_energy0 = pypar.receive(self.enquiry_proc[0])
enq_stage0 = pypar.receive(self.enquiry_proc[0])
if self.myid == self.enquiry_proc[1]:
enq_total_energy1 = self.inlets[1].get_enquiry_total_energy()
enq_stage1 = self.inlets[1].get_enquiry_stage()
else:
enq_total_energy1 = pypar.receive(self.enquiry_proc[1])
enq_stage1 = pypar.receive(self.enquiry_proc[1])
else:
if self.myid == self.enquiry_proc[0]:
pypar.send(self.inlets[0].get_enquiry_total_energy(),
self.master_proc)
pypar.send(self.inlets[0].get_enquiry_stage(),
self.master_proc)
if self.myid == self.enquiry_proc[1]:
pypar.send(self.inlets[1].get_enquiry_total_energy(),
self.master_proc)
pypar.send(self.inlets[1].get_enquiry_stage(),
self.master_proc)
# Determine the direction of the flow
if self.myid == self.master_proc:
# Variables required by anuga's structure operator which are not
# used
barrel_velocity = numpy.nan
outlet_culvert_depth = numpy.nan
flow_area = numpy.nan
case = ''
# 'Timescale' for smoothed discharge and energy
ts = self.domain.timestep / max(self.domain.timestep,
self.smoothing_timescale, 1.0e-30)
# Energy or stage as head
if self.use_velocity_head:
E0 = enq_total_energy0
E1 = enq_total_energy1
else:
E0 = enq_stage0
E1 = enq_stage1
self.delta_total_energy = E0 - E1
self.driving_energy = max(E0, E1)
# Compute 'smoothed' versions of key variables
self.smooth_delta_total_energy += ts * (
self.delta_total_energy - self.smooth_delta_total_energy)
if numpy.sign(self.smooth_delta_total_energy) != numpy.sign(
self.delta_total_energy):
self.smooth_delta_total_energy = 0.
# Compute the 'tailwater' energy from the 'headwater' energy
# and the smooth_delta_total_energy. Note if ts = 1 (no
# smoothing), then the raw inlet energies are used
if E0 >= E1:
inlet0_energy = 1.0 * E0
inlet1_energy = inlet0_energy - self.smooth_delta_total_energy
else:
inlet1_energy = 1.0 * E1
inlet0_energy = inlet1_energy + self.smooth_delta_total_energy
# Compute discharge
Q = self.internal_boundary_function(inlet0_energy, inlet1_energy)
self.smooth_Q = self.smooth_Q + ts * (Q - self.smooth_Q)
if numpy.sign(self.smooth_Q) != numpy.sign(Q):
# The flow direction of the 'instantaneous Q' based on the
# 'smoothed delta_total_energy' is not the same as the
# direction of smooth_Q. To prevent 'jumping around', let's
# set Q to zero
Q = 0.
else:
# Make Q positive (for anuga's structure operator)
Q = min(abs(self.smooth_Q), abs(Q))
else:
self.delta_total_energy = numpy.nan
self.driving_energy = numpy.nan
self.inflow_index = 0
self.outflow_index = 1
# master proc orders reversal if applicable
if self.myid == self.master_proc:
# Reverse the inflow and outflow direction?
if self.smooth_Q < 0.:
self.inflow_index = 1
self.outflow_index = 0
for i in self.procs:
if i == self.master_proc: continue
pypar.send(True, i)
else:
for i in self.procs:
if i == self.master_proc: continue
pypar.send(False, i)
else:
reverse = pypar.receive(self.master_proc)
if reverse:
self.inflow_index = 1
self.outflow_index = 0
# Master proc computes return values
if self.myid == self.master_proc:
return Q, barrel_velocity, outlet_culvert_depth
else:
return None, None, None
def run_simulation(parallel=False):
domain = create_domain_from_file(mesh_filename)
domain.set_quantity('stage', Set_Stage(756000.0, 756500.0, 2.0))
#--------------------------------------------------------------------------
# Create parallel domain if requested
#--------------------------------------------------------------------------
if parallel:
if myid == 0 and verbose: print 'DISTRIBUTING PARALLEL DOMAIN'
domain = distribute(domain)
#------------------------------------------------------------------------------
# Setup boundary conditions
# This must currently happen *after* domain has been distributed
#------------------------------------------------------------------------------
domain.store = False
Br = Reflective_boundary(domain) # Solid reflective wall
domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
#------------------------------------------------------------------------------
# Setup diagnostic arrays
#------------------------------------------------------------------------------
l1list = []
l2list = []
linflist = []
l1norm = num.zeros(3, num.float)
l2norm = num.zeros(3, num.float)
linfnorm = num.zeros(3, num.float)
recv_norm = num.zeros(3, num.float)
#------------------------------------------------------------------------------
# Evolution
#------------------------------------------------------------------------------
if parallel:
if myid == 0 and verbose: print 'PARALLEL EVOLVE'
else:
if verbose: print 'SEQUENTIAL EVOLVE'
for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
edges = domain.quantities[quantity].edge_values.take(num.flatnonzero(domain.tri_full_flag),axis=0)
l1norm[0] = l1_norm(edges[:,0])
l1norm[1] = l1_norm(edges[:,1])
l1norm[2] = l1_norm(edges[:,2])
l2norm[0] = l2_norm(edges[:,0])
l2norm[1] = l2_norm(edges[:,1])
l2norm[2] = l2_norm(edges[:,2])
linfnorm[0] = linf_norm(edges[:,0])
linfnorm[1] = linf_norm(edges[:,1])
linfnorm[2] = linf_norm(edges[:,2])
if parallel:
l2norm[0] = pow(l2norm[0], 2)
l2norm[1] = pow(l2norm[1], 2)
l2norm[2] = pow(l2norm[2], 2)
if myid == 0:
#domain.write_time()
#print edges[:,1]
for p in range(1, numprocs):
recv_norm = pypar.receive(p)
l1norm += recv_norm
recv_norm = pypar.receive(p)
l2norm += recv_norm
recv_norm = pypar.receive(p)
linfnorm[0] = max(linfnorm[0], recv_norm[0])
linfnorm[1] = max(linfnorm[1], recv_norm[1])
linfnorm[2] = max(linfnorm[2], recv_norm[2])
l2norm[0] = pow(l2norm[0], 0.5)
l2norm[1] = pow(l2norm[1], 0.5)
l2norm[2] = pow(l2norm[2], 0.5)
l1list.append(l1norm)
l2list.append(l2norm)
linflist.append(linfnorm)
else:
pypar.send(l1norm, 0)
pypar.send(l2norm, 0)
pypar.send(linfnorm, 0)
else:
#domain.write_time()
l1list.append(l1norm)
l2list.append(l2norm)
linflist.append(linfnorm)
return (l1list, l2list, linflist)
def Boyd_pipe_operator(domain,
losses,
diameter,
blockage, #added by DMP 28/7/2016
end_points=None,
exchange_lines=None,
enquiry_points=None,
invert_elevations=None,
apron=0.1,
manning=0.013,
enquiry_gap=0.0,
use_momentum_jet=True,
use_velocity_head=True,
description=None,
label=None,
structure_type='boyd_pipe',
logging=False,
verbose=False,
master_proc=0,
procs=None):
# If not parallel domain then allocate serial Boyd box operator
if isinstance(domain, Parallel_domain) is False:
if verbose: print "Allocating non parallel boyd pipe operator ....."
return anuga.structures.boyd_pipe_operator.Boyd_pipe_operator(domain=domain,
losses=losses,
diameter=diameter,
blockage=blockage, #added by DMP 28/7/2016
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevations=invert_elevations,
apron=apron,
manning=manning,
enquiry_gap=enquiry_gap,
use_momentum_jet=use_momentum_jet,
use_velocity_head=use_velocity_head,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose)
import pypar
if procs is None:
procs = range(0,pypar.size())
myid = pypar.rank()
end_points = ensure_numeric(end_points)
exchange_lines = ensure_numeric(exchange_lines)
enquiry_points = ensure_numeric(enquiry_points)
width = diameter
assert diameter is not None
if apron is None:
apron = width
# Calculate location of inlet enquiry points and exchange lines
if myid == master_proc:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = __process_skew_culvert(exchange_lines, end_points, enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(enquiry_points_tmp, i)
elif end_points is not None:
exchange_lines_tmp, enquiry_points_tmp = __process_non_skew_culvert(end_points, width,
enquiry_points, apron, enquiry_gap)
for i in procs:
if i == master_proc: continue
pypar.send(exchange_lines_tmp, i)
pypar.send(enquiry_points_tmp, i)
else:
raise Exception, 'Define either exchange_lines or end_points'
else:
if exchange_lines is not None:
exchange_lines_tmp = exchange_lines
enquiry_points_tmp = pypar.receive(master_proc)
elif end_points is not None:
exchange_lines_tmp = pypar.receive(master_proc)
enquiry_points_tmp = pypar.receive(master_proc)
# Determine processors associated with first inlet
line0 = exchange_lines_tmp[0]
enquiry_point0 = enquiry_points_tmp[0]
alloc0, inlet0_master_proc, inlet0_procs, enquiry0_proc = allocate_inlet_procs(domain, line0, enquiry_point = enquiry_point0,
master_proc = master_proc,
procs = procs, verbose=verbose)
# Determine processors associated with second inlet
line1 = exchange_lines_tmp[1]
enquiry_point1 = enquiry_points_tmp[1]
alloc1, inlet1_master_proc, inlet1_procs, enquiry1_proc = allocate_inlet_procs(domain, line1, enquiry_point = enquiry_point1,
master_proc = master_proc,
procs = procs, verbose=verbose)
structure_procs = list(set(inlet0_procs + inlet1_procs))
inlet_master_proc = [inlet0_master_proc, inlet1_master_proc]
inlet_procs = [inlet0_procs, inlet1_procs]
enquiry_proc = [enquiry0_proc, enquiry1_proc]
if myid == master_proc and verbose:
print "Parallel Boyd Pipe Operator ============================="
print "Structure Master Proc is P" + str(inlet0_master_proc)
print "Structure Procs are P" + str(structure_procs)
print "Inlet Master Procs are P" + str(inlet_master_proc)
print "Inlet Procs are P" + str(inlet_procs[0]) + " and " + str(inlet_procs[1])
print "Inlet Enquiry Procs are P" + str(enquiry_proc)
print "Enquiry Points are " + str(enquiry_point0) + " and " + str(enquiry_point1)
print "Inlet Exchange Lines are " + str(line0) + " and " + str(line1)
print "========================================================"
if alloc0 or alloc1:
return Parallel_Boyd_pipe_operator(domain=domain,
losses=losses,
diameter=diameter,
blockage=blockage, #added by DPM 24/7/2016
end_points=end_points,
exchange_lines=exchange_lines,
enquiry_points=enquiry_points,
invert_elevations=invert_elevations,
apron=apron,
manning=manning,
enquiry_gap=enquiry_gap,
use_momentum_jet=use_momentum_jet,
use_velocity_head=use_velocity_head,
description=description,
label=label,
structure_type=structure_type,
logging=logging,
verbose=verbose,
master_proc = inlet0_master_proc,
procs = structure_procs,
inlet_master_proc = inlet_master_proc,
inlet_procs = inlet_procs,
enquiry_proc = enquiry_proc)
else:
return None
processor_name = pypar.get_processor_name()
print 'Processor %d initialised on node %s' % (p, processor_name)
# Balanced work partitioning (row wise)
Mlo, Mhi = pypar.balance(M, P, p)
print 'p%d: [%d, %d], Interval length=%d' % (p, Mlo, Mhi, Mhi - Mlo)
# Parallel computation
A = calculate_region(real_min, real_max, imag_min, imag_max, kmax,
M, N, Mlo=Mlo, Mhi=Mhi)
print 'Processor %d: time = %.2f' % (p, pypar.time() - t)
# Communication phase
if p == 0:
for d in range(1, P):
A += pypar.receive(source=d)
print 'Computed region in %.2f seconds' % (pypar.time() - t)
try:
plot(A, kmax)
except:
pass
else:
pypar.send(A, destination=0)
pypar.finalize()
def discharge_routine(self):
"""
Get info from inlets and then call sequential function
"""
import pypar
local_debug = False
#Send attributes of both enquiry points to the master proc
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[0]:
enq_total_energy0 = self.inlets[0].get_enquiry_total_energy()
enq_stage0 = self.inlets[0].get_enquiry_stage()
else:
enq_total_energy0 = pypar.receive(self.enquiry_proc[0])
enq_stage0 = pypar.receive(self.enquiry_proc[0])
if self.myid == self.enquiry_proc[1]:
enq_total_energy1 = self.inlets[1].get_enquiry_total_energy()
enq_stage1 = self.inlets[1].get_enquiry_stage()
else:
enq_total_energy1 = pypar.receive(self.enquiry_proc[1])
enq_stage1 = pypar.receive(self.enquiry_proc[1])
else:
if self.myid == self.enquiry_proc[0]:
pypar.send(self.inlets[0].get_enquiry_total_energy(), self.master_proc)
pypar.send(self.inlets[0].get_enquiry_stage(), self.master_proc)
if self.myid == self.enquiry_proc[1]:
pypar.send(self.inlets[1].get_enquiry_total_energy(), self.master_proc)
pypar.send(self.inlets[1].get_enquiry_stage(), self.master_proc)
# Determine the direction of the flow
if self.myid == self.master_proc:
if self.use_velocity_head:
self.delta_total_energy = enq_total_energy0 - enq_total_energy1
else:
self.delta_total_energy = enq_stage0 - enq_stage1
self.inflow_index = 0
self.outflow_index = 1
# master proc orders reversal if applicable
if self.myid == self.master_proc:
# Reverse the inflow and outflow direction?
if self.delta_total_energy < 0:
self.inflow_index = 1
self.outflow_index = 0
self.delta_total_energy = -self.delta_total_energy
for i in self.procs:
if i == self.master_proc: continue
pypar.send(True, i)
else:
for i in self.procs:
if i == self.master_proc: continue
pypar.send(False, i)
#print "ZZZZ: Delta total energy = %f" %(self.delta_total_energy)
else:
reverse = pypar.receive(self.master_proc)
if reverse:
self.inflow_index = 1
self.outflow_index = 0
# Get attribute from inflow enquiry point
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[self.inflow_index]:
inflow_enq_depth = self.inlets[self.inflow_index].get_enquiry_depth()
inflow_enq_specific_energy = self.inlets[self.inflow_index].get_enquiry_specific_energy()
else:
inflow_enq_depth = pypar.receive(self.enquiry_proc[self.inflow_index])
inflow_enq_specific_energy = pypar.receive(self.enquiry_proc[self.inflow_index])
else:
if self.myid == self.enquiry_proc[self.inflow_index]:
pypar.send(self.inlets[self.inflow_index].get_enquiry_depth(), self.master_proc)
pypar.send(self.inlets[self.inflow_index].get_enquiry_specific_energy(), self.master_proc)
# Get attribute from outflow enquiry point
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[self.outflow_index]:
outflow_enq_depth = self.inlets[self.outflow_index].get_enquiry_depth()
else:
outflow_enq_depth = pypar.receive(self.enquiry_proc[self.outflow_index])
#print "ZZZZZ: outflow_enq_depth = %f" %(outflow_enq_depth)
else:
if self.myid == self.enquiry_proc[self.outflow_index]:
pypar.send(self.inlets[self.outflow_index].get_enquiry_depth(), self.master_proc)
# Master proc computes return values
if self.myid == self.master_proc:
#inflow_enq_specific_energy
if inflow_enq_depth > 0.01: #this value was 0.01:
if local_debug:
anuga.log.critical('Specific E & Deltat Tot E = %s, %s'
% (str(inflow_enq_specific_energy),
str(self.delta_total_energy)))
anuga.log.critical('culvert type = %s' % str(culvert_type))
# Water has risen above inlet
msg = 'Specific energy at inlet is negative'
assert inflow_enq_specific_energy >= 0.0, msg
if self.use_velocity_head :
self.driving_energy = inflow_enq_specific_energy
else:
self.driving_energy = inflow_enq_depth
Q, barrel_velocity, outlet_culvert_depth, flow_area, case = \
boyd_pipe_function(depth =inflow_enq_depth,
diameter =self.culvert_diameter,
length =self.culvert_length,
driving_energy =self.driving_energy,
delta_total_energy =self.delta_total_energy,
outlet_enquiry_depth =outflow_enq_depth,
sum_loss =self.sum_loss,
manning =self.manning)
# END CODE BLOCK for DEPTH > Required depth for CULVERT Flow
else: # self.inflow.get_enquiry_depth() < 0.01:
Q = barrel_velocity = outlet_culvert_depth = 0.0
case = 'Inlet dry'
self.case = case
# Temporary flow limit
if barrel_velocity > self.max_velocity:
barrel_velocity = self.max_velocity
Q = flow_area * barrel_velocity
return Q, barrel_velocity, outlet_culvert_depth
else:
return None, None, None
def mpi_recvsend(target_array, message_range_field, target, tag_mark):
send_range = message_range_field[0]
recv_range = message_range_field[1]
target_array[recv_range] = pypar.receive(target, tag=tag_mark)
pypar.send(target_array[send_range].copy(), target, tag=tag_mark)
def statistics(self):
# WARNING: requires synchronization, must be called by all procs associated
# with this inlet
import pypar
message = ''
tri_indices = {}
if self.myid == self.master_proc:
tri_indices[self.myid] = self.triangle_indices
for proc in self.procs:
if proc == self.master_proc: continue
tri_indices[proc] = pypar.receive(proc)
else:
pypar.send(self.triangle_indices, self.master_proc)
if self.myid == self.master_proc:
message += '=====================================\n'
message += 'Inlet\n'
message += '=====================================\n'
for proc in self.procs:
message += '======> inlet triangle indices and centres and elevation at P%d\n' % (
proc)
message += '%s' % tri_indices[proc]
message += '\n'
message += '%s' % self.domain.get_centroid_coordinates()[
tri_indices[proc]]
message += '\n'
elev = self.domain.quantities['elevation'].centroid_values[
tri_indices[proc]]
message += '%s' % elev
message += '\n'
try:
elevation_difference = elev.max() - elev.min()
except ValueError:
elevation_difference = 0.0
if not num.allclose(elevation_difference, 0.):
message += 'Elevation range of ' + str(
elevation_difference)
message += 'Warning: Non-constant inlet elevation can lead to well-balancing problems'
try:
# If the inlet does not have an enquiry point this will
# fail gracefully
message += '\n'
message += 'Enquiry point:'
message += '%s' % self.domain.get_centroid_coordinates()[
self.enquiry_index]
message += '\n'
message += 'Enquiry Index:'
message += '%s' % self.enquiry_index
message += '\n'
except:
pass
message += 'line\n'
message += '%s' % self.line
message += '\n'
return message
# work_array[i] = arange(0.0, 10.0)
# list of startParams indices to pass to the workers
work_array = range(len(startParamsList))
work_size = len(work_array)
# Dispatch jobs to worker processes
work_index = 0
num_completed = 0
# Start all worker processes
for i in range(1, min(num_processors, work_size+1)):
pypar.send(work_index, i, tag=WORK_TAG)
pypar.send(work_array[work_index], i)
print "Sent work index " + str(work_index) + " to processor " + str(i)
work_index += 1
# Receive results from each worker, and send it new data
for i in range(num_processors, work_size+1):
results, status = pypar.receive(source=pypar.any_source, tag=pypar.any_tag, return_status=True)
# save results
result_index = results.pop()
allOutputsDict[result_index] = results
index = status.tag
proc = status.source
num_completed += 1
# start next
pypar.send(work_index, proc, tag=WORK_TAG)
def discharge_routine_implicit(self):
"""
Uses semi-implicit discharge estimation:
Discharge = (1-theta)*Q(H0, T0) + theta*Q(H0 + delta_H, T0+delta_T))
where H0 = headwater stage, T0 = tailwater stage, delta_H = change in
headwater stage over a timestep, delta_T = change in tailwater stage over a
timestep, and Q is the discharge function, and theta is a constant in
[0,1] determining the degree of implicitness (currently hardcoded).
Note this is effectively assuming:
1) Q is a function of stage, not energy (so we can relate mass change directly to delta_H, delta_T). We
could generalise it to the energy case ok.
2) The stage is computed on the exchange line (or the change in
stage at the enquiry point is effectively the same as that on the exchange
line)
"""
import pypar
local_debug = False
# If the structure has been closed, then no water gets through
if self.height <= 0.0:
if self.myid == self.master_proc:
Q = 0.0
barrel_velocity = 0.0
outlet_culvert_depth = 0.0
self.case = "Structure is blocked"
self.inflow = self.inlets[0]
self.outflow = self.inlets[1]
return Q, barrel_velocity, outlet_culvert_depth
else:
return None, None, None
#Send attributes of both enquiry points to the master proc
if self.myid == self.master_proc:
if self.myid == self.enquiry_proc[0]:
enq_total_energy0 = self.inlets[0].get_enquiry_total_energy()
enq_stage0 = self.inlets[0].get_enquiry_stage()
else:
enq_total_energy0 = pypar.receive(self.enquiry_proc[0])
enq_stage0 = pypar.receive(self.enquiry_proc[0])
if self.myid == self.enquiry_proc[1]:
enq_total_energy1 = self.inlets[1].get_enquiry_total_energy()
enq_stage1 = self.inlets[1].get_enquiry_stage()
else:
enq_total_energy1 = pypar.receive(self.enquiry_proc[1])
enq_stage1 = pypar.receive(self.enquiry_proc[1])
else:
if self.myid == self.enquiry_proc[0]:
pypar.send(self.inlets[0].get_enquiry_total_energy(),
self.master_proc)
pypar.send(self.inlets[0].get_enquiry_stage(),
self.master_proc)
if self.myid == self.enquiry_proc[1]:
pypar.send(self.inlets[1].get_enquiry_total_energy(),
self.master_proc)
pypar.send(self.inlets[1].get_enquiry_stage(),
self.master_proc)
# Send inlet areas to the master proc. FIXME: Inlet areas don't change
# -- perhaps we could just do this once?
# area0
if self.myid in self.inlet_procs[0]:
area0 = self.inlets[0].get_global_area()
if self.myid == self.master_proc:
if self.myid != self.inlet_master_proc[0]:
area0 = pypar.receive(self.inlet_master_proc[0])
elif self.myid == self.inlet_master_proc[0]:
pypar.send(area0, self.master_proc)
# area1
if self.myid in self.inlet_procs[1]:
area1 = self.inlets[1].get_global_area()
if self.myid == self.master_proc:
if self.myid != self.inlet_master_proc[1]:
area1 = pypar.receive(self.inlet_master_proc[1])
elif self.myid == self.inlet_master_proc[1]:
pypar.send(area1, self.master_proc)
# Compute discharge
if self.myid == self.master_proc:
# Energy or stage as head
if self.use_velocity_head:
E0 = enq_total_energy0
E1 = enq_total_energy1
else:
E0 = enq_stage0
E1 = enq_stage1
# Variables for anuga's structure operator
self.delta_total_energy = E0 - E1
self.driving_energy = max(E0, E1)
Q0 = self.internal_boundary_function(E0, E1)
dt = self.domain.get_timestep()
if dt > 0.:
# Key constants for iterative solution
theta = 1.0
sol = numpy.array([0., 0.]) # estimate of (delta_H, delta_T)
areas = numpy.array([area0, area1])
# Use scipy root finding
def F_to_solve(sol):
Q1 = self.internal_boundary_function(
E0 + sol[0], E1 + sol[1])
discharge = (1 - theta) * Q0 + theta * Q1
output = sol * areas - discharge * dt * numpy.array(
[-1., 1.])
return (output)
final_sol = sco.root(F_to_solve, sol, method='lm').x
Q1 = self.internal_boundary_function(E0 + final_sol[0],
E1 + final_sol[1])
Q = (1.0 - theta) * Q0 + theta * Q1
else:
Q = Q0
# Smooth discharge
if dt > 0.:
ts = dt / max(dt, self.smoothing_timescale, 1.0e-30)
else:
# No smoothing
ts = 1.0
self.smooth_Q = self.smooth_Q + ts * (Q - self.smooth_Q)
else:
self.delta_total_energy = numpy.nan
self.driving_energy = numpy.nan
self.inflow_index = 0
self.outflow_index = 1
# master proc orders reversal if applicable
if self.myid == self.master_proc:
# Reverse the inflow and outflow direction?
if Q < 0.:
self.inflow_index = 1
self.outflow_index = 0
for i in self.procs:
if i == self.master_proc: continue
pypar.send(True, i)
else:
for i in self.procs:
if i == self.master_proc: continue
pypar.send(False, i)
else:
reverse = pypar.receive(self.master_proc)
if reverse:
self.inflow_index = 1
self.outflow_index = 0
# Master proc computes return values
if self.myid == self.master_proc:
# Zero Q if sign's of smooth_Q and Q differ
if numpy.sign(self.smooth_Q) != numpy.sign(Q):
Q = 0.
self.smooth_Q = 0.
else:
# Make Q positive (for anuga's structure operator)
Q = min(abs(self.smooth_Q), abs(Q))
# Variables required by anuga's structure operator which are
# not used
barrel_velocity = numpy.nan
outlet_culvert_depth = numpy.nan
return Q, barrel_velocity, outlet_culvert_depth
else:
return None, None, None
if __name__ == "__main__":
test_points = []
if myid == 0:
for i in range(samples):
x = random.randrange(0, 1000) / 1000.0 * length
y = random.randrange(0, 1000) / 1000.0 * width
point = [x, y]
test_points.append(point)
for i in range(1, numprocs):
pypar.send(test_points, i)
else:
test_points = pypar.receive(0)
print "Test Points::"
print test_points
if myid == 0:
control_data = run_simulation(parallel=False,
test_points=test_points,
verbose=True)
for proc in range(1, numprocs):
pypar.send(control_data, proc)
else:
control_data = pypar.receive(0)
def mpi_queue(items, worker_func, result_func, queue_rank=0, dynamic_thresh=4):
"""Implements a generic shared work queue over mpi.
== Inputs ==
items -- List of items to be distributed to workers.
worker_func -- Function which takes one work item and produces
a result.
result_func -- Function which processes the result (can be 0).
Only process queue_rank calls this function.
== Optional inputs ==
queue_rank -- Rank of the process which owns the queue.
dynamic_thresh -- Threshold for dynamic scheduling (see notes).
static_sync_size -- Block size for static scheduling.
== Notes ==
If the nprocs < dynamic_thresh, a static scheduling algorithm is
used (all processes do work) with synchronization occurring after
each work item. Otherwise a dynamic algorithm is used (process
queue_rank does not do work).
"""
if nprocs < dynamic_thresh: # STATIC LOAD QUEUE
if rank == queue_rank:
while len(items):
# Send items to work on.
item = items.pop()
expected_procs = []
for proc_num in range(nprocs):
if proc_num!=queue_rank and len(items):
mpi.send('go', proc_num)
mpi.send(items.pop(), proc_num)
expected_procs.append(proc_num)
result = worker_func(item)
result_func(result)
for proc_num in expected_procs:
result = mpi.receive(proc_num)
result_func(result)
for proc_num in range(nprocs):
if proc_num!=queue_rank:
mpi.send('stop', proc_num)
else:
while 1:
msg = mpi.receive(queue_rank)
if msg=='go':
item = mpi.receive(queue_rank)
result = worker_func(item)
mpi.send(result, queue_rank)
elif msg=='stop':
break
else:
print >>sys.stderr, 'message %s not recognized' % msg
else: # DYNAMIC QUEUE
if rank == queue_rank:
expected_procs = set()
nprocs_to_stop = nprocs - 1
while len(items) or len(expected_procs):
cmd, status = mpi.receive(mpi.any_source, return_status=True)
proc = status.source
if cmd=='pop': # proc wants another item
if len(items):
mpi.send('go', proc)
mpi.send(items.pop(), proc)
expected_procs.add(proc)
else:
mpi.send('stop', proc)
nprocs_to_stop-=1
elif cmd=='result': # proc wants to send a result
result = mpi.receive(proc)
result_func(result)
expected_procs.remove(proc)
while nprocs_to_stop:
cmd, status = mpi.receive(mpi.any_source, return_status=True)
proc = status.source
mpi.send('stop', proc)
nprocs_to_stop-=1
else:
while 1:
mpi.send('pop', queue_rank) # We want some work
msg = mpi.receive(queue_rank)
if msg=='go':
item = mpi.receive(queue_rank)
result = worker_func(item)
mpi.send('result', queue_rank)
mpi.send(result, queue_rank)
elif msg=='stop':
break
assert P > 1, 'Must have at least one slave'
assert B > P - 1, 'Must have more work packets than slaves'
A = numpy.zeros((M, N), dtype='i')
if p == 0:
# Create work pool (B blocks)
# using balanced work partitioning
workpool = []
for i in range(B):
Mlo, Mhi = balance(M, B, i)
workpool.append((Mlo, Mhi))
# Distribute initial work to slaves
w = 0
for d in range(1, P):
pypar.send(workpool[w], destination=d, tag=work_tag)
w += 1
#Receive computed work and distribute more
terminated = 0
while (terminated < P - 1):
R, status = pypar.receive(pypar.any_source,
tag=result_tag,
return_status=True)
A += R #Aggregate data
d = status.source #Id of slave that just finished
if w < len(workpool):
#Send new work to slave d
pypar.send(workpool[w], destination=d, tag=work_tag)
w += 1
