def getHandles(systemConfig, subConfigSettings):
subConfigs = gen25DSubConfigs(**subConfigSettings)
nsp = len(subConfigs)
# Set up dictionary for subproblem objects and push base configuration for the system
#setupCache(systemConfig)
dview['localSystem'] = {}
dview['baseSystemConfig'] = systemConfig
# Create a function to get a subproblem forward modelling function
dview['forwardFromTag'] = lambda tag, isrc, dOnly=True: localSystem[
tag].forward(isrc, dOnly)
forwardFromTag = Reference('forwardFromTag')
# Create a function to get a subproblem gradient function
dview['gradientFromTag'] = lambda tag, isrc, dresid=1.: localSystem[
tag].gradient(isrc, dresid)
gradientFromTag = Reference('gradientFromTag')
dview['clearFromTag'] = lambda tag: localSystem[tag].clear()
clearFromTag = Reference('clearFromTag')
# Set up the subproblem objects with each new configuration
tags = setupSystem(subConfigs) #dview.map_sync(setupSystem, subConfigs)
# Forward model in 2.5D (in parallel) for an arbitrary source location
# TODO: Write code to handle multiple data residuals for nom>1
handles = {
'forward':
lambda isrc, dOnly=True: reduce(
np.add, dview.map(forwardFromTag, tags, [isrc] * nsp, [dOnly] * nsp
)),
'forwardSep':
lambda isrc, dOnly=True: dview.map_sync(forwardFromTag, tags, [isrc] *
nsp, [dOnly] * nsp),
'gradient':
lambda isrc, dresid=1.0: reduce(
np.add,
dview.map(gradientFromTag, tags, [isrc] * nsp, [dresid] * nsp)
), # problems here for multiple freqs.
'gradSep':
lambda isrc, dresid=1.0: dview.map_sync(gradientFromTag, tags, [
isrc
] * nsp, [dresid] * nsp), # problems here for multiple freqs.
'clear':
lambda: print('Cleared stored matrix terms for %d systems.' % len(
dview.map_sync(clearFromTag, tags))),
}
return handles
def _getHandles(self, systemConfig, subConfigSettings):
pclient = self.remote.pclient
dview = self.remote.dview
lview = self.remote.lview
subConfigs = self._gen25DSubConfigs(**subConfigSettings)
nsp = len(subConfigs)
# Set up dictionary for subproblem objects and push base configuration for the system
dview['localSystem'] = {}
self.remote[
'baseSystemConfig'] = systemConfig # Faster if MPI is available
dview['dPred'] = CommonReducer()
dview['fWave'] = CommonReducer()
dview['bWave'] = CommonReducer()
dview['forwardFromTagAccumulate'] = forwardFromTagAccumulate
dview['forwardFromTagAccumulateAll'] = forwardFromTagAccumulateAll
dview['backpropFromTagAccumulate'] = backpropFromTagAccumulate
dview['backpropFromTagAccumulateAll'] = backpropFromTagAccumulateAll
dview['clearFromTag'] = clearFromTag
dview.wait()
schedule = {
'forward': {
'solve': Reference('forwardFromTagAccumulateAll'),
'clear': Reference('clearFromTag'),
'reduce': ['dPred', 'fWave']
},
'backprop': {
'solve': Reference('backpropFromTagAccumulateAll'),
'clear': Reference('clearFromTag'),
'reduce': ['bWave']
},
}
self.systemsolver = SystemSolver(self, schedule)
if 'parFac' in systemConfig:
parFac = systemConfig['parFac']
else:
parFac = 1
while parFac > 0:
tags = lview.map_sync(setupSystem, subConfigs)
parFac -= 1
def phistogram(view, a, bins=10, rng=None, normed=False):
"""Compute the histogram of a remote array a.
Parameters
----------
view
IPython DirectView instance
a : str
String name of the remote array
bins : int
Number of histogram bins
rng : (float, float)
Tuple of min, max of the range to histogram
normed : boolean
Should the histogram counts be normalized to 1
"""
nengines = len(view.targets)
# view.push(dict(bins=bins, rng=rng))
with view.sync_imports():
import numpy
rets = view.apply_sync(lambda a, b, rng: numpy.histogram(a, b, rng),
Reference(a), bins, rng)
hists = [r[0] for r in rets]
lower_edges = [r[1] for r in rets]
# view.execute('hist, lower_edges = numpy.histogram(%s, bins, rng)' % a)
lower_edges = view.pull('lower_edges', targets=0)
hist_array = numpy.array(hists).reshape(nengines, -1)
# hist_array.shape = (nengines,-1)
total_hist = numpy.sum(hist_array, 0)
if normed:
total_hist = total_hist / numpy.sum(total_hist, dtype=float)
return total_hist, lower_edges
def backprop(self, **kwargs):
dview = self.par['dview']
dview['backpropResultTracker'] = commonReducer()
G = self._systemSolve(Reference('backpropFromTagAccumulateAll'), isrcs)
# self.par['lview'].wait(G.predecessors('End'))
# uB = self._remote.reduce('backpropResultTracker')
return G
def remote_iterator(view,name):
"""Return an iterator on an object living on a remote engine.
"""
view.execute('it%s=iter(%s)'%(name,name), block=True)
while True:
try:
result = view.apply_sync(lambda x: x.next(), Reference('it'+name))
# This causes the StopIteration exception to be raised.
except RemoteError as e:
if e.ename == 'StopIteration':
raise StopIteration
else:
raise e
else:
yield result
def pwordfreq(view, fnames):
"""Parallel word frequency counter.
view - An IPython DirectView
fnames - The filenames containing the split data.
"""
assert len(fnames) == len(view.targets)
view.scatter('fname', fnames, flatten=True)
ar = view.apply(wordfreq, Reference('fname'))
freqs_list = ar.get()
word_set = set()
for f in freqs_list:
word_set.update(f.keys())
freqs = dict(zip(word_set, repeat(0)))
for f in freqs_list:
for word, count in f.items():
freqs[word] += count
return freqs
def forward(self, isrcs=None, **kwargs):
dview = self.par['dview']
dview['dataResultTracker'] = commonReducer()
dview['forwardResultTracker'] = commonReducer()
G = self._systemSolve(Reference('forwardFromTagAccumulateAll'), isrcs)
# self.par['lview'].wait(G.predecessors('End'))
# d = self._remote.reduce('dataResultTracker')
# if not kwargs.get('dOnly', True):
# uF = self._remote.reduce('forwardResultTracker')
# return uF, d
# return d
return G
view['u_hist'] = []
# set vector/scalar implementation details
impl = {}
impl['ic'] = 'vectorized'
impl['inner'] = 'scalar'
impl['bc'] = 'vectorized'
# execute some files so that the classes we need will be defined on the engines:
view.run('RectPartitioner.py')
view.run('wavesolver.py')
# setup remote partitioner
# note that Reference means that the argument passed to setup_partitioner will be the
# object named 'my_id' in the engine's namespace
view.apply_sync(setup_partitioner, Reference('my_id'), num_procs, grid, partition)
# wait for initial communication to complete
view.execute('mpi.barrier()')
# setup remote solvers
view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
# lambda for calling solver.solve:
_solve = lambda *args, **kwargs: solver.solve(*args, **kwargs)
if ns.scalar:
impl['inner'] = 'scalar'
# run first with element-wise Python operations for each cell
t0 = time.time()
ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test, user_action=user_action)
if final_test:
# this sum is performed element-wise as results finish
# create the Communicator objects on the engines
view.execute('com = BinaryTreeCommunicator(id, root = id==root_id )')
pub_url = root.apply_sync(lambda : com.pub_url)
# gather the connection information into a dict
ar = view.apply_async(lambda : com.info)
peers = ar.get_dict()
# this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators
# connect the engines to each other:
def connect(com, peers, tree, pub_url, root_id):
"""this function will be called on the engines"""
com.connect(peers, tree, pub_url, root_id)
view.apply_sync(connect, Reference('com'), peers, btree, pub_url, root_id)
# functions that can be used for reductions
# max and min builtins can be used as well
def add(a,b):
"""cumulative sum reduction"""
return a+b
def mul(a,b):
"""cumulative product reduction"""
return a*b
view['add'] = add
view['mul'] = mul
# scatter some data
# scatter engine IDs
view.scatter('my_id', range(num_procs), flatten=True)
# create the engine connectors
view.execute('com = EngineCommunicator()')
# gather the connection information into a single dict
ar = view.apply_async(lambda: com.info)
peers = ar.get_dict()
# print peers
# this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators
# setup remote partitioner
# note that Reference means that the argument passed to setup_partitioner will be the
# object named 'com' in the engine's namespace
view.apply_sync(setup_partitioner, Reference('com'), peers,
Reference('my_id'), num_procs, grid, partition)
time.sleep(1)
# convenience lambda to call solver.solve:
_solve = lambda *args, **kwargs: solver.solve(*args, **kwargs)
if ns.scalar:
impl['inner'] = 'scalar'
# setup remote solvers
view.apply_sync(setup_solver,
I,
f,
c,
bc,
Lx,
Ly,
def _systemSolve(self,
fnRef,
isrcs,
clearRef=Reference('clearFromTag'),
**kwargs):
dview = self.par['dview']
lview = self.par['lview']
chunksPerWorker = self.systemConfig.get('chunksPerWorker', 1)
G = networkx.DiGraph()
mainNode = 'Beginning'
G.add_node(mainNode)
# Parse sources
nsrc = len(self.systemConfig['geom']['src'])
if isrcs is None:
isrcslist = range(nsrc)
elif isinstance(isrcs, slice):
isrcslist = range(isrcs.start or 0, isrcs.stop or nsrc, isrcs.step
or 1)
else:
try:
_ = isrcs[0]
isrcslist = isrcs
except TypeError:
isrcslist = [isrcs]
systemsOnWorkers = dview['localSystem.keys()']
ids = dview['rank']
tags = set()
for ltags in systemsOnWorkers:
tags = tags.union(set(ltags))
endNodes = {}
tailNodes = []
for tag in tags:
tagNode = 'Head: %d, %d' % tag
G.add_edge(mainNode, tagNode)
relIDs = []
for i in xrange(len(ids)):
systems = systemsOnWorkers[i]
rank = ids[i]
if tag in systems:
relIDs.append(i)
systemJobs = []
endNodes[tag] = []
systemNodes = []
with lview.temp_flags(block=False):
iworks = 0
for work in getChunks(
isrcslist, int(round(chunksPerWorker * len(relIDs)))):
if work:
job = lview.apply(fnRef, tag, work, **kwargs)
systemJobs.append(job)
label = 'Compute: %d, %d, %d' % (tag[0], tag[1],
iworks)
systemNodes.append(label)
G.add_node(label, job=job)
G.add_edge(tagNode, label)
iworks += 1
if self.systemConfig.get(
'ensembleClear', False
): # True for ensemble ending, False for individual ending
tagNode = 'Wrap: %d, %d' % tag
for label in systemNodes:
G.add_edge(label, tagNode)
for i in relIDs:
rank = ids[i]
with lview.temp_flags(block=False, after=systemJobs):
job = lview.apply(
depend(hasSystemRank, tag, rank)(clearRef), tag)
label = 'Wrap: %d, %d, %d' % (tag[0], tag[1], i)
G.add_node(label, job=job)
endNodes[tag].append(label)
G.add_edge(tagNode, label)
else:
for i, sjob in enumerate(systemJobs):
with lview.temp_flags(block=False, follow=sjob):
job = lview.apply(clearRef, tag)
label = 'Wrap: %d, %d, %d' % (tag[0], tag[1], i)
G.add_node(label, job=job)
endNodes[tag].append(label)
G.add_edge(systemNodes[i], label)
tagNode = 'Tail: %d, %d' % tag
for label in endNodes[tag]:
G.add_edge(label, tagNode)
tailNodes.append(tagNode)
endNode = 'End'
for node in tailNodes:
G.add_edge(node, endNode)
return G
def _getHandles(self, systemConfig, subConfigSettings):
pclient = self.par['pclient']
dview = self.par['dview']
lview = self.par['lview']
subConfigs = self._gen25DSubConfigs(**subConfigSettings)
nsp = len(subConfigs)
# Set up dictionary for subproblem objects and push base configuration for the system
dview['localSystem'] = {}
self._remote[
'baseSystemConfig'] = systemConfig # Faster if MPI is available
dview['dataResultTracker'] = commonReducer()
dview['forwardResultTracker'] = commonReducer()
dview['backpropResultTracker'] = commonReducer()
dview.execute(
"localLocator = Kernel.SeisLocator25D(baseSystemConfig['geom'])")
# Create a function to get a subproblem forward modelling function
dview['forwardFromTag'] = lambda tag, isrc, dOnly=True: localSystem[
tag].forward(isrc, dOnly)
forwardFromTag = Reference('forwardFromTag')
# Create a function to get a subproblem gradient function
dview['gradientFromTag'] = lambda tag, isrc, dresid=1.: localSystem[
tag].gradient(isrc, dresid)
gradientFromTag = Reference('gradientFromTag')
dview['forwardFromTagAccumulate'] = forwardFromTagAccumulate
dview['forwardFromTagAccumulateAll'] = forwardFromTagAccumulateAll
dview['clearFromTag'] = clearFromTag
dview.wait()
if 'parFac' in systemConfig:
parFac = systemConfig['parFac']
else:
parFac = 1
while parFac > 0:
tags = lview.map_sync(setupSystem, subConfigs)
parFac -= 1
# Forward model in 2.5D (in parallel) for an arbitrary source location
# TODO: Write code to handle multiple data residuals for nom>1
handles = {
'forward':
lambda isrc, dOnly=True: reduce(
np.add,
dview.map(forwardFromTag, tags, [isrc] * nsp, [dOnly] * nsp)),
'forwardSep':
lambda isrc, dOnly=True: dview.map_sync(
forwardFromTag, tags, [isrc] * nsp, [dOnly] * nsp),
'gradient':
lambda isrc, dresid=1.0: reduce(
np.add,
dview.map(gradientFromTag, tags, [isrc] * nsp, [dresid] * nsp)
),
'gradSep':
lambda isrc, dresid=1.0: dview.map_sync(
gradientFromTag, tags, [isrc] * nsp, [dresid] * nsp),
#from __future__ import print_function
# 'clear': lambda: print('Cleared stored matrix terms for %d systems.'%len(dview.map_sync(clearFromTag, tags))),
}
return handles
def __init__(self, systemConfig):
if 'profile' in systemConfig:
pupdate = {'profile': systemConfig['profile']}
else:
pupdate = {}
pclient = Client(**pupdate)
if not cdSame(pclient):
print(
'Could not change all workers to the same directory as the client!'
)
dview = pclient[:]
dview.block = True
dview.clear()
remoteSetup = '''
import os
import numpy as np
import scipy as scipy
import scipy.sparse
import mkl
import SimPEG
import zephyr.Kernel as Kernel'''
parMPISetup = '''
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()'''
for command in remoteSetup.strip().split('\n'):
dview.execute(command.strip())
dview.scatter('rank', pclient.ids, flatten=True)
dview.apply(noMKLVectorization)
self.useMPI = False
if systemConfig.get('MPI', DEFAULT_MPI):
MPISafe = False
for var in MPI_BELLWETHERS:
MPISafe = MPISafe or all(dview["os.getenv('%s')" % (var, )])
if MPISafe:
for command in parMPISetup.strip().split('\n'):
dview.execute(command.strip())
ranks = dview['rank']
reorder = [ranks.index(i) for i in xrange(len(ranks))]
dview = pclient[reorder]
dview.block = True
dview.activate()
# Set up necessary parts for broadcast-based communication
self.e0 = pclient[reorder[0]]
self.e0.block = True
self.comm = Reference('comm')
self.useMPI = MPISafe
self.pclient = pclient
self.dview = dview
self.lview = pclient.load_balanced_view()
# Generate 'par' object for Problem to grab
self.par = {
'pclient': self.pclient,
'dview': self.dview,
'lview': self.pclient.load_balanced_view(),
}