def getHandles(systemConfig, subConfigSettings): subConfigs = gen25DSubConfigs(**subConfigSettings) nsp = len(subConfigs) # Set up dictionary for subproblem objects and push base configuration for the system #setupCache(systemConfig) dview['localSystem'] = {} dview['baseSystemConfig'] = systemConfig # Create a function to get a subproblem forward modelling function dview['forwardFromTag'] = lambda tag, isrc, dOnly=True: localSystem[ tag].forward(isrc, dOnly) forwardFromTag = Reference('forwardFromTag') # Create a function to get a subproblem gradient function dview['gradientFromTag'] = lambda tag, isrc, dresid=1.: localSystem[ tag].gradient(isrc, dresid) gradientFromTag = Reference('gradientFromTag') dview['clearFromTag'] = lambda tag: localSystem[tag].clear() clearFromTag = Reference('clearFromTag') # Set up the subproblem objects with each new configuration tags = setupSystem(subConfigs) #dview.map_sync(setupSystem, subConfigs) # Forward model in 2.5D (in parallel) for an arbitrary source location # TODO: Write code to handle multiple data residuals for nom>1 handles = { 'forward': lambda isrc, dOnly=True: reduce( np.add, dview.map(forwardFromTag, tags, [isrc] * nsp, [dOnly] * nsp )), 'forwardSep': lambda isrc, dOnly=True: dview.map_sync(forwardFromTag, tags, [isrc] * nsp, [dOnly] * nsp), 'gradient': lambda isrc, dresid=1.0: reduce( np.add, dview.map(gradientFromTag, tags, [isrc] * nsp, [dresid] * nsp) ), # problems here for multiple freqs. 'gradSep': lambda isrc, dresid=1.0: dview.map_sync(gradientFromTag, tags, [ isrc ] * nsp, [dresid] * nsp), # problems here for multiple freqs. 'clear': lambda: print('Cleared stored matrix terms for %d systems.' % len( dview.map_sync(clearFromTag, tags))), } return handles
def _getHandles(self, systemConfig, subConfigSettings): pclient = self.remote.pclient dview = self.remote.dview lview = self.remote.lview subConfigs = self._gen25DSubConfigs(**subConfigSettings) nsp = len(subConfigs) # Set up dictionary for subproblem objects and push base configuration for the system dview['localSystem'] = {} self.remote[ 'baseSystemConfig'] = systemConfig # Faster if MPI is available dview['dPred'] = CommonReducer() dview['fWave'] = CommonReducer() dview['bWave'] = CommonReducer() dview['forwardFromTagAccumulate'] = forwardFromTagAccumulate dview['forwardFromTagAccumulateAll'] = forwardFromTagAccumulateAll dview['backpropFromTagAccumulate'] = backpropFromTagAccumulate dview['backpropFromTagAccumulateAll'] = backpropFromTagAccumulateAll dview['clearFromTag'] = clearFromTag dview.wait() schedule = { 'forward': { 'solve': Reference('forwardFromTagAccumulateAll'), 'clear': Reference('clearFromTag'), 'reduce': ['dPred', 'fWave'] }, 'backprop': { 'solve': Reference('backpropFromTagAccumulateAll'), 'clear': Reference('clearFromTag'), 'reduce': ['bWave'] }, } self.systemsolver = SystemSolver(self, schedule) if 'parFac' in systemConfig: parFac = systemConfig['parFac'] else: parFac = 1 while parFac > 0: tags = lview.map_sync(setupSystem, subConfigs) parFac -= 1
def phistogram(view, a, bins=10, rng=None, normed=False): """Compute the histogram of a remote array a. Parameters ---------- view IPython DirectView instance a : str String name of the remote array bins : int Number of histogram bins rng : (float, float) Tuple of min, max of the range to histogram normed : boolean Should the histogram counts be normalized to 1 """ nengines = len(view.targets) # view.push(dict(bins=bins, rng=rng)) with view.sync_imports(): import numpy rets = view.apply_sync(lambda a, b, rng: numpy.histogram(a, b, rng), Reference(a), bins, rng) hists = [r[0] for r in rets] lower_edges = [r[1] for r in rets] # view.execute('hist, lower_edges = numpy.histogram(%s, bins, rng)' % a) lower_edges = view.pull('lower_edges', targets=0) hist_array = numpy.array(hists).reshape(nengines, -1) # hist_array.shape = (nengines,-1) total_hist = numpy.sum(hist_array, 0) if normed: total_hist = total_hist / numpy.sum(total_hist, dtype=float) return total_hist, lower_edges
def update_state_parallel(Para,Gamma,quadratic = True): ''' Updates the state using parallel code ''' v.block = True v['Gamma'] = Gamma v.execute('approx = approximate.approximate(Gamma)') approx = Reference('approx') diff = 100. n = 0. while diff > 0.001 and n < 1: Gamma_new_t,Y_t,Shocks_t,y_t = v.apply(lambda approx,quadratic = quadratic: approx.iterate(quadratic),approx)[0] error = np.linalg.norm(np.mean(Gamma_new_t[:,:2],0)) if error < diff: diff = error Gamma_new,Y,Shocks,y = Gamma_new_t,Y_t,Shocks_t,y_t n += 1 return Para.nomalize(Gamma_new.copy()),Y.copy(),Shocks.copy(),y.copy()
def backprop(self, **kwargs): dview = self.par['dview'] dview['backpropResultTracker'] = commonReducer() G = self._systemSolve(Reference('backpropFromTagAccumulateAll'), isrcs) # self.par['lview'].wait(G.predecessors('End')) # uB = self._remote.reduce('backpropResultTracker') return G
def remote_iterator(view,name): """Return an iterator on an object living on a remote engine. """ view.execute('it%s=iter(%s)'%(name,name), block=True) while True: try: result = view.apply_sync(lambda x: x.next(), Reference('it'+name)) # This causes the StopIteration exception to be raised. except RemoteError as e: if e.ename == 'StopIteration': raise StopIteration else: raise e else: yield result
def pwordfreq(view, fnames): """Parallel word frequency counter. view - An IPython DirectView fnames - The filenames containing the split data. """ assert len(fnames) == len(view.targets) view.scatter('fname', fnames, flatten=True) ar = view.apply(wordfreq, Reference('fname')) freqs_list = ar.get() word_set = set() for f in freqs_list: word_set.update(f.keys()) freqs = dict(zip(word_set, repeat(0))) for f in freqs_list: for word, count in f.items(): freqs[word] += count return freqs
def forward(self, isrcs=None, **kwargs): dview = self.par['dview'] dview['dataResultTracker'] = commonReducer() dview['forwardResultTracker'] = commonReducer() G = self._systemSolve(Reference('forwardFromTagAccumulateAll'), isrcs) # self.par['lview'].wait(G.predecessors('End')) # d = self._remote.reduce('dataResultTracker') # if not kwargs.get('dOnly', True): # uF = self._remote.reduce('forwardResultTracker') # return uF, d # return d return G
view['u_hist'] = [] # set vector/scalar implementation details impl = {} impl['ic'] = 'vectorized' impl['inner'] = 'scalar' impl['bc'] = 'vectorized' # execute some files so that the classes we need will be defined on the engines: view.run('RectPartitioner.py') view.run('wavesolver.py') # setup remote partitioner # note that Reference means that the argument passed to setup_partitioner will be the # object named 'my_id' in the engine's namespace view.apply_sync(setup_partitioner, Reference('my_id'), num_procs, grid, partition) # wait for initial communication to complete view.execute('mpi.barrier()') # setup remote solvers view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl) # lambda for calling solver.solve: _solve = lambda *args, **kwargs: solver.solve(*args, **kwargs) if ns.scalar: impl['inner'] = 'scalar' # run first with element-wise Python operations for each cell t0 = time.time() ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test, user_action=user_action) if final_test: # this sum is performed element-wise as results finish
# create the Communicator objects on the engines view.execute('com = BinaryTreeCommunicator(id, root = id==root_id )') pub_url = root.apply_sync(lambda : com.pub_url) # gather the connection information into a dict ar = view.apply_async(lambda : com.info) peers = ar.get_dict() # this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators # connect the engines to each other: def connect(com, peers, tree, pub_url, root_id): """this function will be called on the engines""" com.connect(peers, tree, pub_url, root_id) view.apply_sync(connect, Reference('com'), peers, btree, pub_url, root_id) # functions that can be used for reductions # max and min builtins can be used as well def add(a,b): """cumulative sum reduction""" return a+b def mul(a,b): """cumulative product reduction""" return a*b view['add'] = add view['mul'] = mul # scatter some data
# scatter engine IDs view.scatter('my_id', range(num_procs), flatten=True) # create the engine connectors view.execute('com = EngineCommunicator()') # gather the connection information into a single dict ar = view.apply_async(lambda: com.info) peers = ar.get_dict() # print peers # this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators # setup remote partitioner # note that Reference means that the argument passed to setup_partitioner will be the # object named 'com' in the engine's namespace view.apply_sync(setup_partitioner, Reference('com'), peers, Reference('my_id'), num_procs, grid, partition) time.sleep(1) # convenience lambda to call solver.solve: _solve = lambda *args, **kwargs: solver.solve(*args, **kwargs) if ns.scalar: impl['inner'] = 'scalar' # setup remote solvers view.apply_sync(setup_solver, I, f, c, bc, Lx, Ly,
def _systemSolve(self, fnRef, isrcs, clearRef=Reference('clearFromTag'), **kwargs): dview = self.par['dview'] lview = self.par['lview'] chunksPerWorker = self.systemConfig.get('chunksPerWorker', 1) G = networkx.DiGraph() mainNode = 'Beginning' G.add_node(mainNode) # Parse sources nsrc = len(self.systemConfig['geom']['src']) if isrcs is None: isrcslist = range(nsrc) elif isinstance(isrcs, slice): isrcslist = range(isrcs.start or 0, isrcs.stop or nsrc, isrcs.step or 1) else: try: _ = isrcs[0] isrcslist = isrcs except TypeError: isrcslist = [isrcs] systemsOnWorkers = dview['localSystem.keys()'] ids = dview['rank'] tags = set() for ltags in systemsOnWorkers: tags = tags.union(set(ltags)) endNodes = {} tailNodes = [] for tag in tags: tagNode = 'Head: %d, %d' % tag G.add_edge(mainNode, tagNode) relIDs = [] for i in xrange(len(ids)): systems = systemsOnWorkers[i] rank = ids[i] if tag in systems: relIDs.append(i) systemJobs = [] endNodes[tag] = [] systemNodes = [] with lview.temp_flags(block=False): iworks = 0 for work in getChunks( isrcslist, int(round(chunksPerWorker * len(relIDs)))): if work: job = lview.apply(fnRef, tag, work, **kwargs) systemJobs.append(job) label = 'Compute: %d, %d, %d' % (tag[0], tag[1], iworks) systemNodes.append(label) G.add_node(label, job=job) G.add_edge(tagNode, label) iworks += 1 if self.systemConfig.get( 'ensembleClear', False ): # True for ensemble ending, False for individual ending tagNode = 'Wrap: %d, %d' % tag for label in systemNodes: G.add_edge(label, tagNode) for i in relIDs: rank = ids[i] with lview.temp_flags(block=False, after=systemJobs): job = lview.apply( depend(hasSystemRank, tag, rank)(clearRef), tag) label = 'Wrap: %d, %d, %d' % (tag[0], tag[1], i) G.add_node(label, job=job) endNodes[tag].append(label) G.add_edge(tagNode, label) else: for i, sjob in enumerate(systemJobs): with lview.temp_flags(block=False, follow=sjob): job = lview.apply(clearRef, tag) label = 'Wrap: %d, %d, %d' % (tag[0], tag[1], i) G.add_node(label, job=job) endNodes[tag].append(label) G.add_edge(systemNodes[i], label) tagNode = 'Tail: %d, %d' % tag for label in endNodes[tag]: G.add_edge(label, tagNode) tailNodes.append(tagNode) endNode = 'End' for node in tailNodes: G.add_edge(node, endNode) return G
def _getHandles(self, systemConfig, subConfigSettings): pclient = self.par['pclient'] dview = self.par['dview'] lview = self.par['lview'] subConfigs = self._gen25DSubConfigs(**subConfigSettings) nsp = len(subConfigs) # Set up dictionary for subproblem objects and push base configuration for the system dview['localSystem'] = {} self._remote[ 'baseSystemConfig'] = systemConfig # Faster if MPI is available dview['dataResultTracker'] = commonReducer() dview['forwardResultTracker'] = commonReducer() dview['backpropResultTracker'] = commonReducer() dview.execute( "localLocator = Kernel.SeisLocator25D(baseSystemConfig['geom'])") # Create a function to get a subproblem forward modelling function dview['forwardFromTag'] = lambda tag, isrc, dOnly=True: localSystem[ tag].forward(isrc, dOnly) forwardFromTag = Reference('forwardFromTag') # Create a function to get a subproblem gradient function dview['gradientFromTag'] = lambda tag, isrc, dresid=1.: localSystem[ tag].gradient(isrc, dresid) gradientFromTag = Reference('gradientFromTag') dview['forwardFromTagAccumulate'] = forwardFromTagAccumulate dview['forwardFromTagAccumulateAll'] = forwardFromTagAccumulateAll dview['clearFromTag'] = clearFromTag dview.wait() if 'parFac' in systemConfig: parFac = systemConfig['parFac'] else: parFac = 1 while parFac > 0: tags = lview.map_sync(setupSystem, subConfigs) parFac -= 1 # Forward model in 2.5D (in parallel) for an arbitrary source location # TODO: Write code to handle multiple data residuals for nom>1 handles = { 'forward': lambda isrc, dOnly=True: reduce( np.add, dview.map(forwardFromTag, tags, [isrc] * nsp, [dOnly] * nsp)), 'forwardSep': lambda isrc, dOnly=True: dview.map_sync( forwardFromTag, tags, [isrc] * nsp, [dOnly] * nsp), 'gradient': lambda isrc, dresid=1.0: reduce( np.add, dview.map(gradientFromTag, tags, [isrc] * nsp, [dresid] * nsp) ), 'gradSep': lambda isrc, dresid=1.0: dview.map_sync( gradientFromTag, tags, [isrc] * nsp, [dresid] * nsp), #from __future__ import print_function # 'clear': lambda: print('Cleared stored matrix terms for %d systems.'%len(dview.map_sync(clearFromTag, tags))), } return handles
def __init__(self, systemConfig): if 'profile' in systemConfig: pupdate = {'profile': systemConfig['profile']} else: pupdate = {} pclient = Client(**pupdate) if not cdSame(pclient): print( 'Could not change all workers to the same directory as the client!' ) dview = pclient[:] dview.block = True dview.clear() remoteSetup = ''' import os import numpy as np import scipy as scipy import scipy.sparse import mkl import SimPEG import zephyr.Kernel as Kernel''' parMPISetup = ''' from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank()''' for command in remoteSetup.strip().split('\n'): dview.execute(command.strip()) dview.scatter('rank', pclient.ids, flatten=True) dview.apply(noMKLVectorization) self.useMPI = False if systemConfig.get('MPI', DEFAULT_MPI): MPISafe = False for var in MPI_BELLWETHERS: MPISafe = MPISafe or all(dview["os.getenv('%s')" % (var, )]) if MPISafe: for command in parMPISetup.strip().split('\n'): dview.execute(command.strip()) ranks = dview['rank'] reorder = [ranks.index(i) for i in xrange(len(ranks))] dview = pclient[reorder] dview.block = True dview.activate() # Set up necessary parts for broadcast-based communication self.e0 = pclient[reorder[0]] self.e0.block = True self.comm = Reference('comm') self.useMPI = MPISafe self.pclient = pclient self.dview = dview self.lview = pclient.load_balanced_view() # Generate 'par' object for Problem to grab self.par = { 'pclient': self.pclient, 'dview': self.dview, 'lview': self.pclient.load_balanced_view(), }