def fmultiprocess(log, function, inputArray, poolSize=False, timeout=3600, **kwargs): """multiprocess pool **Key Arguments:** - ``log`` -- logger - ``function`` -- the function to multiprocess - ``inputArray`` -- the array to be iterated over - ``poolSize`` -- limit the number of CPU that are used in multiprocess job - ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed **Return:** - ``resultArray`` -- the array of results **Usage:** .. code-block:: python from fundamentals import multiprocess # DEFINE AN INPUT ARRAY inputArray = range(10000) results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300, inputArray=inputArray, otherFunctionKeyword="cheese") """ log.debug('starting the ``multiprocess`` function') # DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1) if not poolSize: poolSize = psutil.cpu_count() if poolSize: p = Pool(processes=poolSize) else: p = Pool() # MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES if "log" in inspect.getargspec(function)[0]: mapfunc = partial(function, log=log, **kwargs) resultArray = p.map_async(mapfunc, inputArray) else: mapfunc = partial(function, **kwargs) resultArray = p.map_async(mapfunc, inputArray) resultArray = resultArray.get(timeout=timeout) p.close() p.terminate() log.debug('completed the ``multiprocess`` function') return resultArray
class MultiThreading(object): def __init__(self, funct, data, threads='all'): raise Exception("Not functionnal yet !") self.funct = funct if threads == 'all': threads = cpu_count() self.pool = Pool(processes=threads) self.data = data self.PG = None self.initializer = None self.finalizer = None def add_progress_counter(self, init_mess="Beginning", end_mess="Done", name_things='things', perc_interv=5): self.PG = ProgressCounter(init_mess=init_mess, end_mess=end_mess, nmb_max=len(self.data), name_things=name_things, perc_interv=perc_interv) self.manager = Manager() self.manager.register("PG", self.PG) def run(self): res = self.pool.map_async(self.PG_func_wrapper, self.data) self.pool.close() self.pool.join() return res
def predict(self, inputData, transientTime=0, update_processor=lambda x: x, verbose=0): rank = len(inputData.shape) - 1 if rank != self.n_inputDimensions: raise ValueError( "The `inputData` does not have a suitable shape. It has to have {0} spatial dimensions and 1 temporal dimension.".format( self.n_inputDimensions)) manager = Manager() predictQueue = manager.Queue() # workaround as predict does not support batches atm # add dummy dimension to let embedInputData work properly (is optimized to work for batches) inputData = inputData.reshape(1, *inputData.shape) modifiedInputData = self._embedInputData(inputData) modifiedInputData = modifiedInputData[0] inputData = inputData[0] self.transientTime = transientTime self.sharedNamespace.transientTime = transientTime predictionOutput = B.zeros(np.insert(self.inputShape, 0, inputData.shape[0] - transientTime)) jobs = np.stack(np.meshgrid(*[np.arange(x) + self._filterWidth for x in inputData.shape[1:]]), axis=rank).reshape(-1, rank).tolist() nJobs = len(jobs) self.resetState() iterator = PredictionArrayIterator(modifiedInputData, jobs, self._filterWidth, self._stride, self) pool = Pool(processes=self._nWorkers, initializer=SpatioTemporalESN._init_predictProcess, initargs=[predictQueue, self]) pool.map_async(self._predictProcess, iterator, chunksize=200)#, chunksize=1) def _processPoolWorkerResults(): nJobsDone = 0 if verbose > 0: bar = progressbar.ProgressBar(max_value=nJobs, redirect_stdout=True, poll_interval=0.0001) bar.update(0) while nJobsDone < nJobs: data = predictQueue.get() # result of predicting indices, prediction, state = data id = self._uniqueIDFromIndices(indices) self._xs[id] = state # update the values predictionOutput[tuple([Ellipsis] + indices)] = prediction nJobsDone += 1 if verbose > 0: bar.update(nJobsDone) if verbose > 1: print(nJobsDone) if verbose > 0: bar.finish() _processPoolWorkerResults() pool.close() return predictionOutput
def fit(self, inputData, outputData, transientTime=0, verbose=0): rank = len(inputData.shape) - 1 if rank != self.n_inputDimensions and rank != self.n_inputDimensions + 1: raise ValueError( "The `inputData` does not have a suitable shape. It has to have {0} spatial dimensions and 1 temporal dimension.".format( self.n_inputDimensions)) # reshape the input so that it has the shape (timeseries, time, input_dimension^n) if rank == self.n_inputDimensions: inputData = inputData.reshape(1, *inputData.shape) outputData = outputData.reshape(1, *outputData.shape) else: # modify rank again rank -= 1 partialLength = (inputData.shape[1] - transientTime) totalLength = inputData.shape[0] * partialLength timeseriesCount = inputData.shape[0] manager = Manager() fitQueue = manager.Queue() modifiedInputData = self._embedInputData(inputData) self.sharedNamespace.transientTime = transientTime self.sharedNamespace.partialLength = partialLength self.sharedNamespace.totalLength = totalLength self.sharedNamespace.timeseriesCount = timeseriesCount jobs = np.stack(np.meshgrid(*[np.arange(x) + self._filterWidth for x in inputData.shape[2:]]), axis=rank).reshape(-1, rank).tolist() nJobs = len(jobs) self.resetState() iterator = FittingArrayIterator(modifiedInputData, outputData, jobs, self._filterWidth, self._stride, self) pool = Pool(processes=self._nWorkers, initializer=SpatioTemporalESN._init_fitProcess, initargs=[fitQueue, self]) pool.map_async(self._fitProcess, iterator, chunksize=16) def _processPoolWorkerResults(): nJobsDone = 0 if verbose > 0: bar = progressbar.ProgressBar(max_value=nJobs, redirect_stdout=True, poll_interval=0.0001) bar.update(0) while nJobsDone < nJobs: data = fitQueue.get() # result of fitting indices, x, WOut = data id = self._uniqueIDFromIndices(indices) if WOut is None: import sys print("WARNING: Fit process for pixel {0} did not succeed".format(indices), file=sys.stderr) # store WOut if self._averageOutputWeights: if WOut is not None: self._WOut += WOut / np.prod(self.inputShape) else: self._WOuts[id] = WOut # store x self._xs[id] = x nJobsDone += 1 if verbose > 0: bar.update(nJobsDone) if verbose > 1: print(nJobsDone) if verbose > 0: bar.finish() _processPoolWorkerResults() pool.close()
from multiprocess import Pool def f(x): return x * x if __name__ == '__main__': p = Pool(4) result = p.map_async(f, range(10)) print(result.get(timeout=1))
def _multi_channel_apply_disk_parallel(self, function, cleanup_function, output_path, from_time, to_time, channels, cast_dtype, pass_batch_info, pass_batch_results, processes, **kwargs): self.logger.debug('Starting parallel operation...') if pass_batch_results: raise NotImplementedError("pass_batch_results is not " "implemented on 'disk' mode") # need to convert to a list, oherwise cannot be pickled data = list( self.multi_channel(from_time, to_time, channels, return_data=False)) n_batches = self.indexer.n_batches(from_time, to_time, channels) self.logger.info('Data will be splitted in %s batches', n_batches) output_path = Path(output_path) # create local variables to avoid pickling problems _path_to_recordings = copy(self.path_to_recordings) _dtype = copy(self.dtype) _n_channels = copy(self.n_channels) _data_order = copy(self.data_order) _loader = copy(self.loader) _buffer_size = copy(self.buffer_size) reader = partial(RecordingsReader, path_to_recordings=_path_to_recordings, dtype=_dtype, n_channels=_n_channels, data_order=_data_order, loader=_loader, return_data_index=True) m = Manager() mapping = m.dict() next_to_write = m.Value('i', 0) def parallel_runner(element): i, _ = element res = util.batch_runner(element, function, reader, pass_batch_info, cast_dtype, kwargs, cleanup_function, _buffer_size, save_chunks=False, output_path=output_path) if i == 0: mapping['dtype'] = str(res.dtype) while True: if next_to_write.value == i: with open(str(output_path), 'wb' if i == 0 else 'ab') as f: res.tofile(f) next_to_write.value += 1 break # run jobs self.logger.debug('Creating processes pool...') p = Pool(processes) res = p.map_async(parallel_runner, enumerate(data)) finished = 0 if self.show_progress_bar: pbar = tqdm(total=n_batches) if self.show_progress_bar: while True: if next_to_write.value > finished: update = next_to_write.value - finished pbar.update(update) finished = next_to_write.value if next_to_write.value == n_batches: break pbar.close() else: res.get() # save metadata params = util.make_metadata(channels, self.n_channels, mapping['dtype'], output_path) return output_path, params
def test(): print('cpuCount() = %d\n' % cpuCount()) # # Create pool # PROCESSES = 4 print('Creating pool with %d processes\n' % PROCESSES) pool = Pool(PROCESSES) # # Tests # TASKS = [(mul, (i, 7)) for i in range(10)] + \ [(plus, (i, 8)) for i in range(10)] results = [pool.apply_async(calculate, t) for t in TASKS] imap_it = pool.imap(calculatestar, TASKS) imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) print('Ordered results using pool.apply_async():') for r in results: print('\t', r.get()) print() print('Ordered results using pool.imap():') for x in imap_it: print('\t', x) print() print('Unordered results using pool.imap_unordered():') for x in imap_unordered_it: print('\t', x) print() print('Ordered results using pool.map() --- will block till complete:') for x in pool.map(calculatestar, TASKS): print('\t', x) print() # # Simple benchmarks # N = 100000 print('def pow3(x): return x**3') t = time.time() A = list(map(pow3, range(N))) print('\tmap(pow3, range(%d)):\n\t\t%s seconds' % \ (N, time.time() - t)) t = time.time() B = pool.map(pow3, range(N)) print('\tpool.map(pow3, range(%d)):\n\t\t%s seconds' % \ (N, time.time() - t)) t = time.time() C = list(pool.imap(pow3, range(N), chunksize=N // 8)) print('\tlist(pool.imap(pow3, range(%d), chunksize=%d)):\n\t\t%s' \ ' seconds' % (N, N//8, time.time() - t)) assert A == B == C, (len(A), len(B), len(C)) print() L = [None] * 1000000 print('def noop(x): pass') print('L = [None] * 1000000') t = time.time() A = list(map(noop, L)) print('\tmap(noop, L):\n\t\t%s seconds' % \ (time.time() - t)) t = time.time() B = pool.map(noop, L) print('\tpool.map(noop, L):\n\t\t%s seconds' % \ (time.time() - t)) t = time.time() C = list(pool.imap(noop, L, chunksize=len(L) // 8)) print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \ (len(L)//8, time.time() - t)) assert A == B == C, (len(A), len(B), len(C)) print() del A, B, C, L # # Test error handling # print('Testing error handling:') try: print(pool.apply(f, (5, ))) except ZeroDivisionError: print('\tGot ZeroDivisionError as expected from pool.apply()') else: raise AssertionError('expected ZeroDivisionError') try: print(pool.map(f, range(10))) except ZeroDivisionError: print('\tGot ZeroDivisionError as expected from pool.map()') else: raise AssertionError('expected ZeroDivisionError') try: print(list(pool.imap(f, range(10)))) except ZeroDivisionError: print('\tGot ZeroDivisionError as expected from list(pool.imap())') else: raise AssertionError('expected ZeroDivisionError') it = pool.imap(f, range(10)) for i in range(10): try: x = it.next() except ZeroDivisionError: if i == 5: pass except StopIteration: break else: if i == 5: raise AssertionError('expected ZeroDivisionError') assert i == 9 print('\tGot ZeroDivisionError as expected from IMapIterator.next()') print() # # Testing timeouts # print('Testing ApplyResult.get() with timeout:', end='') res = pool.apply_async(calculate, TASKS[0]) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % res.get(0.02)) break except TimeoutError: sys.stdout.write('.') print() print() print('Testing IMapIterator.next() with timeout:', end='') it = pool.imap(calculatestar, TASKS) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % it.next(0.02)) except StopIteration: break except TimeoutError: sys.stdout.write('.') print() print() # # Testing callback # print('Testing callback:') A = [] B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729] r = pool.apply_async(mul, (7, 8), callback=A.append) r.wait() r = pool.map_async(pow3, range(10), callback=A.extend) r.wait() if A == B: print('\tcallbacks succeeded\n') else: print('\t*** callbacks failed\n\t\t%s != %s\n' % (A, B)) # # Check there are no outstanding tasks # assert not pool._cache, 'cache = %r' % pool._cache # # Check close() methods # print('Testing close():') for worker in pool._pool: assert worker.is_alive() result = pool.apply_async(time.sleep, [0.5]) pool.close() pool.join() assert result.get() is None for worker in pool._pool: assert not worker.is_alive() print('\tclose() succeeded\n') # # Check terminate() method # print('Testing terminate():') pool = Pool(2) ignore = pool.apply(pow3, [2]) results = [pool.apply_async(time.sleep, [10]) for i in range(10)] pool.terminate() pool.join() for worker in pool._pool: assert not worker.is_alive() print('\tterminate() succeeded\n') # # Check garbage collection # print('Testing garbage collection:') pool = Pool(2) processes = pool._pool ignore = pool.apply(pow3, [2]) results = [pool.apply_async(time.sleep, [10]) for i in range(10)] del results, pool time.sleep(0.2) for worker in processes: assert not worker.is_alive() print('\tgarbage collection succeeded\n')
def fmultiprocess( log, function, inputArray, poolSize=False, timeout=3600, **kwargs): """multiprocess pool **Key Arguments:** - ``log`` -- logger - ``function`` -- the function to multiprocess - ``inputArray`` -- the array to be iterated over - ``poolSize`` -- limit the number of CPU that are used in multiprocess job - ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed **Return:** - ``resultArray`` -- the array of results **Usage:** .. code-block:: python from fundamentals import multiprocess # DEFINE AN INPUT ARRAY inputArray = range(10000) results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300, inputArray=inputArray, otherFunctionKeyword="cheese") """ log.debug('starting the ``multiprocess`` function') # DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1) if not poolSize: poolSize = psutil.cpu_count() if poolSize: p = Pool(processes=poolSize) else: p = Pool() cpuCount = psutil.cpu_count() chunksize = int((len(inputArray) + 1) / (cpuCount * 3)) if chunksize == 0: chunksize = 1 # MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES if "log" in inspect.getargspec(function)[0]: mapfunc = partial(function, log=log, **kwargs) resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize) else: mapfunc = partial(function, **kwargs) resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize) resultArray = resultArray.get(timeout=timeout) p.close() p.terminate() log.debug('completed the ``multiprocess`` function') return resultArray
def FVA(model, settings, fileName='', fluxes=-1, inpFvaSettings=None, epsPerc=0.01, lowerBound=10**-8, upperBound=10**3, nOfProcessors=cpu_count() / 2): # Functions have to be included above parallelization task. Cannot use import in normal sense. Windows only probably. def mesg(logFile, mesgToAdd): fl = open(logFile, "a+") fl.write(mesgToAdd) fl.close() def fvaSubProblem(Orinstance, OrOpt, size, optV, lowerBound, upperBound, i): # import statements need to be inside due to parallelization, only necessary for Windows probably from pyomo.environ import Objective, minimize, maximize, value, Constraint from pyomo.opt import TerminationCondition import logging # Suppress pyomo warnings logging.getLogger('pyomo.core').setLevel(logging.ERROR) # initialize (min, max, termination condition at min, termination condition at max, index, opt. v w/ biomass) fva = [[], [], [], [], [], []] # (ABSOLUTELY NECESSARY) though not entirely sure why... instance = Orinstance Opt = OrOpt # minimize flux if optV[i] < lowerBound: minState = 'Flux is lower than ' + str( lowerBound) + ' in optimal solution' minV = 0 else: try: # change objective to minimizing flux instance.minFlux = Objective( expr=instance.v[i], sense=minimize) # default sense is minimize instance.minFlux.activate() # try to avoid sub. term. instance.boxMin = Constraint(expr=instance.v[i] >= lowerBound) instance.boxMin.activate() # solve min. problem Soln = Opt.solve(instance, tee=False) if Soln.solver.termination_condition == TerminationCondition.unbounded: minV = lowerBound elif Soln.solver.termination_condition == TerminationCondition.optimal: minV = value(instance.minFlux) else: minV = -1 minState = str(Soln.solver.termination_condition) instance.minFlux.deactivate() instance.boxMin.deactivate() except Exception as e: instance.minFlux.deactivate() instance.boxMin.deactivate() minState = 'Cannot load a SolverResults object with bad status: error' minV = -1 # print(e) # maximize flux if optV[i] >= upperBound: maxState = 'Flux is larger than ' + str( upperBound) + ' in optimal solution.' maxV = upperBound else: try: # change objective to maximizing flux instance.maxFlux = Objective(expr=instance.v[i], sense=maximize) instance.maxFlux.activate() # set box constraint on flux to roughly predict unboundedness. instance.boxMax = Constraint(expr=instance.v[i] <= upperBound) instance.boxMax.activate() # solve max. problem Soln = Opt.solve(instance, tee=False) if Soln.solver.termination_condition == TerminationCondition.unbounded: maxV = upperBound elif Soln.solver.termination_condition == TerminationCondition.optimal: maxV = value(instance.maxFlux) else: maxV = -1 maxState = str(Soln.solver.termination_condition) instance.maxFlux.deactivate() instance.boxMax.deactivate() except Exception as e: instance.maxFlux.deactivate() instance.boxMax.deactivate() maxState = 'Cannot load a SolverResults object with bad status: error' maxV = -1 # print(e) # update FVA list fva[0] = minV fva[1] = maxV fva[2] = minState fva[3] = maxState fva[4] = i fva[5] = optV[i] # check for errors to log minError = '' maxError = '' if minV == -1: minError = '. Error in minimization problem: ' + minState if maxV == -1: maxError = 'Error in maximization problem: ' + maxState # send update to screen mesg( settings['logFile'], '\tSolved FVA subproblem ' + str(i) + ' of ' + str(size - 1) + minError + '. ' + maxError + '\n') # we're done return fva # following if-sentence necessary in Windows for use of multiprocess package if __name__ == '__main__': # initialize fvaData = [] # Sets logfile to default filename and checks if logFile already exists if 'logFile' not in settings: settings['logFile'] = 'logFileFVA' if os.path.exists(settings['logFile']): os.remove(settings['logFile']) try: # set fvaSettings to default if none is chosen fvaSettings = { 'solver': 'gurobi', 'NumericFocus': 3, 'Aggregate': 0, 'BarQCPConvTol': 10**-3, 'BarHomogeneous': 1, 'Presolve': -1, 'DualReductions': 1, 'BarCorrectors': -1, 'Method': 2 } if inpFvaSettings is not None: if set(inpFvaSettings.keys()).issubset(fvaSettings.keys()): for key, item in inpFvaSettings.items(): fvaSettings[key] = item except: mesg(settings['logFile'], 'Error: could not load settings argument properly') return fvaData mesg(settings['logFile'], 'Initializing FVA process\n\n') # measure time start_time = time.time() # set solver options for FVAsubproblem. Interior point experiences less problems differentiating feasibility/unboundedness # setting all settings to default except for Aggregate, which is turned off (=0), and BarQCPConvTol set to 10**-3, seems to work fine. solver = fvaSettings['solver'] OrOpt = SolverFactory(solver) OrOpt.options['NumericFocus'] = fvaSettings[ 'NumericFocus'] # 0 default. When set to zero, multiple max. problems cannot differentiate infeasible and unbounded problems. OrOpt.options['Presolve'] = fvaSettings['Presolve'] # -1 default OrOpt.options['Aggregate'] = fvaSettings[ 'Aggregate'] # Presolver options. Gurobi docs recommend turning this off if num. trouble, 1 - default, 0 - off OrOpt.options['DualReductions'] = fvaSettings[ 'DualReductions'] # put to zero to verify if unbounded or infeasible OrOpt.options['BarQCPConvTol'] = fvaSettings['BarQCPConvTol'] OrOpt.options['BarHomogeneous'] = fvaSettings[ 'BarHomogeneous'] # -1 default (automatic), 0 off, 1 on: Some min problems results in unloadable SolveResults object if not turned on OrOpt.options['BarCorrectors'] = fvaSettings['BarCorrectors'] OrOpt.options['Method'] = fvaSettings['Method'] # set this automatically since we will need the original instance anyway settings['retrieveInstance'] = True mesg( settings['logFile'], 'Starting to solve initial optimization problem to find optimal growth rate\n' ) # fetch opt. growth rate and flux state and pyomo concrete model. try: sol, stat = uncModel(model, settings) grwthRate = sol['grwthRate'] Oinstance = sol['instance'] optV = sol['optFlux'] cS = sol['cS'] except: mesg(settings['logFile'], 'Error: could not solve initial optimization problem\n') return fvaData mesg( settings['logFile'], 'Initial optimization problem solved. Optimal growth rate is ' + str(grwthRate) + ' g/gDWh\n\n') # constrain biomass reaction def bioConstr(instance): return sum(cS[j] * instance.v[j] for j in instance.flux) >= (1 - epsPerc) * grwthRate # fix growth rate and deactivate biomass as objective Oinstance.con = Constraint(rule=bioConstr) Oinstance.grwthRate.deactivate() # set flux to default if type(fluxes) is not list and fluxes == -1: fluxes = range(len(optV)) mesg( settings['logFile'], 'Initializing parallelization FVA subproblem tasks on ' + str(nOfProcessors) + ' processors\n') try: # Parallelize size = len(fluxes) fvaData = [] p = Pool(nOfProcessors) func = partial(fvaSubProblem, Oinstance, OrOpt, size, optV, lowerBound, upperBound) r = p.map_async(func, fluxes, callback=fvaData.append) r.wait() except: mesg(settings['logFile'], 'Error: could not parallelize FVA subproblems\n') return fvaData # it is created as a single list of list fvaData = fvaData[0] # Add the model and its settings to fvaData try: arguments = ', epsPerc = ' + str( epsPerc) + ', lowerBound = ' + str( lowerBound) + ', upperBound = ' + str(upperBound) fvaData.append(model + ' ' + str(settings) + arguments) except: mesg( settings['logFile'], 'Error: could not append settings to FVA data list. Check if output is empty' ) if fileName != '': try: # saves fva using pickle with open(fileName, 'wb') as f: pickle.dump(fvaData, f) except: mesg(settings['logFile'], 'Error: could not save FVA data list to file') return fvaData # print where FVA is saved and time duration if fileName != '': mesg( settings['logFile'], '\nFVA results saved as ' + fileName + ' using the pickle package.') mesg( settings['logFile'], '\nDone. FVA results saved as ' + fileName + ' using the pickle package. Duration of FVA: ' + str(round(time.time() - start_time)) + ' s\n\n') return fvaData
def test(): print('cpuCount() = %d\n' % cpuCount()) # # Create pool # PROCESSES = 4 print('Creating pool with %d processes\n' % PROCESSES) pool = Pool(PROCESSES) # # Tests # TASKS = [(mul, (i, 7)) for i in range(10)] + \ [(plus, (i, 8)) for i in range(10)] results = [pool.apply_async(calculate, t) for t in TASKS] imap_it = pool.imap(calculatestar, TASKS) imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) print('Ordered results using pool.apply_async():') for r in results: print('\t', r.get()) print() print('Ordered results using pool.imap():') for x in imap_it: print('\t', x) print() print('Unordered results using pool.imap_unordered():') for x in imap_unordered_it: print('\t', x) print() print('Ordered results using pool.map() --- will block till complete:') for x in pool.map(calculatestar, TASKS): print('\t', x) print() # # Simple benchmarks # N = 100000 print('def pow3(x): return x**3') t = time.time() A = list(map(pow3, xrange(N))) print('\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \ (N, time.time() - t)) t = time.time() B = pool.map(pow3, xrange(N)) print('\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \ (N, time.time() - t)) t = time.time() C = list(pool.imap(pow3, xrange(N), chunksize=N//8)) print('\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \ ' seconds' % (N, N//8, time.time() - t)) assert A == B == C, (len(A), len(B), len(C)) print() L = [None] * 1000000 print('def noop(x): pass') print('L = [None] * 1000000') t = time.time() A = list(map(noop, L)) print('\tmap(noop, L):\n\t\t%s seconds' % \ (time.time() - t)) t = time.time() B = pool.map(noop, L) print('\tpool.map(noop, L):\n\t\t%s seconds' % \ (time.time() - t)) t = time.time() C = list(pool.imap(noop, L, chunksize=len(L)//8)) print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \ (len(L)//8, time.time() - t)) assert A == B == C, (len(A), len(B), len(C)) print() del A, B, C, L # # Test error handling # print('Testing error handling:') try: print(pool.apply(f, (5,))) except ZeroDivisionError: print('\tGot ZeroDivisionError as expected from pool.apply()') else: raise AssertionError('expected ZeroDivisionError') try: print(pool.map(f, range(10))) except ZeroDivisionError: print('\tGot ZeroDivisionError as expected from pool.map()') else: raise AssertionError('expected ZeroDivisionError') try: print(list(pool.imap(f, range(10)))) except ZeroDivisionError: print('\tGot ZeroDivisionError as expected from list(pool.imap())') else: raise AssertionError('expected ZeroDivisionError') it = pool.imap(f, range(10)) for i in range(10): try: x = it.next() except ZeroDivisionError: if i == 5: pass except StopIteration: break else: if i == 5: raise AssertionError('expected ZeroDivisionError') assert i == 9 print('\tGot ZeroDivisionError as expected from IMapIterator.next()') print() # # Testing timeouts # print('Testing ApplyResult.get() with timeout:', end='') res = pool.apply_async(calculate, TASKS[0]) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % res.get(0.02)) break except TimeoutError: sys.stdout.write('.') print() print() print('Testing IMapIterator.next() with timeout:', end='') it = pool.imap(calculatestar, TASKS) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % it.next(0.02)) except StopIteration: break except TimeoutError: sys.stdout.write('.') print() print() # # Testing callback # print('Testing callback:') A = [] B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729] r = pool.apply_async(mul, (7, 8), callback=A.append) r.wait() r = pool.map_async(pow3, range(10), callback=A.extend) r.wait() if A == B: print('\tcallbacks succeeded\n') else: print('\t*** callbacks failed\n\t\t%s != %s\n' % (A, B)) # # Check there are no outstanding tasks # assert not pool._cache, 'cache = %r' % pool._cache # # Check close() methods # print('Testing close():') for worker in pool._pool: assert worker.is_alive() result = pool.apply_async(time.sleep, [0.5]) pool.close() pool.join() assert result.get() is None for worker in pool._pool: assert not worker.is_alive() print('\tclose() succeeded\n') # # Check terminate() method # print('Testing terminate():') pool = Pool(2) ignore = pool.apply(pow3, [2]) results = [pool.apply_async(time.sleep, [10]) for i in range(10)] pool.terminate() pool.join() for worker in pool._pool: assert not worker.is_alive() print('\tterminate() succeeded\n') # # Check garbage collection # print('Testing garbage collection:') pool = Pool(2) processes = pool._pool ignore = pool.apply(pow3, [2]) results = [pool.apply_async(time.sleep, [10]) for i in range(10)] del results, pool time.sleep(0.2) for worker in processes: assert not worker.is_alive() print('\tgarbage collection succeeded\n')