def fmultiprocess(log,
function,
inputArray,
poolSize=False,
timeout=3600,
**kwargs):
"""multiprocess pool
**Key Arguments:**
- ``log`` -- logger
- ``function`` -- the function to multiprocess
- ``inputArray`` -- the array to be iterated over
- ``poolSize`` -- limit the number of CPU that are used in multiprocess job
- ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed
**Return:**
- ``resultArray`` -- the array of results
**Usage:**
.. code-block:: python
from fundamentals import multiprocess
# DEFINE AN INPUT ARRAY
inputArray = range(10000)
results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300,
inputArray=inputArray, otherFunctionKeyword="cheese")
"""
log.debug('starting the ``multiprocess`` function')
# DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1)
if not poolSize:
poolSize = psutil.cpu_count()
if poolSize:
p = Pool(processes=poolSize)
else:
p = Pool()
# MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES
if "log" in inspect.getargspec(function)[0]:
mapfunc = partial(function, log=log, **kwargs)
resultArray = p.map_async(mapfunc, inputArray)
else:
mapfunc = partial(function, **kwargs)
resultArray = p.map_async(mapfunc, inputArray)
resultArray = resultArray.get(timeout=timeout)
p.close()
p.terminate()
log.debug('completed the ``multiprocess`` function')
return resultArray
class MultiThreading(object):
def __init__(self, funct, data, threads='all'):
raise Exception("Not functionnal yet !")
self.funct = funct
if threads == 'all':
threads = cpu_count()
self.pool = Pool(processes=threads)
self.data = data
self.PG = None
self.initializer = None
self.finalizer = None
def add_progress_counter(self, init_mess="Beginning", end_mess="Done",
name_things='things', perc_interv=5):
self.PG = ProgressCounter(init_mess=init_mess, end_mess=end_mess,
nmb_max=len(self.data),
name_things=name_things,
perc_interv=perc_interv)
self.manager = Manager()
self.manager.register("PG", self.PG)
def run(self):
res = self.pool.map_async(self.PG_func_wrapper, self.data)
self.pool.close()
self.pool.join()
return res
def predict(self, inputData, transientTime=0, update_processor=lambda x: x, verbose=0):
rank = len(inputData.shape) - 1
if rank != self.n_inputDimensions:
raise ValueError(
"The `inputData` does not have a suitable shape. It has to have {0} spatial dimensions and 1 temporal dimension.".format(
self.n_inputDimensions))
manager = Manager()
predictQueue = manager.Queue()
# workaround as predict does not support batches atm
# add dummy dimension to let embedInputData work properly (is optimized to work for batches)
inputData = inputData.reshape(1, *inputData.shape)
modifiedInputData = self._embedInputData(inputData)
modifiedInputData = modifiedInputData[0]
inputData = inputData[0]
self.transientTime = transientTime
self.sharedNamespace.transientTime = transientTime
predictionOutput = B.zeros(np.insert(self.inputShape, 0, inputData.shape[0] - transientTime))
jobs = np.stack(np.meshgrid(*[np.arange(x) + self._filterWidth for x in inputData.shape[1:]]),
axis=rank).reshape(-1, rank).tolist()
nJobs = len(jobs)
self.resetState()
iterator = PredictionArrayIterator(modifiedInputData, jobs, self._filterWidth, self._stride, self)
pool = Pool(processes=self._nWorkers, initializer=SpatioTemporalESN._init_predictProcess,
initargs=[predictQueue, self])
pool.map_async(self._predictProcess, iterator, chunksize=200)#, chunksize=1)
def _processPoolWorkerResults():
nJobsDone = 0
if verbose > 0:
bar = progressbar.ProgressBar(max_value=nJobs, redirect_stdout=True, poll_interval=0.0001)
bar.update(0)
while nJobsDone < nJobs:
data = predictQueue.get()
# result of predicting
indices, prediction, state = data
id = self._uniqueIDFromIndices(indices)
self._xs[id] = state
# update the values
predictionOutput[tuple([Ellipsis] + indices)] = prediction
nJobsDone += 1
if verbose > 0:
bar.update(nJobsDone)
if verbose > 1:
print(nJobsDone)
if verbose > 0:
bar.finish()
_processPoolWorkerResults()
pool.close()
return predictionOutput
def fit(self, inputData, outputData, transientTime=0, verbose=0):
rank = len(inputData.shape) - 1
if rank != self.n_inputDimensions and rank != self.n_inputDimensions + 1:
raise ValueError(
"The `inputData` does not have a suitable shape. It has to have {0} spatial dimensions and 1 temporal dimension.".format(
self.n_inputDimensions))
# reshape the input so that it has the shape (timeseries, time, input_dimension^n)
if rank == self.n_inputDimensions:
inputData = inputData.reshape(1, *inputData.shape)
outputData = outputData.reshape(1, *outputData.shape)
else:
# modify rank again
rank -= 1
partialLength = (inputData.shape[1] - transientTime)
totalLength = inputData.shape[0] * partialLength
timeseriesCount = inputData.shape[0]
manager = Manager()
fitQueue = manager.Queue()
modifiedInputData = self._embedInputData(inputData)
self.sharedNamespace.transientTime = transientTime
self.sharedNamespace.partialLength = partialLength
self.sharedNamespace.totalLength = totalLength
self.sharedNamespace.timeseriesCount = timeseriesCount
jobs = np.stack(np.meshgrid(*[np.arange(x) + self._filterWidth for x in inputData.shape[2:]]),
axis=rank).reshape(-1, rank).tolist()
nJobs = len(jobs)
self.resetState()
iterator = FittingArrayIterator(modifiedInputData, outputData, jobs, self._filterWidth, self._stride, self)
pool = Pool(processes=self._nWorkers, initializer=SpatioTemporalESN._init_fitProcess, initargs=[fitQueue, self])
pool.map_async(self._fitProcess, iterator, chunksize=16)
def _processPoolWorkerResults():
nJobsDone = 0
if verbose > 0:
bar = progressbar.ProgressBar(max_value=nJobs, redirect_stdout=True, poll_interval=0.0001)
bar.update(0)
while nJobsDone < nJobs:
data = fitQueue.get()
# result of fitting
indices, x, WOut = data
id = self._uniqueIDFromIndices(indices)
if WOut is None:
import sys
print("WARNING: Fit process for pixel {0} did not succeed".format(indices), file=sys.stderr)
# store WOut
if self._averageOutputWeights:
if WOut is not None:
self._WOut += WOut / np.prod(self.inputShape)
else:
self._WOuts[id] = WOut
# store x
self._xs[id] = x
nJobsDone += 1
if verbose > 0:
bar.update(nJobsDone)
if verbose > 1:
print(nJobsDone)
if verbose > 0:
bar.finish()
_processPoolWorkerResults()
pool.close()
from multiprocess import Pool
def f(x):
return x * x
if __name__ == '__main__':
p = Pool(4)
result = p.map_async(f, range(10))
print(result.get(timeout=1))
def _multi_channel_apply_disk_parallel(self, function, cleanup_function,
output_path, from_time, to_time,
channels, cast_dtype,
pass_batch_info, pass_batch_results,
processes, **kwargs):
self.logger.debug('Starting parallel operation...')
if pass_batch_results:
raise NotImplementedError("pass_batch_results is not "
"implemented on 'disk' mode")
# need to convert to a list, oherwise cannot be pickled
data = list(
self.multi_channel(from_time, to_time, channels,
return_data=False))
n_batches = self.indexer.n_batches(from_time, to_time, channels)
self.logger.info('Data will be splitted in %s batches', n_batches)
output_path = Path(output_path)
# create local variables to avoid pickling problems
_path_to_recordings = copy(self.path_to_recordings)
_dtype = copy(self.dtype)
_n_channels = copy(self.n_channels)
_data_order = copy(self.data_order)
_loader = copy(self.loader)
_buffer_size = copy(self.buffer_size)
reader = partial(RecordingsReader,
path_to_recordings=_path_to_recordings,
dtype=_dtype,
n_channels=_n_channels,
data_order=_data_order,
loader=_loader,
return_data_index=True)
m = Manager()
mapping = m.dict()
next_to_write = m.Value('i', 0)
def parallel_runner(element):
i, _ = element
res = util.batch_runner(element,
function,
reader,
pass_batch_info,
cast_dtype,
kwargs,
cleanup_function,
_buffer_size,
save_chunks=False,
output_path=output_path)
if i == 0:
mapping['dtype'] = str(res.dtype)
while True:
if next_to_write.value == i:
with open(str(output_path), 'wb' if i == 0 else 'ab') as f:
res.tofile(f)
next_to_write.value += 1
break
# run jobs
self.logger.debug('Creating processes pool...')
p = Pool(processes)
res = p.map_async(parallel_runner, enumerate(data))
finished = 0
if self.show_progress_bar:
pbar = tqdm(total=n_batches)
if self.show_progress_bar:
while True:
if next_to_write.value > finished:
update = next_to_write.value - finished
pbar.update(update)
finished = next_to_write.value
if next_to_write.value == n_batches:
break
pbar.close()
else:
res.get()
# save metadata
params = util.make_metadata(channels, self.n_channels,
mapping['dtype'], output_path)
return output_path, params
def test():
print('cpuCount() = %d\n' % cpuCount())
#
# Create pool
#
PROCESSES = 4
print('Creating pool with %d processes\n' % PROCESSES)
pool = Pool(PROCESSES)
#
# Tests
#
TASKS = [(mul, (i, 7)) for i in range(10)] + \
[(plus, (i, 8)) for i in range(10)]
results = [pool.apply_async(calculate, t) for t in TASKS]
imap_it = pool.imap(calculatestar, TASKS)
imap_unordered_it = pool.imap_unordered(calculatestar, TASKS)
print('Ordered results using pool.apply_async():')
for r in results:
print('\t', r.get())
print()
print('Ordered results using pool.imap():')
for x in imap_it:
print('\t', x)
print()
print('Unordered results using pool.imap_unordered():')
for x in imap_unordered_it:
print('\t', x)
print()
print('Ordered results using pool.map() --- will block till complete:')
for x in pool.map(calculatestar, TASKS):
print('\t', x)
print()
#
# Simple benchmarks
#
N = 100000
print('def pow3(x): return x**3')
t = time.time()
A = list(map(pow3, range(N)))
print('\tmap(pow3, range(%d)):\n\t\t%s seconds' % \
(N, time.time() - t))
t = time.time()
B = pool.map(pow3, range(N))
print('\tpool.map(pow3, range(%d)):\n\t\t%s seconds' % \
(N, time.time() - t))
t = time.time()
C = list(pool.imap(pow3, range(N), chunksize=N // 8))
print('\tlist(pool.imap(pow3, range(%d), chunksize=%d)):\n\t\t%s' \
' seconds' % (N, N//8, time.time() - t))
assert A == B == C, (len(A), len(B), len(C))
print()
L = [None] * 1000000
print('def noop(x): pass')
print('L = [None] * 1000000')
t = time.time()
A = list(map(noop, L))
print('\tmap(noop, L):\n\t\t%s seconds' % \
(time.time() - t))
t = time.time()
B = pool.map(noop, L)
print('\tpool.map(noop, L):\n\t\t%s seconds' % \
(time.time() - t))
t = time.time()
C = list(pool.imap(noop, L, chunksize=len(L) // 8))
print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \
(len(L)//8, time.time() - t))
assert A == B == C, (len(A), len(B), len(C))
print()
del A, B, C, L
#
# Test error handling
#
print('Testing error handling:')
try:
print(pool.apply(f, (5, )))
except ZeroDivisionError:
print('\tGot ZeroDivisionError as expected from pool.apply()')
else:
raise AssertionError('expected ZeroDivisionError')
try:
print(pool.map(f, range(10)))
except ZeroDivisionError:
print('\tGot ZeroDivisionError as expected from pool.map()')
else:
raise AssertionError('expected ZeroDivisionError')
try:
print(list(pool.imap(f, range(10))))
except ZeroDivisionError:
print('\tGot ZeroDivisionError as expected from list(pool.imap())')
else:
raise AssertionError('expected ZeroDivisionError')
it = pool.imap(f, range(10))
for i in range(10):
try:
x = it.next()
except ZeroDivisionError:
if i == 5:
pass
except StopIteration:
break
else:
if i == 5:
raise AssertionError('expected ZeroDivisionError')
assert i == 9
print('\tGot ZeroDivisionError as expected from IMapIterator.next()')
print()
#
# Testing timeouts
#
print('Testing ApplyResult.get() with timeout:', end='')
res = pool.apply_async(calculate, TASKS[0])
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % res.get(0.02))
break
except TimeoutError:
sys.stdout.write('.')
print()
print()
print('Testing IMapIterator.next() with timeout:', end='')
it = pool.imap(calculatestar, TASKS)
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % it.next(0.02))
except StopIteration:
break
except TimeoutError:
sys.stdout.write('.')
print()
print()
#
# Testing callback
#
print('Testing callback:')
A = []
B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729]
r = pool.apply_async(mul, (7, 8), callback=A.append)
r.wait()
r = pool.map_async(pow3, range(10), callback=A.extend)
r.wait()
if A == B:
print('\tcallbacks succeeded\n')
else:
print('\t*** callbacks failed\n\t\t%s != %s\n' % (A, B))
#
# Check there are no outstanding tasks
#
assert not pool._cache, 'cache = %r' % pool._cache
#
# Check close() methods
#
print('Testing close():')
for worker in pool._pool:
assert worker.is_alive()
result = pool.apply_async(time.sleep, [0.5])
pool.close()
pool.join()
assert result.get() is None
for worker in pool._pool:
assert not worker.is_alive()
print('\tclose() succeeded\n')
#
# Check terminate() method
#
print('Testing terminate():')
pool = Pool(2)
ignore = pool.apply(pow3, [2])
results = [pool.apply_async(time.sleep, [10]) for i in range(10)]
pool.terminate()
pool.join()
for worker in pool._pool:
assert not worker.is_alive()
print('\tterminate() succeeded\n')
#
# Check garbage collection
#
print('Testing garbage collection:')
pool = Pool(2)
processes = pool._pool
ignore = pool.apply(pow3, [2])
results = [pool.apply_async(time.sleep, [10]) for i in range(10)]
del results, pool
time.sleep(0.2)
for worker in processes:
assert not worker.is_alive()
print('\tgarbage collection succeeded\n')
def fmultiprocess(
log,
function,
inputArray,
poolSize=False,
timeout=3600,
**kwargs):
"""multiprocess pool
**Key Arguments:**
- ``log`` -- logger
- ``function`` -- the function to multiprocess
- ``inputArray`` -- the array to be iterated over
- ``poolSize`` -- limit the number of CPU that are used in multiprocess job
- ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed
**Return:**
- ``resultArray`` -- the array of results
**Usage:**
.. code-block:: python
from fundamentals import multiprocess
# DEFINE AN INPUT ARRAY
inputArray = range(10000)
results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300,
inputArray=inputArray, otherFunctionKeyword="cheese")
"""
log.debug('starting the ``multiprocess`` function')
# DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1)
if not poolSize:
poolSize = psutil.cpu_count()
if poolSize:
p = Pool(processes=poolSize)
else:
p = Pool()
cpuCount = psutil.cpu_count()
chunksize = int((len(inputArray) + 1) / (cpuCount * 3))
if chunksize == 0:
chunksize = 1
# MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES
if "log" in inspect.getargspec(function)[0]:
mapfunc = partial(function, log=log, **kwargs)
resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)
else:
mapfunc = partial(function, **kwargs)
resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)
resultArray = resultArray.get(timeout=timeout)
p.close()
p.terminate()
log.debug('completed the ``multiprocess`` function')
return resultArray
def FVA(model,
settings,
fileName='',
fluxes=-1,
inpFvaSettings=None,
epsPerc=0.01,
lowerBound=10**-8,
upperBound=10**3,
nOfProcessors=cpu_count() / 2):
# Functions have to be included above parallelization task. Cannot use import in normal sense. Windows only probably.
def mesg(logFile, mesgToAdd):
fl = open(logFile, "a+")
fl.write(mesgToAdd)
fl.close()
def fvaSubProblem(Orinstance, OrOpt, size, optV, lowerBound, upperBound,
i):
# import statements need to be inside due to parallelization, only necessary for Windows probably
from pyomo.environ import Objective, minimize, maximize, value, Constraint
from pyomo.opt import TerminationCondition
import logging
# Suppress pyomo warnings
logging.getLogger('pyomo.core').setLevel(logging.ERROR)
# initialize (min, max, termination condition at min, termination condition at max, index, opt. v w/ biomass)
fva = [[], [], [], [], [], []]
# (ABSOLUTELY NECESSARY) though not entirely sure why...
instance = Orinstance
Opt = OrOpt
# minimize flux
if optV[i] < lowerBound:
minState = 'Flux is lower than ' + str(
lowerBound) + ' in optimal solution'
minV = 0
else:
try:
# change objective to minimizing flux
instance.minFlux = Objective(
expr=instance.v[i],
sense=minimize) # default sense is minimize
instance.minFlux.activate()
# try to avoid sub. term.
instance.boxMin = Constraint(expr=instance.v[i] >= lowerBound)
instance.boxMin.activate()
# solve min. problem
Soln = Opt.solve(instance, tee=False)
if Soln.solver.termination_condition == TerminationCondition.unbounded:
minV = lowerBound
elif Soln.solver.termination_condition == TerminationCondition.optimal:
minV = value(instance.minFlux)
else:
minV = -1
minState = str(Soln.solver.termination_condition)
instance.minFlux.deactivate()
instance.boxMin.deactivate()
except Exception as e:
instance.minFlux.deactivate()
instance.boxMin.deactivate()
minState = 'Cannot load a SolverResults object with bad status: error'
minV = -1
# print(e)
# maximize flux
if optV[i] >= upperBound:
maxState = 'Flux is larger than ' + str(
upperBound) + ' in optimal solution.'
maxV = upperBound
else:
try:
# change objective to maximizing flux
instance.maxFlux = Objective(expr=instance.v[i],
sense=maximize)
instance.maxFlux.activate()
# set box constraint on flux to roughly predict unboundedness.
instance.boxMax = Constraint(expr=instance.v[i] <= upperBound)
instance.boxMax.activate()
# solve max. problem
Soln = Opt.solve(instance, tee=False)
if Soln.solver.termination_condition == TerminationCondition.unbounded:
maxV = upperBound
elif Soln.solver.termination_condition == TerminationCondition.optimal:
maxV = value(instance.maxFlux)
else:
maxV = -1
maxState = str(Soln.solver.termination_condition)
instance.maxFlux.deactivate()
instance.boxMax.deactivate()
except Exception as e:
instance.maxFlux.deactivate()
instance.boxMax.deactivate()
maxState = 'Cannot load a SolverResults object with bad status: error'
maxV = -1
# print(e)
# update FVA list
fva[0] = minV
fva[1] = maxV
fva[2] = minState
fva[3] = maxState
fva[4] = i
fva[5] = optV[i]
# check for errors to log
minError = ''
maxError = ''
if minV == -1:
minError = '. Error in minimization problem: ' + minState
if maxV == -1:
maxError = 'Error in maximization problem: ' + maxState
# send update to screen
mesg(
settings['logFile'], '\tSolved FVA subproblem ' + str(i) + ' of ' +
str(size - 1) + minError + '. ' + maxError + '\n')
# we're done
return fva
# following if-sentence necessary in Windows for use of multiprocess package
if __name__ == '__main__':
# initialize
fvaData = []
# Sets logfile to default filename and checks if logFile already exists
if 'logFile' not in settings:
settings['logFile'] = 'logFileFVA'
if os.path.exists(settings['logFile']):
os.remove(settings['logFile'])
try:
# set fvaSettings to default if none is chosen
fvaSettings = {
'solver': 'gurobi',
'NumericFocus': 3,
'Aggregate': 0,
'BarQCPConvTol': 10**-3,
'BarHomogeneous': 1,
'Presolve': -1,
'DualReductions': 1,
'BarCorrectors': -1,
'Method': 2
}
if inpFvaSettings is not None:
if set(inpFvaSettings.keys()).issubset(fvaSettings.keys()):
for key, item in inpFvaSettings.items():
fvaSettings[key] = item
except:
mesg(settings['logFile'],
'Error: could not load settings argument properly')
return fvaData
mesg(settings['logFile'], 'Initializing FVA process\n\n')
# measure time
start_time = time.time()
# set solver options for FVAsubproblem. Interior point experiences less problems differentiating feasibility/unboundedness
# setting all settings to default except for Aggregate, which is turned off (=0), and BarQCPConvTol set to 10**-3, seems to work fine.
solver = fvaSettings['solver']
OrOpt = SolverFactory(solver)
OrOpt.options['NumericFocus'] = fvaSettings[
'NumericFocus'] # 0 default. When set to zero, multiple max. problems cannot differentiate infeasible and unbounded problems.
OrOpt.options['Presolve'] = fvaSettings['Presolve'] # -1 default
OrOpt.options['Aggregate'] = fvaSettings[
'Aggregate'] # Presolver options. Gurobi docs recommend turning this off if num. trouble, 1 - default, 0 - off
OrOpt.options['DualReductions'] = fvaSettings[
'DualReductions'] # put to zero to verify if unbounded or infeasible
OrOpt.options['BarQCPConvTol'] = fvaSettings['BarQCPConvTol']
OrOpt.options['BarHomogeneous'] = fvaSettings[
'BarHomogeneous'] # -1 default (automatic), 0 off, 1 on: Some min problems results in unloadable SolveResults object if not turned on
OrOpt.options['BarCorrectors'] = fvaSettings['BarCorrectors']
OrOpt.options['Method'] = fvaSettings['Method']
# set this automatically since we will need the original instance anyway
settings['retrieveInstance'] = True
mesg(
settings['logFile'],
'Starting to solve initial optimization problem to find optimal growth rate\n'
)
# fetch opt. growth rate and flux state and pyomo concrete model.
try:
sol, stat = uncModel(model, settings)
grwthRate = sol['grwthRate']
Oinstance = sol['instance']
optV = sol['optFlux']
cS = sol['cS']
except:
mesg(settings['logFile'],
'Error: could not solve initial optimization problem\n')
return fvaData
mesg(
settings['logFile'],
'Initial optimization problem solved. Optimal growth rate is ' +
str(grwthRate) + ' g/gDWh\n\n')
# constrain biomass reaction
def bioConstr(instance):
return sum(cS[j] * instance.v[j]
for j in instance.flux) >= (1 - epsPerc) * grwthRate
# fix growth rate and deactivate biomass as objective
Oinstance.con = Constraint(rule=bioConstr)
Oinstance.grwthRate.deactivate()
# set flux to default
if type(fluxes) is not list and fluxes == -1:
fluxes = range(len(optV))
mesg(
settings['logFile'],
'Initializing parallelization FVA subproblem tasks on ' +
str(nOfProcessors) + ' processors\n')
try:
# Parallelize
size = len(fluxes)
fvaData = []
p = Pool(nOfProcessors)
func = partial(fvaSubProblem, Oinstance, OrOpt, size, optV,
lowerBound, upperBound)
r = p.map_async(func, fluxes, callback=fvaData.append)
r.wait()
except:
mesg(settings['logFile'],
'Error: could not parallelize FVA subproblems\n')
return fvaData
# it is created as a single list of list
fvaData = fvaData[0]
# Add the model and its settings to fvaData
try:
arguments = ', epsPerc = ' + str(
epsPerc) + ', lowerBound = ' + str(
lowerBound) + ', upperBound = ' + str(upperBound)
fvaData.append(model + ' ' + str(settings) + arguments)
except:
mesg(
settings['logFile'],
'Error: could not append settings to FVA data list. Check if output is empty'
)
if fileName != '':
try:
# saves fva using pickle
with open(fileName, 'wb') as f:
pickle.dump(fvaData, f)
except:
mesg(settings['logFile'],
'Error: could not save FVA data list to file')
return fvaData
# print where FVA is saved and time duration
if fileName != '':
mesg(
settings['logFile'], '\nFVA results saved as ' + fileName +
' using the pickle package.')
mesg(
settings['logFile'], '\nDone. FVA results saved as ' + fileName +
' using the pickle package. Duration of FVA: ' +
str(round(time.time() - start_time)) + ' s\n\n')
return fvaData
def test():
print('cpuCount() = %d\n' % cpuCount())
#
# Create pool
#
PROCESSES = 4
print('Creating pool with %d processes\n' % PROCESSES)
pool = Pool(PROCESSES)
#
# Tests
#
TASKS = [(mul, (i, 7)) for i in range(10)] + \
[(plus, (i, 8)) for i in range(10)]
results = [pool.apply_async(calculate, t) for t in TASKS]
imap_it = pool.imap(calculatestar, TASKS)
imap_unordered_it = pool.imap_unordered(calculatestar, TASKS)
print('Ordered results using pool.apply_async():')
for r in results:
print('\t', r.get())
print()
print('Ordered results using pool.imap():')
for x in imap_it:
print('\t', x)
print()
print('Unordered results using pool.imap_unordered():')
for x in imap_unordered_it:
print('\t', x)
print()
print('Ordered results using pool.map() --- will block till complete:')
for x in pool.map(calculatestar, TASKS):
print('\t', x)
print()
#
# Simple benchmarks
#
N = 100000
print('def pow3(x): return x**3')
t = time.time()
A = list(map(pow3, xrange(N)))
print('\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t))
t = time.time()
B = pool.map(pow3, xrange(N))
print('\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t))
t = time.time()
C = list(pool.imap(pow3, xrange(N), chunksize=N//8))
print('\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \
' seconds' % (N, N//8, time.time() - t))
assert A == B == C, (len(A), len(B), len(C))
print()
L = [None] * 1000000
print('def noop(x): pass')
print('L = [None] * 1000000')
t = time.time()
A = list(map(noop, L))
print('\tmap(noop, L):\n\t\t%s seconds' % \
(time.time() - t))
t = time.time()
B = pool.map(noop, L)
print('\tpool.map(noop, L):\n\t\t%s seconds' % \
(time.time() - t))
t = time.time()
C = list(pool.imap(noop, L, chunksize=len(L)//8))
print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \
(len(L)//8, time.time() - t))
assert A == B == C, (len(A), len(B), len(C))
print()
del A, B, C, L
#
# Test error handling
#
print('Testing error handling:')
try:
print(pool.apply(f, (5,)))
except ZeroDivisionError:
print('\tGot ZeroDivisionError as expected from pool.apply()')
else:
raise AssertionError('expected ZeroDivisionError')
try:
print(pool.map(f, range(10)))
except ZeroDivisionError:
print('\tGot ZeroDivisionError as expected from pool.map()')
else:
raise AssertionError('expected ZeroDivisionError')
try:
print(list(pool.imap(f, range(10))))
except ZeroDivisionError:
print('\tGot ZeroDivisionError as expected from list(pool.imap())')
else:
raise AssertionError('expected ZeroDivisionError')
it = pool.imap(f, range(10))
for i in range(10):
try:
x = it.next()
except ZeroDivisionError:
if i == 5:
pass
except StopIteration:
break
else:
if i == 5:
raise AssertionError('expected ZeroDivisionError')
assert i == 9
print('\tGot ZeroDivisionError as expected from IMapIterator.next()')
print()
#
# Testing timeouts
#
print('Testing ApplyResult.get() with timeout:', end='')
res = pool.apply_async(calculate, TASKS[0])
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % res.get(0.02))
break
except TimeoutError:
sys.stdout.write('.')
print()
print()
print('Testing IMapIterator.next() with timeout:', end='')
it = pool.imap(calculatestar, TASKS)
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % it.next(0.02))
except StopIteration:
break
except TimeoutError:
sys.stdout.write('.')
print()
print()
#
# Testing callback
#
print('Testing callback:')
A = []
B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729]
r = pool.apply_async(mul, (7, 8), callback=A.append)
r.wait()
r = pool.map_async(pow3, range(10), callback=A.extend)
r.wait()
if A == B:
print('\tcallbacks succeeded\n')
else:
print('\t*** callbacks failed\n\t\t%s != %s\n' % (A, B))
#
# Check there are no outstanding tasks
#
assert not pool._cache, 'cache = %r' % pool._cache
#
# Check close() methods
#
print('Testing close():')
for worker in pool._pool:
assert worker.is_alive()
result = pool.apply_async(time.sleep, [0.5])
pool.close()
pool.join()
assert result.get() is None
for worker in pool._pool:
assert not worker.is_alive()
print('\tclose() succeeded\n')
#
# Check terminate() method
#
print('Testing terminate():')
pool = Pool(2)
ignore = pool.apply(pow3, [2])
results = [pool.apply_async(time.sleep, [10]) for i in range(10)]
pool.terminate()
pool.join()
for worker in pool._pool:
assert not worker.is_alive()
print('\tterminate() succeeded\n')
#
# Check garbage collection
#
print('Testing garbage collection:')
pool = Pool(2)
processes = pool._pool
ignore = pool.apply(pow3, [2])
results = [pool.apply_async(time.sleep, [10]) for i in range(10)]
del results, pool
time.sleep(0.2)
for worker in processes:
assert not worker.is_alive()
print('\tgarbage collection succeeded\n')
