class ParallelExecutor(TaskExecutor):
"""
====================
ParallelExecutor
====================
:class:`ParallelExecutor` splits the data for multiprocessing
"""
def __init__(self, nbParal=-1, logger=None, verbosity=0, tempFolder=None):
TaskExecutor.__init__(self, logger, verbosity)
self._nbParal = nbParal
self._tmpFolder = tempFolder
self._numpyFactory = NumpyFactory(tempFolder)
def execute(self, desc, function, data, *args, **kwargs):
#Splitting task
tSplitter = TaskSplitter()
nbJobs, splittedData, starts = tSplitter.partition(self._nbParal, data)
#Logging
self.setTask(1, ("Starting parallelization : "+desc))
#Parallelization
parallelizer = Parallel(n_jobs=nbJobs, temp_folder=self._tmpFolder,
verbose=self.verbosity,)
if len(args) == 0:
if len(kwargs) == 0:
allData = parallelizer(delayed(function)(
splittedData[i]) for i in xrange(nbJobs))
else:
allData = parallelizer(delayed(function)(
splittedData[i], **kwargs) for i in xrange(nbJobs))
elif len(kwargs) == 0:
allData = parallelizer(delayed(function)(
splittedData[i], *args) for i in xrange(nbJobs))
else:
allData = parallelizer(delayed(function)(
splittedData[i], *args, **kwargs) for i in xrange(nbJobs))
self.endTask()
return allData
def executeWithStart(self, desc, function, data, *args, **kwargs):
#Splitting task
tSplitter = TaskSplitter()
nbJobs, splittedData, starts = tSplitter.partition(self._nbParal, data)
#Logging
self.setTask(1, ("Starting parallelization : "+desc))
#Parallelization
parallelizer = Parallel(n_jobs=nbJobs, temp_folder=self._tmpFolder,
verbose=self.verbosity,)
if len(args) == 0:
if len(kwargs) == 0:
allData = parallelizer(delayed(function)(
splittedData[i], startIndex=starts[i])
for i in xrange(nbJobs))
else:
allData = parallelizer(delayed(function)(
splittedData[i], startIndex=starts[i], **kwargs)
for i in xrange(nbJobs))
elif len(kwargs) == 0:
allData = parallelizer(delayed(function)(
splittedData[i], startIndex=starts[i], *args)
for i in xrange(nbJobs))
else:
allData = parallelizer(delayed(function)(
splittedData[i], startIndex=starts[i], *args, **kwargs)
for i in xrange(nbJobs))
self.endTask()
return allData
def createArray(self, shape, dtype=float):
return self._numpyFactory.createArray(shape, dtype)
def clean(self, array):
self._numpyFactory.clean(array)
class ParallelExecutor(TaskExecutor):
"""
====================
ParallelExecutor
====================
:class:`ParallelExecutor` splits the data for multiprocessing
"""
def __init__(self, nbParal=-1, logger=None, verbosity=0, tempFolder=None):
TaskExecutor.__init__(self, logger, verbosity)
self._nbParal = nbParal
self._tmpFolder = tempFolder
self._numpyFactory = NumpyFactory(tempFolder)
def execute(self, desc, function, data, *args, **kwargs):
#Splitting task
tSplitter = TaskSplitter()
nbJobs, splittedData, starts = tSplitter.partition(self._nbParal, data)
#Logging
self.setTask(1, ("Starting parallelization : " + desc))
#Parallelization
parallelizer = Parallel(
n_jobs=nbJobs,
temp_folder=self._tmpFolder,
verbose=self.verbosity,
)
if len(args) == 0:
if len(kwargs) == 0:
allData = parallelizer(
delayed(function)(splittedData[i]) for i in xrange(nbJobs))
else:
allData = parallelizer(
delayed(function)(splittedData[i], **kwargs)
for i in xrange(nbJobs))
elif len(kwargs) == 0:
allData = parallelizer(
delayed(function)(splittedData[i], *args)
for i in xrange(nbJobs))
else:
allData = parallelizer(
delayed(function)(splittedData[i], *args, **kwargs)
for i in xrange(nbJobs))
self.endTask()
return allData
def executeWithStart(self, desc, function, data, *args, **kwargs):
#Splitting task
tSplitter = TaskSplitter()
nbJobs, splittedData, starts = tSplitter.partition(self._nbParal, data)
#Logging
self.setTask(1, ("Starting parallelization : " + desc))
#Parallelization
parallelizer = Parallel(
n_jobs=nbJobs,
temp_folder=self._tmpFolder,
verbose=self.verbosity,
)
if len(args) == 0:
if len(kwargs) == 0:
allData = parallelizer(
delayed(function)(splittedData[i], startIndex=starts[i])
for i in xrange(nbJobs))
else:
allData = parallelizer(
delayed(function)(
splittedData[i], startIndex=starts[i], **kwargs)
for i in xrange(nbJobs))
elif len(kwargs) == 0:
allData = parallelizer(
delayed(function)(splittedData[i], startIndex=starts[i], *args)
for i in xrange(nbJobs))
else:
allData = parallelizer(
delayed(function)(
splittedData[i], startIndex=starts[i], *args, **kwargs)
for i in xrange(nbJobs))
self.endTask()
return allData
def createArray(self, shape, dtype=float):
return self._numpyFactory.createArray(shape, dtype)
def clean(self, array):
self._numpyFactory.clean(array)
