def process ( self , task , args , **kwargs ) :
"""Process callable object or Task :
- process Task
>>> class MyTask(Task) : ....
>>> my_task = MyTask ( ... )
>>> wm = WorkManager ( ... )
>>> result = wm.process ( my_task , items )
- process callable object
>>> def my_fun ( jobid , x ) : return x**2
>>> wm = WorkManager ( ... )
>>> items = range ( 10 )
>>> result1 = wm.process ( my_fun , items , merger = TaskMerger ( lambda a,b : a+[b] , init = [] ) )
>>> result2 = wm.process ( my_fun , items , merger = TaskMerger () )
"""
job_chunk = kwargs.pop ( 'chunk_size', 10000 )
from ostap.utils.utils import chunked
chunks = list ( chunked ( args , job_chunk ) )
if isinstance ( task , Task ) :
result = self.__process_task ( task , chunks , **kwargs )
else :
result = self.__process_func ( task , chunks , **kwargs )
return result
def copy_files(file_pairs, progress=True, maxfiles=5, copier=None, **kwargs):
"""Copy files in parallel
"""
if not copier:
from ostap.utils.utils import copy_file
copier = copy_file
task = CopyTask(copier=copier)
wmgr = WorkManager(silent=not progress, **kwargs)
if maxfiles < 1: maxfiles = 1
from ostap.utils.utils import chunked
data = chunked(file_pairs, maxfiles)
wmgr.process(task, data)
copied = task.results()
return copied
def merge_args ( num , *args ) :
"""merge arguments into smaller chunks
args = ...
margs = merge_args ( 3 , *args )
"""
assert isinstance ( num , integer_types ) and 1 <= num ,\
"merge_args: invalid chunk size ``%s''" % num
if len ( args ) < num : return tuple ( args )
lst = flat_args ( *args )
keep = [ l for l in lst ]
while num < len ( lst ) :
nlst = chunked ( lst , 4 )
ll = [ ROOT.RooFit.MultiArg ( *l ) if 1 < len ( l ) else l [ 0 ] for l in nlst ]
for l in ll : keep.append ( l )
lst = tuple ( ll )
return lst