def ez_map(func, *arglist, **kwds): """higher-level map interface for selected mapper and launcher maps function 'func' across arguments 'arglist'. arguments and results are stored and sent as pickled strings, while function 'func' is inspected and written as a source file to be imported. Further Input: nodes -- the number of parallel nodes launcher -- the launcher object scheduler -- the scheduler object mapper -- the mapper object timelimit -- string representation of maximum run time (e.g. '00:02') queue -- string name of selected queue (e.g. 'normal') """ import dill as pickle import os.path, tempfile, subprocess from pyina.tools import which_strategy # mapper = None (allow for use of default mapper) if kwds.has_key('mapper'): mapper = kwds['mapper'] if mapper() == "mpi_pool": scatter = False elif mapper() == "mpi_scatter": scatter = True else: raise NotImplementedError, "Mapper '%s' not found." % mapper() ezdefaults['program'] = which_strategy(scatter, lazy=True) # override the defaults if kwds.has_key('nnodes'): ezdefaults['nodes'] = kwds['nnodes'] if kwds.has_key('nodes'): ezdefaults['nodes'] = kwds['nodes'] if kwds.has_key('timelimit'): ezdefaults['timelimit'] = kwds['timelimit'] if kwds.has_key('queue'): ezdefaults['queue'] = kwds['queue'] # set the scheduler & launcher (or use the given default) if kwds.has_key('launcher'): launcher = kwds['launcher'] else: launcher = mpirun_launcher #XXX: default = non_mpi? if kwds.has_key('scheduler'): scheduler = kwds['scheduler'] else: scheduler = '' # set scratch directory (most often required for queue launcher) if kwds.has_key('workdir'): ezdefaults['workdir'] = kwds['workdir'] else: if launcher in [torque_launcher, moab_launcher] \ or scheduler in [torque_scheduler, moab_scheduler]: ezdefaults['workdir'] = os.path.expanduser("~") from dill.temp import dump, dump_source # write func source to a NamedTemporaryFile (instead of pickle.dump) # ezrun requires 'FUNC = <function>' to be included as module.FUNC modfile = dump_source(func, alias='FUNC', dir=ezdefaults['workdir']) # standard pickle.dump of inputs to a NamedTemporaryFile kwd = {'onall':kwds.get('onall',True)} argfile = dump((arglist,kwd), suffix='.arg', dir=ezdefaults['workdir']) # Keep the above return values for as long as you want the tempfile to exist resfilename = tempfile.mktemp(dir=ezdefaults['workdir']) modname = os.path.splitext(os.path.basename(modfile.name))[0] ezdefaults['progargs'] = ' '.join([modname, argfile.name, resfilename, \ ezdefaults['workdir']]) #HOLD.append(modfile) #HOLD.append(argfile) if launcher in [torque_launcher, moab_launcher] \ or scheduler in [torque_scheduler, moab_scheduler]: jobfilename = tempfile.mktemp(dir=ezdefaults['workdir']) outfilename = tempfile.mktemp(dir=ezdefaults['workdir']) errfilename = tempfile.mktemp(dir=ezdefaults['workdir']) ezdefaults['jobfile'] = jobfilename ezdefaults['outfile'] = outfilename ezdefaults['errfile'] = errfilename # get the appropriate launcher for the scheduler if scheduler in [torque_scheduler] and launcher in [mpirun_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().mpirun elif scheduler in [moab_scheduler] and launcher in [mpirun_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().mpirun elif scheduler in [torque_scheduler] and launcher in [srun_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().srun elif scheduler in [moab_scheduler] and launcher in [srun_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().srun elif scheduler in [torque_scheduler] and launcher in [aprun_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().aprun elif scheduler in [moab_scheduler] and launcher in [aprun_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().aprun elif scheduler in [torque_scheduler] and launcher in [serial_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().serial elif scheduler in [moab_scheduler] and launcher in [serial_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().serial #else: scheduler = None # counting on the function below to block until done. #print 'executing: ', launcher(ezdefaults) launch(launcher(ezdefaults)) #FIXME: use subprocessing if launcher in [torque_launcher, moab_launcher] \ or scheduler in [torque_scheduler, moab_scheduler]: import time #BLOCKING while (not os.path.exists(resfilename)): #XXX: or out* to confirm start time.sleep(sleeptime) #XXX: wait for results... may infinite loop? subprocess.call('rm -f %s' % jobfilename, shell=True) subprocess.call('rm -f %s' % outfilename, shell=True) subprocess.call('rm -f %s' % errfilename, shell=True) # debuggery... output = function(inputs) #subprocess.call('cp -f %s modfile.py' % modfile.name, shell=True) # getsource; FUNC=func #subprocess.call('cp -f %s argfile.py' % argfile.name, shell=True) # pickled list of inputs #subprocess.call('cp -f %s resfile.py' % resfilename, shell=True) # pickled list of output # read result back res = pickle.load(open(resfilename,'r')) subprocess.call('rm -f %s' % resfilename, shell=True) subprocess.call('rm -f %sc' % modfile.name, shell=True) return res
def ez_map(func, *arglist, **kwds): """higher-level map interface for selected mapper and launcher maps function 'func' across arguments 'arglist'. arguments and results are stored and sent as pickled strings, while function 'func' is inspected and written as a source file to be imported. Further Input: nodes -- the number of parallel nodes launcher -- the launcher object scheduler -- the scheduler object mapper -- the mapper object timelimit -- string representation of maximum run time (e.g. '00:02') queue -- string name of selected queue (e.g. 'normal') """ import dill as pickle import os.path, tempfile, subprocess from pyina.tools import which_strategy # mapper = None (allow for use of default mapper) if kwds.has_key('mapper'): mapper = kwds['mapper'] if mapper() == "mpi_pool": scatter = False elif mapper() == "mpi_scatter": scatter = True else: raise NotImplementedError, "Mapper '%s' not found." % mapper() ezdefaults['program'] = which_strategy(scatter, lazy=True) # override the defaults if kwds.has_key('nnodes'): ezdefaults['nodes'] = kwds['nnodes'] if kwds.has_key('nodes'): ezdefaults['nodes'] = kwds['nodes'] if kwds.has_key('timelimit'): ezdefaults['timelimit'] = kwds['timelimit'] if kwds.has_key('queue'): ezdefaults['queue'] = kwds['queue'] # set the scheduler & launcher (or use the given default) if kwds.has_key('launcher'): launcher = kwds['launcher'] else: launcher = mpirun_launcher #XXX: default = non_mpi? if kwds.has_key('scheduler'): scheduler = kwds['scheduler'] else: scheduler = '' # set scratch directory (most often required for queue launcher) if kwds.has_key('workdir'): ezdefaults['workdir'] = kwds['workdir'] else: if launcher in [torque_launcher, moab_launcher] \ or scheduler in [torque_scheduler, moab_scheduler]: ezdefaults['workdir'] = os.path.expanduser("~") from dill.temp import dump, dump_source # write func source to a NamedTemporaryFile (instead of pickle.dump) # ezrun requires 'FUNC = <function>' to be included as module.FUNC modfile = dump_source(func, alias='FUNC', dir=ezdefaults['workdir']) # standard pickle.dump of inputs to a NamedTemporaryFile kwd = {'onall': kwds.get('onall', True)} argfile = dump((arglist, kwd), suffix='.arg', dir=ezdefaults['workdir']) # Keep the above return values for as long as you want the tempfile to exist resfilename = tempfile.mktemp(dir=ezdefaults['workdir']) modname = os.path.splitext(os.path.basename(modfile.name))[0] ezdefaults['progargs'] = ' '.join([modname, argfile.name, resfilename, \ ezdefaults['workdir']]) #HOLD.append(modfile) #HOLD.append(argfile) if launcher in [torque_launcher, moab_launcher] \ or scheduler in [torque_scheduler, moab_scheduler]: jobfilename = tempfile.mktemp(dir=ezdefaults['workdir']) outfilename = tempfile.mktemp(dir=ezdefaults['workdir']) errfilename = tempfile.mktemp(dir=ezdefaults['workdir']) ezdefaults['jobfile'] = jobfilename ezdefaults['outfile'] = outfilename ezdefaults['errfile'] = errfilename # get the appropriate launcher for the scheduler if scheduler in [torque_scheduler] and launcher in [mpirun_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().mpirun elif scheduler in [moab_scheduler] and launcher in [mpirun_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().mpirun elif scheduler in [torque_scheduler] and launcher in [srun_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().srun elif scheduler in [moab_scheduler] and launcher in [srun_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().srun elif scheduler in [torque_scheduler] and launcher in [aprun_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().aprun elif scheduler in [moab_scheduler] and launcher in [aprun_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().aprun elif scheduler in [torque_scheduler] and launcher in [serial_launcher]: launcher = torque_launcher ezdefaults['scheduler'] = scheduler().serial elif scheduler in [moab_scheduler] and launcher in [serial_launcher]: launcher = moab_launcher ezdefaults['scheduler'] = scheduler().serial #else: scheduler = None # counting on the function below to block until done. #print 'executing: ', launcher(ezdefaults) launch(launcher(ezdefaults)) #FIXME: use subprocessing if launcher in [torque_launcher, moab_launcher] \ or scheduler in [torque_scheduler, moab_scheduler]: import time #BLOCKING while (not os.path.exists(resfilename) ): #XXX: or out* to confirm start time.sleep(sleeptime) #XXX: wait for results... may infinite loop? subprocess.call('rm -f %s' % jobfilename, shell=True) subprocess.call('rm -f %s' % outfilename, shell=True) subprocess.call('rm -f %s' % errfilename, shell=True) # debuggery... output = function(inputs) #subprocess.call('cp -f %s modfile.py' % modfile.name, shell=True) # getsource; FUNC=func #subprocess.call('cp -f %s argfile.py' % argfile.name, shell=True) # pickled list of inputs #subprocess.call('cp -f %s resfile.py' % resfilename, shell=True) # pickled list of output # read result back res = pickle.load(open(resfilename, 'r')) subprocess.call('rm -f %s' % resfilename, shell=True) subprocess.call('rm -f %sc' % modfile.name, shell=True) return res