def get_app(self, function: str) -> Callable: """Obtains a Parsl python_application. Parameters ---------- function : str A full path to a function Returns ------- callable The desired AppFactory Examples -------- >>> get_app("numpy.einsum") <class PythonApp"AppFactory for einsum> """ from parsl.app.app import python_app if function in self.app_map: return self.app_map[function] func = self.get_function(function) # TODO set walltime and the like self.app_map[function] = python_app(func, data_flow_kernel=self.client) return self.app_map[function]
def get_function(self, function): """Obtains a Python function wrapped in a Parsl Python App Parameters ---------- function : str A full path to a function Returns ------- callable The desired AppFactory Examples -------- >>> get_function("numpy.einsum") <class PythonApp"AppFactory for einsum> """ from parsl.app.app import python_app if function in self.function_map: return self.function_map[function] module_name, func_name = function.split(".", 1) module = importlib.import_module(module_name) func = operator.attrgetter(func_name)(module) # TODO set walltime and the like self.function_map[function] = python_app( func, data_flow_kernel=self.dataflow) return self.function_map[function]
def run(workflow, config, decorate, directory, capillary, gsasloc, pdf, gpx): """ GSAS Parsl CLI runner """ import importlib import gsas.configs config = importlib.import_module(config) funcname = ''.join(workflow.rsplit('.')[-1]) modulename = '.'.join(workflow.rsplit('.')[:-1]) print(workflow, funcname, modulename) globals()['executor'] = config.executor gsas.configs.executor = config.executor module = importlib.import_module(modulename) func = getattr(module, funcname) _dfunc = func if decorate == 'python_app': from parsl.app.app import python_app _dfunc = python_app(func, executors=[config.executor]) if decorate == 'container_app': from parsl.app.app import container_app _dfunc = container_app(func, executors=[config.executor]) if decorate == 'bash_app': from parsl.app.app import bash_app _dfunc = bash_app(func, executors=[config.executor]) setattr(module, funcname,_dfunc) run = getattr(module, 'run') _pdf = {} if pdf is not None: with open(pdf, 'r') as configfile: _pdf = json.loads(configfile.read()) run(directory=directory, gpx=gpx, config=config, pdf=_pdf, capillary=capillary, gsasloc=gsasloc)
def _globus_stage_out_app(self): return python_app(executors=['data_manager'])(self._globus_stage_out)
def _http_stage_in_app(self, executor): return python_app(executors=[executor])(_http_stage_in)
def parsl_executor(items, function, accumulator, **kwargs): """Execute using parsl pyapp wrapper Parameters ---------- items : list List of input arguments function : callable A function to be called on each input, which returns an accumulator instance accumulator : AccumulatorABC An accumulator to collect the output of the function config : parsl.config.Config, optional A parsl DataFlow configuration object. Necessary if there is no active kernel .. note:: In general, it is safer to construct the DFK with ``parsl.load(config)`` prior to calling this function status : bool If true (default), enable progress bar unit : str Label of progress bar unit desc : str Label of progress bar description compression : int, optional Compress accumulator outputs in flight with LZ4, at level specified (default 1) Set to ``None`` for no compression. """ if len(items) == 0: return accumulator import parsl from parsl.app.app import python_app from .parsl.timeout import timeout status = kwargs.pop('status', True) unit = kwargs.pop('unit', 'items') desc = kwargs.pop('desc', 'Processing') clevel = kwargs.pop('compression', 1) if clevel is not None: function = _compression_wrapper(clevel, function) add_fn = _iadd cleanup = False config = kwargs.pop('config', None) try: parsl.dfk() except RuntimeError: cleanup = True pass if cleanup and config is None: raise RuntimeError("No active parsl DataFlowKernel, must specify a config to construct one") elif not cleanup and config is not None: raise RuntimeError("An active parsl DataFlowKernel already exists") elif config is not None: parsl.clear() parsl.load(config) app = timeout(python_app(function)) futures = set(app(item) for item in items) _futures_handler(futures, accumulator, status, unit, desc, add_fn) if cleanup: parsl.dfk().cleanup() parsl.clear() return accumulator
def _globus_stage_out_app(self): return python_app(executors=['data_manager'], data_flow_kernel=self.dfk)(self._globus_stage_out)
def _http_stage_in_app(self, executor): return python_app(executors=[executor], data_flow_kernel=self.dfk)(_http_stage_in)
import numpy as np import time from parsl.app.app import python_app from .timeout import timeout from ..executor import _futures_handler lz4_clevel = 1 def coffea_pyapp_func(dataset, fn, treename, chunksize, index, procstr, timeout=None, flatten=True, **kwargs): raise RuntimeError('parsl_executor.coffea pyapp cannot be used any more,' 'please use a wrapped _work_function from processor.executor') coffea_pyapp = timeout(python_app(coffea_pyapp_func)) class ParslExecutor(object): def __init__(self): self._counts = {} @property def counts(self): return self._counts def __call__(self, items, processor_instance, output, status=True, unit='items', desc='Processing', timeout=None, flatten=True, **kwargs): raise RuntimeError('ParslExecutor.__call__ cannot be used any more,' 'please use processor.parsl_executor')
def _globus_stage_out_app(self, executor, dfk): executor_obj = dfk.executors[executor] f = partial(_globus_stage_out, self, executor_obj) return python_app(executors=['data_manager'], data_flow_kernel=dfk)(f)
def _file_stage_in_app(self): return python_app(executors=['data_manager'])(self._file_stage_in)
def _globus_stage_in_app(self, executor, dfk): executor_obj = dfk.executors[executor] f = partial(_globus_stage_in, self, executor_obj) return python_app(executors=['_parsl_internal'], data_flow_kernel=dfk)(f)