def load_dfk(config): """Load the dfk before running a test. The special path `local` indicates that whatever configuration is loaded locally in the test should not be replaced. Otherwise, it is expected that the supplied file contains a dictionary called `config`, which will be loaded before the test runs. Args: config (str) : path to config to load (this is a parameterized pytest fixture) """ if config != 'local': spec = importlib.util.spec_from_file_location('', config) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) if 'globals' not in module.config: module.config['globals'] = {} module.config['globals']['runDir'] = get_rundir( ) # Give unique rundir; needed running with -n=X where X > 1. parsl.clear() dfk = parsl.load(module.config) yield dfk.cleanup() else: yield
def load_dfk_session(request, pytestconfig): """Load a dfk around entire test suite, except in local mode. The special path `local` indicates that configuration will not come from a pytest managed configuration file; in that case, see load_dfk_local_module for module-level configuration management. """ config = pytestconfig.getoption('config')[0] if config != 'local': spec = importlib.util.spec_from_file_location('', config) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.config.run_dir = get_rundir( ) # Give unique rundir; needed running with -n=X where X > 1. if DataFlowKernelLoader._dfk is not None: raise ValueError( "DFK didn't start as None - there was a DFK from somewhere already" ) dfk = parsl.load(module.config) yield if (parsl.dfk() != dfk): raise ValueError("DFK changed unexpectedly during test") dfk.cleanup() parsl.clear() else: yield
def load_dfk(config): """Load the dfk before running a test. The special path `local` indicates that whatever configuration is loaded locally in the test should not be replaced. Otherwise, it is expected that the supplied file contains a dictionary called `config`, which will be loaded before the test runs. Args: config (str) : path to config to load (this is a parameterized pytest fixture) """ if config != 'local': spec = importlib.util.spec_from_file_location('', config) try: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.config.run_dir = get_rundir() # Give unique rundir; needed running with -n=X where X > 1. if DataFlowKernelLoader._dfk is not None: raise ValueError("DFK didn't start as None - there was a DFK from somewhere already") parsl.clear() dfk = parsl.load(module.config) yield if(parsl.dfk() != dfk): raise ValueError("DFK changed unexpectedly during test") dfk.cleanup() parsl.clear() except KeyError: pytest.skip('options in user_opts.py not configured for {}'.format(config)) else: yield
def load_dfk(config): """Load the dfk before running a test. The special path `local` indicates that whatever configuration is loaded locally in the test should not be replaced. Otherwise, it is expected that the supplied file contains a dictionary called `config`, which will be loaded before the test runs. Args: config (str) : path to config to load (this is a parameterized pytest fixture) """ if config != 'local': spec = importlib.util.spec_from_file_location('', config) try: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.config.run_dir = get_rundir( ) # Give unique rundir; needed running with -n=X where X > 1. parsl.clear() dfk = parsl.load(module.config) yield dfk.cleanup() except KeyError: pytest.skip( 'options in user_opts.py not configured for {}'.format(config)) else: yield
"site": "local_threads", "auth": { "channel": None, }, "execution": { "executor": "threads", "provider": None, "maxThreads": 4 } }, { "site": "local_ipp", "auth": { "channel": None, }, "execution": { "executor": "ipp", "provider": "local", "block": { "nodes": 1, "taskBlocks": 1, "walltime": "00:05:00", "initBlocks": 4, } } }], "globals": { "lazyErrors": True, "runDir": get_rundir() } }
from libsubmit.providers.aws.aws.EC2Provider import EC2Provider from parsl.config import Config from parsl.executors.ipp import IPyParallelExecutor from parsl.tests.user_opts import user_opts from parsl.tests.utils import get_rundir config = Config( executors=[ IPyParallelExecutor( label='ec2_bad_spot', provider=EC2Provider( user_opts['ec2']['image_id'], region=user_opts['ec2']['region'], key_name=user_opts['ec2']['key_name'], state_file='awsproviderstate.json', spot_max_bid='0.001', # price too low nodes_per_block=1, tasks_per_node=1, init_blocks=1, max_blocks=1, min_blocks=0, walltime='00:25:00', )) ], run_dir=get_rundir(), )
if 'cc_in2p3' in user_opts: info = user_opts['cc_in2p3'] else: pytest.skip('cc_in2p3 user_opts not configured', allow_module_level=True) config = { "sites": [{ "site": "cc_in2p3_local_single_node", "auth": { "channel": "local", "username": info['username'], "scriptDir": info['script_dir'] }, "execution": { "executor": "ipp", "provider": "gridEngine", "block": { "nodes": 1, "taskBlocks": 1, "initBlocks": 1, "maxBlocks": 1, "options": info['options'] } } }], "globals": { "lazyErrors": True, 'runDir': get_rundir() } }
def fresh_config(): return Config( executors=[ThreadPoolExecutor()], run_dir=get_rundir(), )
def fresh_config(): return Config( executors=[ThreadPoolExecutor(label='local_threads_checkpoint', )], run_dir=get_rundir(), )