def load_dfk(config):
"""Load the dfk before running a test.
The special path `local` indicates that whatever configuration is loaded
locally in the test should not be replaced. Otherwise, it is expected that
the supplied file contains a dictionary called `config`, which will be
loaded before the test runs.
Args:
config (str) : path to config to load (this is a parameterized pytest fixture)
"""
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if 'globals' not in module.config:
module.config['globals'] = {}
module.config['globals']['runDir'] = get_rundir(
) # Give unique rundir; needed running with -n=X where X > 1.
parsl.clear()
dfk = parsl.load(module.config)
yield
dfk.cleanup()
else:
yield
def load_dfk_session(request, pytestconfig):
"""Load a dfk around entire test suite, except in local mode.
The special path `local` indicates that configuration will not come
from a pytest managed configuration file; in that case, see
load_dfk_local_module for module-level configuration management.
"""
config = pytestconfig.getoption('config')[0]
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.config.run_dir = get_rundir(
) # Give unique rundir; needed running with -n=X where X > 1.
if DataFlowKernelLoader._dfk is not None:
raise ValueError(
"DFK didn't start as None - there was a DFK from somewhere already"
)
dfk = parsl.load(module.config)
yield
if (parsl.dfk() != dfk):
raise ValueError("DFK changed unexpectedly during test")
dfk.cleanup()
parsl.clear()
else:
yield
def load_dfk(config):
"""Load the dfk before running a test.
The special path `local` indicates that whatever configuration is loaded
locally in the test should not be replaced. Otherwise, it is expected that
the supplied file contains a dictionary called `config`, which will be
loaded before the test runs.
Args:
config (str) : path to config to load (this is a parameterized pytest fixture)
"""
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
try:
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.config.run_dir = get_rundir() # Give unique rundir; needed running with -n=X where X > 1.
if DataFlowKernelLoader._dfk is not None:
raise ValueError("DFK didn't start as None - there was a DFK from somewhere already")
parsl.clear()
dfk = parsl.load(module.config)
yield
if(parsl.dfk() != dfk):
raise ValueError("DFK changed unexpectedly during test")
dfk.cleanup()
parsl.clear()
except KeyError:
pytest.skip('options in user_opts.py not configured for {}'.format(config))
else:
yield
def load_dfk(config):
"""Load the dfk before running a test.
The special path `local` indicates that whatever configuration is loaded
locally in the test should not be replaced. Otherwise, it is expected that
the supplied file contains a dictionary called `config`, which will be
loaded before the test runs.
Args:
config (str) : path to config to load (this is a parameterized pytest fixture)
"""
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
try:
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.config.run_dir = get_rundir(
) # Give unique rundir; needed running with -n=X where X > 1.
parsl.clear()
dfk = parsl.load(module.config)
yield
dfk.cleanup()
except KeyError:
pytest.skip(
'options in user_opts.py not configured for {}'.format(config))
else:
yield
"site": "local_threads",
"auth": {
"channel": None,
},
"execution": {
"executor": "threads",
"provider": None,
"maxThreads": 4
}
}, {
"site": "local_ipp",
"auth": {
"channel": None,
},
"execution": {
"executor": "ipp",
"provider": "local",
"block": {
"nodes": 1,
"taskBlocks": 1,
"walltime": "00:05:00",
"initBlocks": 4,
}
}
}],
"globals": {
"lazyErrors": True,
"runDir": get_rundir()
}
}
from libsubmit.providers.aws.aws.EC2Provider import EC2Provider
from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor
from parsl.tests.user_opts import user_opts
from parsl.tests.utils import get_rundir
config = Config(
executors=[
IPyParallelExecutor(
label='ec2_bad_spot',
provider=EC2Provider(
user_opts['ec2']['image_id'],
region=user_opts['ec2']['region'],
key_name=user_opts['ec2']['key_name'],
state_file='awsproviderstate.json',
spot_max_bid='0.001', # price too low
nodes_per_block=1,
tasks_per_node=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='00:25:00',
))
],
run_dir=get_rundir(),
)
if 'cc_in2p3' in user_opts:
info = user_opts['cc_in2p3']
else:
pytest.skip('cc_in2p3 user_opts not configured', allow_module_level=True)
config = {
"sites": [{
"site": "cc_in2p3_local_single_node",
"auth": {
"channel": "local",
"username": info['username'],
"scriptDir": info['script_dir']
},
"execution": {
"executor": "ipp",
"provider": "gridEngine",
"block": {
"nodes": 1,
"taskBlocks": 1,
"initBlocks": 1,
"maxBlocks": 1,
"options": info['options']
}
}
}],
"globals": {
"lazyErrors": True,
'runDir': get_rundir()
}
}
def fresh_config():
return Config(
executors=[ThreadPoolExecutor()],
run_dir=get_rundir(),
)
def fresh_config():
return Config(
executors=[ThreadPoolExecutor(label='local_threads_checkpoint', )],
run_dir=get_rundir(),
)
