def test_initial_checkpoint_write(n=2):
"""1. Launch a few apps and write the checkpoint once a few have completed
"""
config = fresh_config()
config.checkpoint_mode = 'manual'
parsl.load(config)
results = launch_n_random(n)
cpt_dir = parsl.dfk().checkpoint()
cptpath = cpt_dir + '/dfk.pkl'
print("Path exists : ", os.path.exists(cptpath))
assert os.path.exists(cptpath), "DFK checkpoint missing: {0}".format(
cptpath)
cptpath = cpt_dir + '/tasks.pkl'
print("Path exists : ", os.path.exists(cptpath))
assert os.path.exists(cptpath), "Tasks checkpoint missing: {0}".format(
cptpath)
run_dir = parsl.dfk().run_dir
parsl.dfk().cleanup()
parsl.clear()
return run_dir, results
def load_dfk(config):
"""Load the dfk before running a test.
The special path `local` indicates that whatever configuration is loaded
locally in the test should not be replaced. Otherwise, it is expected that
the supplied file contains a dictionary called `config`, which will be
loaded before the test runs.
Args:
config (str) : path to config to load (this is a parameterized pytest fixture)
"""
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
try:
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.config.run_dir = get_rundir(
) # Give unique rundir; needed running with -n=X where X > 1.
parsl.clear()
dfk = parsl.load(module.config)
yield
dfk.cleanup()
except KeyError:
pytest.skip(
'options in user_opts.py not configured for {}'.format(config))
else:
yield
def test_one_block():
oneshot_provider = OneShotLocalProvider(
channel=LocalChannel(),
init_blocks=0,
min_blocks=0,
max_blocks=10,
launcher=SimpleLauncher(),
)
config = Config(
executors=[
HighThroughputExecutor(
label="htex_local",
worker_debug=True,
cores_per_worker=1,
provider=oneshot_provider,
)
],
strategy='simple',
)
parsl.load(config)
f = app()
f.result()
parsl.clear()
assert oneshot_provider.recorded_submits == 1
def test_regression_stage_out_does_not_stage_in():
no_stageout_config = Config(
executors=[
ThreadPoolExecutor(
label='local_threads',
storage_access=[NoOpTestingFileStaging(allow_stage_in=False)]
)
]
)
parsl.load(no_stageout_config)
# Test that the helper app runs with no staging
touch("test.1", outputs=[]).result()
# Test with stage-out, checking that provider stage in is never
# invoked. If stage-in is invoked, the the NoOpTestingFileStaging
# provider will raise an exception, which should propagate to
# .result() here.
touch("test.2", outputs=[File("test.2")]).result()
# Test that stage-in exceptions propagate out to user code.
with pytest.raises(NoOpError):
touch("test.3", inputs=[File("test.3")]).result()
parsl.dfk().cleanup()
parsl.clear()
def load_dfk(config):
"""Load the dfk before running a test.
The special path `local` indicates that whatever configuration is loaded
locally in the test should not be replaced. Otherwise, it is expected that
the supplied file contains a dictionary called `config`, which will be
loaded before the test runs.
Args:
config (str) : path to config to load (this is a parameterized pytest fixture)
"""
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
try:
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.config.run_dir = get_rundir() # Give unique rundir; needed running with -n=X where X > 1.
if DataFlowKernelLoader._dfk is not None:
raise ValueError("DFK didn't start as None - there was a DFK from somewhere already")
parsl.clear()
dfk = parsl.load(module.config)
yield
if(parsl.dfk() != dfk):
raise ValueError("DFK changed unexpectedly during test")
dfk.cleanup()
parsl.clear()
except KeyError:
pytest.skip('options in user_opts.py not configured for {}'.format(config))
else:
yield
def load_dfk_session(request, pytestconfig):
"""Load a dfk around entire test suite, except in local mode.
The special path `local` indicates that configuration will not come
from a pytest managed configuration file; in that case, see
load_dfk_local_module for module-level configuration management.
"""
config = pytestconfig.getoption('config')[0]
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if DataFlowKernelLoader._dfk is not None:
raise ValueError(
"DFK didn't start as None - there was a DFK from somewhere already"
)
dfk = parsl.load(module.config)
yield
if (parsl.dfk() != dfk):
raise ValueError("DFK changed unexpectedly during test")
dfk.cleanup()
parsl.clear()
else:
yield
def test_platform(n=2, sleep_dur=10):
""" This should sleep to make sure that concurrent apps will go to different workers
on different nodes.
"""
config = fresh_config()
if config.executors[0].label == "htex_local":
return
parsl.load(fresh_config())
dfk = parsl.dfk()
name = list(dfk.executors.keys())[0]
print("Trying to get executor : ", name)
x = [platform(sleep=1) for i in range(2)]
print([i.result() for i in x])
print("Executor : ", dfk.executors[name])
print("Connected : ", dfk.executors[name].connected_workers)
print("Outstanding : ", dfk.executors[name].outstanding)
d = []
for i in range(0, n):
x = platform(sleep=sleep_dur)
d.append(x)
pinfo = set([i.result() for i in d])
assert len(pinfo) == 2, "Expected two nodes, instead got {}".format(pinfo)
print("Test passed")
dfk.cleanup()
parsl.clear()
return True
def test_non_lazy_behavior():
"""Testing non lazy errors to work"""
parsl.clear()
config.lazy_errors = False
parsl.load(config)
@App('python')
def divide(a, b):
return a / b
try:
items = []
for i in range(0, 1):
items.append(divide(10, i))
while True:
if items[0].done:
break
except Exception as e:
assert isinstance(
e,
ZeroDivisionError), "Expected ZeroDivisionError, got: {}".format(e)
else:
raise Exception("Expected ZeroDivisionError, got nothing")
return
def test_dynamic_executor():
dfk = parsl.load()
tasks = [sleeper() for i in range(5)]
results = [i.result() for i in tasks]
print("Done with initial test. The results are", results)
# Here we add a new executor to an active DFK
thread_executors = [ThreadPoolExecutor(label='threads2', max_threads=4)]
dfk.add_executors(executors=thread_executors)
tasks = [cpu_stress() for i in range(8)]
results = [i.result() for i in tasks]
print(
"Successfully added thread executor and ran with it. The results are",
results)
# We add a htex executor to an active DFK
executors = [
HighThroughputExecutor(
label='htex_local',
cores_per_worker=1,
max_workers=5,
provider=LocalProvider(
init_blocks=1,
max_blocks=1,
),
)
]
dfk.add_executors(executors=executors)
tasks = [add() for i in range(10)]
results = [i.result() for i in tasks]
print("Successfully added htex executor and ran with it. The results are",
results)
print("Done testing")
parsl.clear()
def load_dfk(config):
"""Load the dfk before running a test.
The special path `local` indicates that whatever configuration is loaded
locally in the test should not be replaced. Otherwise, it is expected that
the supplied file contains a dictionary called `config`, which will be
loaded before the test runs.
Args:
config (str) : path to config to load (this is a parameterized pytest fixture)
"""
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if 'globals' not in module.config:
module.config['globals'] = {}
module.config['globals']['runDir'] = get_rundir(
) # Give unique rundir; needed running with -n=X where X > 1.
parsl.clear()
dfk = parsl.load(module.config)
yield
dfk.cleanup()
else:
yield
def run1(*config_changes,
dry_run=False,
expect_fail=False,
expect_outputs=True):
try:
with tempfile.TemporaryDirectory() as dirname:
out_dir = os.path.join(dirname, "output")
log_dir = os.path.join(dirname, "logs")
config = [f"output_dir={out_dir}", f"log_dir={log_dir}"]
config += config_changes
pipe_config = Pipeline.build_config("tests/test.yml", config,
dry_run)
status = run(pipe_config, "tests/test.yml", config, dry_run)
if expect_fail:
assert status != 0
else:
assert status == 0
if expect_outputs:
assert os.path.exists(
os.path.join(out_dir, "wlgc_summary_data.txt"))
assert os.path.exists(
os.path.join(log_dir, "WLGCSummaryStatistic.out"))
finally:
clear()
def _return_value_test_(resume):
expected_status = 0 if resume else 1
# Mini pipeline should not run
launcher_config = {"interval": 0.5, "name": "mini"}
run_config = {
"log_dir": "./tests/logs",
"output_dir": "./tests/inputs",
"resume": resume,
}
pipeline = MiniPipeline([{"name": "FailingStage"}], launcher_config)
pipeline.initialize({}, run_config, "tests/config.yml")
status = pipeline.run()
assert status == expected_status
# Parsl pipeline should not run stage either
launcher_config = {"name": "parsl"}
site_config = {"name": "local", "max_threads": 1}
load(launcher_config, [site_config])
# the above sets the new default to be the parsl-configured site
pipeline = ParslPipeline([{"name": "FailingStage"}], launcher_config)
pipeline.initialize({}, run_config, "tests/config.yml")
status = pipeline.run()
assert status == expected_status
clear() # clear parsl settings
reset_default_site() # reset so default is minirunner again
def test_regression_stage_in_does_not_stage_out():
no_stageout_config = Config(
executors=[
ThreadPoolExecutor(
label='local_threads',
storage_access=[NoOpTestingFileStaging(allow_stage_out=False)]
)
],
)
parsl.load(no_stageout_config)
f = open("test.4", "a")
f.write("test")
f.close()
# Test that stage in does not invoke stage out. If stage out is
# attempted, then the NoOpTestingFileStaging provider will raise
# an exception which should propagate here.
app_test_in(File("test.4")).result()
# Test that stage out exceptions propagate to user code.
with pytest.raises(NoOpError):
touch("test.5", outputs=[File("test.5")]).result()
parsl.dfk().cleanup()
parsl.clear()
def test_loading_checkpoint(n=2):
"""Load memoization table from previous checkpoint
"""
parsl.load(config)
rundir = test1.test_initial_checkpoint_write()
parsl.clear()
local_config = fresh_config()
local_config.checkpoint_files = [os.path.join(rundir, 'checkpoint')]
parsl.load(local_config)
d = {}
start = time.time()
print("Launching : ", n)
for i in range(0, n):
d[i] = slow_double(i)
print("Done launching")
for i in range(0, n):
d[i].result()
print("Done sleeping")
delta = time.time() - start
assert delta < 1, "Took longer than a second ({}), assuming restore from checkpoint failed".format(
delta)
parsl.clear()
def test_row_counts():
# this is imported here rather than at module level because
# it isn't available in a plain parsl install, so this module
# would otherwise fail to import and break even a basic test
# run.
import sqlalchemy
from parsl.tests.configs.htex_local_alternate import fresh_config
if os.path.exists("monitoring.db"):
logger.info("Monitoring database already exists - deleting")
os.remove("monitoring.db")
logger.info("loading parsl")
parsl.load(fresh_config())
logger.info("invoking and waiting for result")
assert this_app().result() == 5
logger.info("cleaning up parsl")
parsl.dfk().cleanup()
parsl.clear()
# at this point, we should find one row in the monitoring database.
logger.info("checking database content")
engine = sqlalchemy.create_engine("sqlite:///monitoring.db")
with engine.begin() as connection:
result = connection.execute("SELECT COUNT(*) FROM workflow")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM task")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM try")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM status, try "
"WHERE status.task_id = try.task_id "
"AND status.task_status_name='exec_done' "
"AND task_try_time_running is NULL")
(c, ) = result.first()
assert c == 0
# Two entries: one showing manager active, one inactive
result = connection.execute("SELECT COUNT(*) FROM node")
(c, ) = result.first()
assert c == 2
# There should be one block polling status
# local provider has a status_polling_interval of 5s
result = connection.execute("SELECT COUNT(*) FROM block")
(c, ) = result.first()
assert c >= 2
logger.info("all done")
def test_summary(caplog):
parsl.load(fresh_config())
succeed().result()
fail().exception()
parsl.dfk().cleanup()
parsl.clear()
assert "Summary of tasks in DFK:" in caplog.text
assert "Tasks in state States.exec_done: 1" in caplog.text
assert "Tasks in state States.failed: 1" in caplog.text
def test_provider():
""" Provider scaling
"""
logger.info("Starting test_provider")
config = fresh_config()
name = config.executors[0].label
parsl.load(config)
dfk = parsl.dfk()
logger.info("Trying to get executor : {}".format(name))
x = platform(sleep=0)
logger.info("Result is {}".format(x.result()))
executor = dfk.executors[name]
provider = dfk.executors[name].provider
# At this point we should have 1 job
_, current_jobs = executor._get_block_and_job_ids()
assert len(current_jobs) == 1, "Expected 1 job at init, got {}".format(
len(current_jobs))
logger.info("Getting provider status (1)")
status = provider.status(current_jobs)
logger.info("Got provider status")
assert status[
0].state == JobState.RUNNING, "Expected job to be in state RUNNING"
# Scale down to 0
scale_in_blocks = executor.scale_in(blocks=1)
logger.info("Now sleeping 60 seconds")
time.sleep(60)
logger.info("Sleep finished")
logger.info("Getting provider status (2)")
status = executor.status()
logger.info("Got executor status")
logger.info("Block status: {}".format(status))
assert status[scale_in_blocks[0]].terminal is True, "Terminal state"
logger.info("Job in terminal state")
_, current_jobs = executor._get_block_and_job_ids()
# PR 1952 stoped removing scale_in blocks from self.blocks
# A new PR will handle removing blocks from self.block
# this includes failed/completed/canceled blocks
assert len(current_jobs) == 1, "Expected current_jobs == 1"
dfk.cleanup()
parsl.clear()
logger.info("Ended test_provider")
return True
def test_parsl_config(plot_label, config, timing_function, task_batch_sizes):
parsl.load(config)
fig, axes = plt.subplots(1, 1)
axes.set_xlabel("Number of tasks")
axes.set_ylabel("Time elapsed (s)")
parallel_time_by_jobs = []
for s in task_batch_sizes:
print("Starting to execute {} tasks".format(s))
t = timing_function(s)
print("{} jobs in {}s.".format(s, t))
parallel_time_by_jobs.append(t)
axes.plot(task_batch_sizes, parallel_time_by_jobs)
plt.tight_layout()
plt.savefig(plot_label)
parsl.clear()
def test_provider():
""" Provider scaling
"""
logger.info("Starting test_provider")
config = fresh_config()
name = config.executors[0].label
parsl.load(config)
dfk = parsl.dfk()
logger.info("Trying to get executor : {}".format(name))
x = platform(sleep=0)
logger.info("Result is {}".format(x.result()))
executor = dfk.executors[name]
provider = dfk.executors[name].provider
# At this point we should have 1 job
current_jobs = executor._get_job_ids()
assert len(current_jobs) == 1, "Expected 1 job at init, got {}".format(len(current_jobs))
logger.info("Getting provider status (1)")
status = provider.status(current_jobs)
logger.info("Got provider status")
assert status[0].state == JobState.RUNNING, "Expected job to be in state RUNNING"
# Scale down to 0
scale_in_status = executor.scale_in(blocks=1)
logger.info("Now sleeping 60 seconds")
time.sleep(60)
logger.info("Sleep finished")
logger.info("Getting provider status (2)")
status = provider.status(scale_in_status)
logger.info("Got provider status")
logger.info("Block status: {}".format(status))
assert status[0].terminal is True, "Terminal state"
logger.info("Job in terminal state")
current_jobs = executor._get_job_ids()
assert len(current_jobs) == 0, "Expected current_jobs == 0"
parsl.clear()
del dfk
logger.info("Ended test_provider")
return True
def test_simple(mem_per_worker):
config = Config(
executors=[
HighThroughputExecutor(
poll_period=1,
label="htex_local",
worker_debug=True,
mem_per_worker=mem_per_worker,
cores_per_worker=0.1,
suppress_failure=True,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
launcher=SingleNodeLauncher(),
),
)
],
strategy=None,
)
parsl.load(config)
print("Configuration requests:")
print("cores_per_worker: ", config.executors[0].cores_per_worker)
print("mem_per_worker: ", config.executors[0].mem_per_worker)
available_mem_on_node = round(psutil.virtual_memory().available / (2**30), 1)
expected_workers = multiprocessing.cpu_count() / config.executors[0].cores_per_worker
if mem_per_worker:
expected_workers = int(available_mem_on_node / config.executors[0].mem_per_worker)
print("Available memory: ", available_mem_on_node)
print("Expected workers: ", expected_workers)
# Prime a worker
double(5).result()
dfk = parsl.dfk()
connected = dfk.executors['htex_local'].connected_workers
print("Connected : ", connected)
assert expected_workers == connected, "Expected {} workers, instead got {} workers".format(expected_workers,
connected)
parsl.clear()
return True
def test_1316_local_path_on_execution_side_sp2():
"""This test demonstrates the ability of a StagingProvider to set the
local_path of a File on the execution side, but that the change does not
modify the local_path of the corresponding submit side File, even when
running in a single python process.
"""
config = Config(executors=[ThreadPoolExecutor(storage_access=[SP2()])])
file = File("sp2://test")
parsl.load(config)
p = observe_input_local_path(file).result()
assert p == "./test1.tmp", "File object on the execution side gets the local_path set by the staging provider"
assert not file.local_path, "The local_path on the submit side should not be set"
parsl.clear()
def test_lazy_behavior():
"""Testing that lazy errors work"""
config = fresh_config()
parsl.load(config)
@python_app
def divide(a, b):
return a / b
futures = []
for i in range(0, 10):
futures.append(divide(10, 0))
for f in futures:
assert isinstance(f.exception(), ZeroDivisionError)
assert f.done()
parsl.clear()
return
def test_lazy_behavior():
"""Testing lazy errors to work"""
parsl.clear()
config.lazy_errors = True
parsl.load(config)
@App('python')
def divide(a, b):
return a / b
items = []
for i in range(0, 1):
items.append(divide(10, i))
while True:
if items[0].done:
break
return
def test_1316_local_path_setting_preserves_dependency_sp2():
config = Config(executors=[ThreadPoolExecutor(storage_access=[SP2()])])
file = File("sp2://test")
parsl.load(config)
wc_app_future = wait_and_create(outputs=[file])
data_future = wc_app_future.outputs[0]
p = observe_input_local_path(data_future).result()
assert wc_app_future.done(), "wait_and_create should finish before observe_input_local_path finishes"
assert p == "./test1.tmp", "File object on the execution side gets the local_path set by the staging provider"
assert not file.local_path, "The local_path on the submit side should not be set"
parsl.dfk().cleanup()
parsl.clear()
def test_row_counts():
# this is imported here rather than at module level because
# it isn't available in a plain parsl install, so this module
# would otherwise fail to import and break even a basic test
# run.
import sqlalchemy
from parsl.tests.configs.htex_local_alternate import fresh_config
if os.path.exists("monitoring.db"):
logger.info("Monitoring database already exists - deleting")
os.remove("monitoring.db")
logger.info("loading parsl")
parsl.load(fresh_config())
logger.info("invoking and waiting for result")
assert this_app().result() == 5
logger.info("cleaning up parsl")
parsl.dfk().cleanup()
parsl.clear()
# at this point, we should find one row in the monitoring database.
logger.info("checking database content")
engine = sqlalchemy.create_engine("sqlite:///monitoring.db")
with engine.begin() as connection:
result = connection.execute("SELECT COUNT(*) FROM workflow")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM task")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM try")
(c, ) = result.first()
assert c == 1
logger.info("all done")
def load_dfk_session(request, pytestconfig):
"""Load a dfk around entire test suite, except in local mode.
The special path `local` indicates that configuration will not come
from a pytest managed configuration file; in that case, see
load_dfk_local_module for module-level configuration management.
"""
config = pytestconfig.getoption('config')[0]
if pytestconfig.getoption('bodge_dfk_per_test'):
yield
return
if config != 'local':
spec = importlib.util.spec_from_file_location('', config)
try:
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.config.run_dir = get_rundir(
) # Give unique rundir; needed running with -n=X where X > 1.
if DataFlowKernelLoader._dfk is not None:
raise ValueError(
"DFK didn't start as None - there was a DFK from somewhere already"
)
dfk = parsl.load(module.config)
yield
if (parsl.dfk() != dfk):
raise ValueError("DFK changed unexpectedly during test")
dfk.cleanup()
parsl.clear()
except KeyError:
pytest.skip(
'options in user_opts.py not configured for {}'.format(config))
else:
yield
def test_loading_checkpoint(n=2):
"""Load memoization table from previous checkpoint
"""
config.checkpoint_mode = 'task_exit'
parsl.load(config)
results = launch_n_random(n)
rundir = parsl.dfk().run_dir
parsl.dfk().cleanup()
parsl.clear()
local_config = fresh_config()
local_config.checkpoint_files = [os.path.join(rundir, 'checkpoint')]
parsl.load(local_config)
relaunched = launch_n_random(n)
assert len(relaunched) == len(
results) == n, "Expected all results to have n items"
for i in range(n):
assert relaunched[i] == results[
i], "Expected relaunched to contain cached results from first run"
parsl.clear()
def load_dfk_local_module(request, pytestconfig):
"""Load the dfk around test modules, in local mode.
If local_config is specified in the test module, it will be loaded using
parsl.load. It should be a parsl Config() object.
If local_setup and/or local_teardown are callables (such as functions) in
the test module, they they will be invoked before/after the tests. This
can be used to perform more interesting DFK initialisation not possible
with local_config.
"""
config = pytestconfig.getoption('config')[0]
if config == 'local':
local_setup = getattr(request.module, "local_setup", None)
local_teardown = getattr(request.module, "local_teardown", None)
local_config = getattr(request.module, "local_config", None)
if (local_config):
dfk = parsl.load(local_config)
if (callable(local_setup)):
local_setup()
yield
if (callable(local_teardown)):
local_teardown()
if (local_config):
if (parsl.dfk() != dfk):
raise ValueError("DFK changed unexpectedly during test")
dfk.cleanup()
parsl.clear()
else:
yield
def test_provider():
""" Provider scaling
"""
config = fresh_config()
name = config.executors[0].label
parsl.load(config)
dfk = parsl.dfk()
print("Trying to get executor : ", name)
x = platform(sleep=0)
print(x.result())
executor = dfk.executors[name]
provider = dfk.executors[name].provider
# At this point we should have 1 job
current_jobs = executor._get_job_ids()
assert len(current_jobs) == 1, "Expected 1 job at init, got {}".format(
len(current_jobs))
status = provider.status(current_jobs)
assert status[
0].state == JobState.RUNNING, "Expected job to be in state RUNNING"
# Scale down to 0
scale_in_status = executor.scale_in(blocks=1)
time.sleep(60)
status = provider.status(scale_in_status)
print("Block status: ", status)
assert status[0].terminal is True, "Terminal state"
print("Job in terminal state")
current_jobs = executor._get_job_ids()
assert len(current_jobs) == 0, "Expected current_jobs == 0"
parsl.clear()
del dfk
return True
def local_teardown():
# explicit clear without dfk.cleanup here, because the
# test does that already
parsl.clear()
