def test_initial_checkpoint_write(n=2): """1. Launch a few apps and write the checkpoint once a few have completed """ config = fresh_config() config.checkpoint_mode = 'manual' parsl.load(config) results = launch_n_random(n) cpt_dir = parsl.dfk().checkpoint() cptpath = cpt_dir + '/dfk.pkl' print("Path exists : ", os.path.exists(cptpath)) assert os.path.exists(cptpath), "DFK checkpoint missing: {0}".format( cptpath) cptpath = cpt_dir + '/tasks.pkl' print("Path exists : ", os.path.exists(cptpath)) assert os.path.exists(cptpath), "Tasks checkpoint missing: {0}".format( cptpath) run_dir = parsl.dfk().run_dir parsl.dfk().cleanup() parsl.clear() return run_dir, results
def load_dfk(config): """Load the dfk before running a test. The special path `local` indicates that whatever configuration is loaded locally in the test should not be replaced. Otherwise, it is expected that the supplied file contains a dictionary called `config`, which will be loaded before the test runs. Args: config (str) : path to config to load (this is a parameterized pytest fixture) """ if config != 'local': spec = importlib.util.spec_from_file_location('', config) try: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.config.run_dir = get_rundir( ) # Give unique rundir; needed running with -n=X where X > 1. parsl.clear() dfk = parsl.load(module.config) yield dfk.cleanup() except KeyError: pytest.skip( 'options in user_opts.py not configured for {}'.format(config)) else: yield
def test_one_block(): oneshot_provider = OneShotLocalProvider( channel=LocalChannel(), init_blocks=0, min_blocks=0, max_blocks=10, launcher=SimpleLauncher(), ) config = Config( executors=[ HighThroughputExecutor( label="htex_local", worker_debug=True, cores_per_worker=1, provider=oneshot_provider, ) ], strategy='simple', ) parsl.load(config) f = app() f.result() parsl.clear() assert oneshot_provider.recorded_submits == 1
def test_regression_stage_out_does_not_stage_in(): no_stageout_config = Config( executors=[ ThreadPoolExecutor( label='local_threads', storage_access=[NoOpTestingFileStaging(allow_stage_in=False)] ) ] ) parsl.load(no_stageout_config) # Test that the helper app runs with no staging touch("test.1", outputs=[]).result() # Test with stage-out, checking that provider stage in is never # invoked. If stage-in is invoked, the the NoOpTestingFileStaging # provider will raise an exception, which should propagate to # .result() here. touch("test.2", outputs=[File("test.2")]).result() # Test that stage-in exceptions propagate out to user code. with pytest.raises(NoOpError): touch("test.3", inputs=[File("test.3")]).result() parsl.dfk().cleanup() parsl.clear()
def load_dfk(config): """Load the dfk before running a test. The special path `local` indicates that whatever configuration is loaded locally in the test should not be replaced. Otherwise, it is expected that the supplied file contains a dictionary called `config`, which will be loaded before the test runs. Args: config (str) : path to config to load (this is a parameterized pytest fixture) """ if config != 'local': spec = importlib.util.spec_from_file_location('', config) try: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.config.run_dir = get_rundir() # Give unique rundir; needed running with -n=X where X > 1. if DataFlowKernelLoader._dfk is not None: raise ValueError("DFK didn't start as None - there was a DFK from somewhere already") parsl.clear() dfk = parsl.load(module.config) yield if(parsl.dfk() != dfk): raise ValueError("DFK changed unexpectedly during test") dfk.cleanup() parsl.clear() except KeyError: pytest.skip('options in user_opts.py not configured for {}'.format(config)) else: yield
def load_dfk_session(request, pytestconfig): """Load a dfk around entire test suite, except in local mode. The special path `local` indicates that configuration will not come from a pytest managed configuration file; in that case, see load_dfk_local_module for module-level configuration management. """ config = pytestconfig.getoption('config')[0] if config != 'local': spec = importlib.util.spec_from_file_location('', config) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) if DataFlowKernelLoader._dfk is not None: raise ValueError( "DFK didn't start as None - there was a DFK from somewhere already" ) dfk = parsl.load(module.config) yield if (parsl.dfk() != dfk): raise ValueError("DFK changed unexpectedly during test") dfk.cleanup() parsl.clear() else: yield
def test_platform(n=2, sleep_dur=10): """ This should sleep to make sure that concurrent apps will go to different workers on different nodes. """ config = fresh_config() if config.executors[0].label == "htex_local": return parsl.load(fresh_config()) dfk = parsl.dfk() name = list(dfk.executors.keys())[0] print("Trying to get executor : ", name) x = [platform(sleep=1) for i in range(2)] print([i.result() for i in x]) print("Executor : ", dfk.executors[name]) print("Connected : ", dfk.executors[name].connected_workers) print("Outstanding : ", dfk.executors[name].outstanding) d = [] for i in range(0, n): x = platform(sleep=sleep_dur) d.append(x) pinfo = set([i.result() for i in d]) assert len(pinfo) == 2, "Expected two nodes, instead got {}".format(pinfo) print("Test passed") dfk.cleanup() parsl.clear() return True
def test_non_lazy_behavior(): """Testing non lazy errors to work""" parsl.clear() config.lazy_errors = False parsl.load(config) @App('python') def divide(a, b): return a / b try: items = [] for i in range(0, 1): items.append(divide(10, i)) while True: if items[0].done: break except Exception as e: assert isinstance( e, ZeroDivisionError), "Expected ZeroDivisionError, got: {}".format(e) else: raise Exception("Expected ZeroDivisionError, got nothing") return
def test_dynamic_executor(): dfk = parsl.load() tasks = [sleeper() for i in range(5)] results = [i.result() for i in tasks] print("Done with initial test. The results are", results) # Here we add a new executor to an active DFK thread_executors = [ThreadPoolExecutor(label='threads2', max_threads=4)] dfk.add_executors(executors=thread_executors) tasks = [cpu_stress() for i in range(8)] results = [i.result() for i in tasks] print( "Successfully added thread executor and ran with it. The results are", results) # We add a htex executor to an active DFK executors = [ HighThroughputExecutor( label='htex_local', cores_per_worker=1, max_workers=5, provider=LocalProvider( init_blocks=1, max_blocks=1, ), ) ] dfk.add_executors(executors=executors) tasks = [add() for i in range(10)] results = [i.result() for i in tasks] print("Successfully added htex executor and ran with it. The results are", results) print("Done testing") parsl.clear()
def load_dfk(config): """Load the dfk before running a test. The special path `local` indicates that whatever configuration is loaded locally in the test should not be replaced. Otherwise, it is expected that the supplied file contains a dictionary called `config`, which will be loaded before the test runs. Args: config (str) : path to config to load (this is a parameterized pytest fixture) """ if config != 'local': spec = importlib.util.spec_from_file_location('', config) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) if 'globals' not in module.config: module.config['globals'] = {} module.config['globals']['runDir'] = get_rundir( ) # Give unique rundir; needed running with -n=X where X > 1. parsl.clear() dfk = parsl.load(module.config) yield dfk.cleanup() else: yield
def run1(*config_changes, dry_run=False, expect_fail=False, expect_outputs=True): try: with tempfile.TemporaryDirectory() as dirname: out_dir = os.path.join(dirname, "output") log_dir = os.path.join(dirname, "logs") config = [f"output_dir={out_dir}", f"log_dir={log_dir}"] config += config_changes pipe_config = Pipeline.build_config("tests/test.yml", config, dry_run) status = run(pipe_config, "tests/test.yml", config, dry_run) if expect_fail: assert status != 0 else: assert status == 0 if expect_outputs: assert os.path.exists( os.path.join(out_dir, "wlgc_summary_data.txt")) assert os.path.exists( os.path.join(log_dir, "WLGCSummaryStatistic.out")) finally: clear()
def _return_value_test_(resume): expected_status = 0 if resume else 1 # Mini pipeline should not run launcher_config = {"interval": 0.5, "name": "mini"} run_config = { "log_dir": "./tests/logs", "output_dir": "./tests/inputs", "resume": resume, } pipeline = MiniPipeline([{"name": "FailingStage"}], launcher_config) pipeline.initialize({}, run_config, "tests/config.yml") status = pipeline.run() assert status == expected_status # Parsl pipeline should not run stage either launcher_config = {"name": "parsl"} site_config = {"name": "local", "max_threads": 1} load(launcher_config, [site_config]) # the above sets the new default to be the parsl-configured site pipeline = ParslPipeline([{"name": "FailingStage"}], launcher_config) pipeline.initialize({}, run_config, "tests/config.yml") status = pipeline.run() assert status == expected_status clear() # clear parsl settings reset_default_site() # reset so default is minirunner again
def test_regression_stage_in_does_not_stage_out(): no_stageout_config = Config( executors=[ ThreadPoolExecutor( label='local_threads', storage_access=[NoOpTestingFileStaging(allow_stage_out=False)] ) ], ) parsl.load(no_stageout_config) f = open("test.4", "a") f.write("test") f.close() # Test that stage in does not invoke stage out. If stage out is # attempted, then the NoOpTestingFileStaging provider will raise # an exception which should propagate here. app_test_in(File("test.4")).result() # Test that stage out exceptions propagate to user code. with pytest.raises(NoOpError): touch("test.5", outputs=[File("test.5")]).result() parsl.dfk().cleanup() parsl.clear()
def test_loading_checkpoint(n=2): """Load memoization table from previous checkpoint """ parsl.load(config) rundir = test1.test_initial_checkpoint_write() parsl.clear() local_config = fresh_config() local_config.checkpoint_files = [os.path.join(rundir, 'checkpoint')] parsl.load(local_config) d = {} start = time.time() print("Launching : ", n) for i in range(0, n): d[i] = slow_double(i) print("Done launching") for i in range(0, n): d[i].result() print("Done sleeping") delta = time.time() - start assert delta < 1, "Took longer than a second ({}), assuming restore from checkpoint failed".format( delta) parsl.clear()
def test_row_counts(): # this is imported here rather than at module level because # it isn't available in a plain parsl install, so this module # would otherwise fail to import and break even a basic test # run. import sqlalchemy from parsl.tests.configs.htex_local_alternate import fresh_config if os.path.exists("monitoring.db"): logger.info("Monitoring database already exists - deleting") os.remove("monitoring.db") logger.info("loading parsl") parsl.load(fresh_config()) logger.info("invoking and waiting for result") assert this_app().result() == 5 logger.info("cleaning up parsl") parsl.dfk().cleanup() parsl.clear() # at this point, we should find one row in the monitoring database. logger.info("checking database content") engine = sqlalchemy.create_engine("sqlite:///monitoring.db") with engine.begin() as connection: result = connection.execute("SELECT COUNT(*) FROM workflow") (c, ) = result.first() assert c == 1 result = connection.execute("SELECT COUNT(*) FROM task") (c, ) = result.first() assert c == 1 result = connection.execute("SELECT COUNT(*) FROM try") (c, ) = result.first() assert c == 1 result = connection.execute("SELECT COUNT(*) FROM status, try " "WHERE status.task_id = try.task_id " "AND status.task_status_name='exec_done' " "AND task_try_time_running is NULL") (c, ) = result.first() assert c == 0 # Two entries: one showing manager active, one inactive result = connection.execute("SELECT COUNT(*) FROM node") (c, ) = result.first() assert c == 2 # There should be one block polling status # local provider has a status_polling_interval of 5s result = connection.execute("SELECT COUNT(*) FROM block") (c, ) = result.first() assert c >= 2 logger.info("all done")
def test_summary(caplog): parsl.load(fresh_config()) succeed().result() fail().exception() parsl.dfk().cleanup() parsl.clear() assert "Summary of tasks in DFK:" in caplog.text assert "Tasks in state States.exec_done: 1" in caplog.text assert "Tasks in state States.failed: 1" in caplog.text
def test_provider(): """ Provider scaling """ logger.info("Starting test_provider") config = fresh_config() name = config.executors[0].label parsl.load(config) dfk = parsl.dfk() logger.info("Trying to get executor : {}".format(name)) x = platform(sleep=0) logger.info("Result is {}".format(x.result())) executor = dfk.executors[name] provider = dfk.executors[name].provider # At this point we should have 1 job _, current_jobs = executor._get_block_and_job_ids() assert len(current_jobs) == 1, "Expected 1 job at init, got {}".format( len(current_jobs)) logger.info("Getting provider status (1)") status = provider.status(current_jobs) logger.info("Got provider status") assert status[ 0].state == JobState.RUNNING, "Expected job to be in state RUNNING" # Scale down to 0 scale_in_blocks = executor.scale_in(blocks=1) logger.info("Now sleeping 60 seconds") time.sleep(60) logger.info("Sleep finished") logger.info("Getting provider status (2)") status = executor.status() logger.info("Got executor status") logger.info("Block status: {}".format(status)) assert status[scale_in_blocks[0]].terminal is True, "Terminal state" logger.info("Job in terminal state") _, current_jobs = executor._get_block_and_job_ids() # PR 1952 stoped removing scale_in blocks from self.blocks # A new PR will handle removing blocks from self.block # this includes failed/completed/canceled blocks assert len(current_jobs) == 1, "Expected current_jobs == 1" dfk.cleanup() parsl.clear() logger.info("Ended test_provider") return True
def test_parsl_config(plot_label, config, timing_function, task_batch_sizes): parsl.load(config) fig, axes = plt.subplots(1, 1) axes.set_xlabel("Number of tasks") axes.set_ylabel("Time elapsed (s)") parallel_time_by_jobs = [] for s in task_batch_sizes: print("Starting to execute {} tasks".format(s)) t = timing_function(s) print("{} jobs in {}s.".format(s, t)) parallel_time_by_jobs.append(t) axes.plot(task_batch_sizes, parallel_time_by_jobs) plt.tight_layout() plt.savefig(plot_label) parsl.clear()
def test_provider(): """ Provider scaling """ logger.info("Starting test_provider") config = fresh_config() name = config.executors[0].label parsl.load(config) dfk = parsl.dfk() logger.info("Trying to get executor : {}".format(name)) x = platform(sleep=0) logger.info("Result is {}".format(x.result())) executor = dfk.executors[name] provider = dfk.executors[name].provider # At this point we should have 1 job current_jobs = executor._get_job_ids() assert len(current_jobs) == 1, "Expected 1 job at init, got {}".format(len(current_jobs)) logger.info("Getting provider status (1)") status = provider.status(current_jobs) logger.info("Got provider status") assert status[0].state == JobState.RUNNING, "Expected job to be in state RUNNING" # Scale down to 0 scale_in_status = executor.scale_in(blocks=1) logger.info("Now sleeping 60 seconds") time.sleep(60) logger.info("Sleep finished") logger.info("Getting provider status (2)") status = provider.status(scale_in_status) logger.info("Got provider status") logger.info("Block status: {}".format(status)) assert status[0].terminal is True, "Terminal state" logger.info("Job in terminal state") current_jobs = executor._get_job_ids() assert len(current_jobs) == 0, "Expected current_jobs == 0" parsl.clear() del dfk logger.info("Ended test_provider") return True
def test_simple(mem_per_worker): config = Config( executors=[ HighThroughputExecutor( poll_period=1, label="htex_local", worker_debug=True, mem_per_worker=mem_per_worker, cores_per_worker=0.1, suppress_failure=True, provider=LocalProvider( channel=LocalChannel(), init_blocks=1, max_blocks=1, launcher=SingleNodeLauncher(), ), ) ], strategy=None, ) parsl.load(config) print("Configuration requests:") print("cores_per_worker: ", config.executors[0].cores_per_worker) print("mem_per_worker: ", config.executors[0].mem_per_worker) available_mem_on_node = round(psutil.virtual_memory().available / (2**30), 1) expected_workers = multiprocessing.cpu_count() / config.executors[0].cores_per_worker if mem_per_worker: expected_workers = int(available_mem_on_node / config.executors[0].mem_per_worker) print("Available memory: ", available_mem_on_node) print("Expected workers: ", expected_workers) # Prime a worker double(5).result() dfk = parsl.dfk() connected = dfk.executors['htex_local'].connected_workers print("Connected : ", connected) assert expected_workers == connected, "Expected {} workers, instead got {} workers".format(expected_workers, connected) parsl.clear() return True
def test_1316_local_path_on_execution_side_sp2(): """This test demonstrates the ability of a StagingProvider to set the local_path of a File on the execution side, but that the change does not modify the local_path of the corresponding submit side File, even when running in a single python process. """ config = Config(executors=[ThreadPoolExecutor(storage_access=[SP2()])]) file = File("sp2://test") parsl.load(config) p = observe_input_local_path(file).result() assert p == "./test1.tmp", "File object on the execution side gets the local_path set by the staging provider" assert not file.local_path, "The local_path on the submit side should not be set" parsl.clear()
def test_lazy_behavior(): """Testing that lazy errors work""" config = fresh_config() parsl.load(config) @python_app def divide(a, b): return a / b futures = [] for i in range(0, 10): futures.append(divide(10, 0)) for f in futures: assert isinstance(f.exception(), ZeroDivisionError) assert f.done() parsl.clear() return
def test_lazy_behavior(): """Testing lazy errors to work""" parsl.clear() config.lazy_errors = True parsl.load(config) @App('python') def divide(a, b): return a / b items = [] for i in range(0, 1): items.append(divide(10, i)) while True: if items[0].done: break return
def test_1316_local_path_setting_preserves_dependency_sp2(): config = Config(executors=[ThreadPoolExecutor(storage_access=[SP2()])]) file = File("sp2://test") parsl.load(config) wc_app_future = wait_and_create(outputs=[file]) data_future = wc_app_future.outputs[0] p = observe_input_local_path(data_future).result() assert wc_app_future.done(), "wait_and_create should finish before observe_input_local_path finishes" assert p == "./test1.tmp", "File object on the execution side gets the local_path set by the staging provider" assert not file.local_path, "The local_path on the submit side should not be set" parsl.dfk().cleanup() parsl.clear()
def test_row_counts(): # this is imported here rather than at module level because # it isn't available in a plain parsl install, so this module # would otherwise fail to import and break even a basic test # run. import sqlalchemy from parsl.tests.configs.htex_local_alternate import fresh_config if os.path.exists("monitoring.db"): logger.info("Monitoring database already exists - deleting") os.remove("monitoring.db") logger.info("loading parsl") parsl.load(fresh_config()) logger.info("invoking and waiting for result") assert this_app().result() == 5 logger.info("cleaning up parsl") parsl.dfk().cleanup() parsl.clear() # at this point, we should find one row in the monitoring database. logger.info("checking database content") engine = sqlalchemy.create_engine("sqlite:///monitoring.db") with engine.begin() as connection: result = connection.execute("SELECT COUNT(*) FROM workflow") (c, ) = result.first() assert c == 1 result = connection.execute("SELECT COUNT(*) FROM task") (c, ) = result.first() assert c == 1 result = connection.execute("SELECT COUNT(*) FROM try") (c, ) = result.first() assert c == 1 logger.info("all done")
def load_dfk_session(request, pytestconfig): """Load a dfk around entire test suite, except in local mode. The special path `local` indicates that configuration will not come from a pytest managed configuration file; in that case, see load_dfk_local_module for module-level configuration management. """ config = pytestconfig.getoption('config')[0] if pytestconfig.getoption('bodge_dfk_per_test'): yield return if config != 'local': spec = importlib.util.spec_from_file_location('', config) try: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.config.run_dir = get_rundir( ) # Give unique rundir; needed running with -n=X where X > 1. if DataFlowKernelLoader._dfk is not None: raise ValueError( "DFK didn't start as None - there was a DFK from somewhere already" ) dfk = parsl.load(module.config) yield if (parsl.dfk() != dfk): raise ValueError("DFK changed unexpectedly during test") dfk.cleanup() parsl.clear() except KeyError: pytest.skip( 'options in user_opts.py not configured for {}'.format(config)) else: yield
def test_loading_checkpoint(n=2): """Load memoization table from previous checkpoint """ config.checkpoint_mode = 'task_exit' parsl.load(config) results = launch_n_random(n) rundir = parsl.dfk().run_dir parsl.dfk().cleanup() parsl.clear() local_config = fresh_config() local_config.checkpoint_files = [os.path.join(rundir, 'checkpoint')] parsl.load(local_config) relaunched = launch_n_random(n) assert len(relaunched) == len( results) == n, "Expected all results to have n items" for i in range(n): assert relaunched[i] == results[ i], "Expected relaunched to contain cached results from first run" parsl.clear()
def load_dfk_local_module(request, pytestconfig): """Load the dfk around test modules, in local mode. If local_config is specified in the test module, it will be loaded using parsl.load. It should be a parsl Config() object. If local_setup and/or local_teardown are callables (such as functions) in the test module, they they will be invoked before/after the tests. This can be used to perform more interesting DFK initialisation not possible with local_config. """ config = pytestconfig.getoption('config')[0] if config == 'local': local_setup = getattr(request.module, "local_setup", None) local_teardown = getattr(request.module, "local_teardown", None) local_config = getattr(request.module, "local_config", None) if (local_config): dfk = parsl.load(local_config) if (callable(local_setup)): local_setup() yield if (callable(local_teardown)): local_teardown() if (local_config): if (parsl.dfk() != dfk): raise ValueError("DFK changed unexpectedly during test") dfk.cleanup() parsl.clear() else: yield
def test_provider(): """ Provider scaling """ config = fresh_config() name = config.executors[0].label parsl.load(config) dfk = parsl.dfk() print("Trying to get executor : ", name) x = platform(sleep=0) print(x.result()) executor = dfk.executors[name] provider = dfk.executors[name].provider # At this point we should have 1 job current_jobs = executor._get_job_ids() assert len(current_jobs) == 1, "Expected 1 job at init, got {}".format( len(current_jobs)) status = provider.status(current_jobs) assert status[ 0].state == JobState.RUNNING, "Expected job to be in state RUNNING" # Scale down to 0 scale_in_status = executor.scale_in(blocks=1) time.sleep(60) status = provider.status(scale_in_status) print("Block status: ", status) assert status[0].terminal is True, "Terminal state" print("Job in terminal state") current_jobs = executor._get_job_ids() assert len(current_jobs) == 0, "Expected current_jobs == 0" parsl.clear() del dfk return True
def local_teardown(): # explicit clear without dfk.cleanup here, because the # test does that already parsl.clear()