def test_worker_one_online_job(xenon_server): wf = mul(6, 7) job = next(iter(wf._workflow.nodes.values())) job_message = JobMessage(1234, job) registry = serial.base() msg = registry.to_json(job_message) m = Machine(scheduler_adaptor='local') scheduler = m.scheduler job_config = XenonJobConfig() executable, arguments = job_config.command_line() xjob_description = xenon.JobDescription(executable=str(executable), arguments=arguments) xjob, xstdout = scheduler.submit_interactive_job(xjob_description, [msg.encode()]) scheduler.wait_until_done(xjob) result_json = ''.join(m.stdout.decode() for m in xstdout if m.stdout) assert len(result_json) > 0 result = registry.from_json(result_json) assert result.status == 'done' assert result.key == 1234 assert result.msg is None assert result.value == 42 scheduler.close()
def test_xenon_42_multi(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = schedule(sum)(gather_all(multiples)) machine = Machine( scheduler_adaptor='slurm', location='ssh://fs0.das5.cs.vu.nl/home/jhidding', credential=xenon.CertificateCredential( username='******', certfile='/home/johannes/.ssh/id_rsa'), jobs_properties={ 'xenon.adaptors.schedulers.ssh.strictHostKeyChecking': 'false' }) worker_config = XenonJobConfig( prefix=Path('/home/jhidding/.local/share/workon/mcfly'), working_dir='/home/jhidding/', time_out=1000000000000, verbose=False) # , options=['-C', 'TitanX', '--gres=gpu:1']) result = run_xenon(C, machine=machine, worker_config=worker_config, n_processes=2) print("The answer is:", result)
def test_broker_02(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) assert run_single(C) == 42
def test_broker_01(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) assert run_parallel(C, 4) == 42
def test_xenon_42(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) result = run_parallel_timing(C, 4, sys.stdout) assert (result == 42)
def test_xenon_42(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) result = run_parallel_timing(C, 4, sys.stdout) assert(result == 42)
def test_broker_logging(): A = log_add(1, 1) B = sub(3, A) multiples = [mul(log_add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) with NCDisplay(title="Running the test") as display: assert run_parallel_with_display(C, 4, display) == 42
def test_prov_03(): db_file = "prov2.json" A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(gather(*multiples)) result = run_parallel(C, 4, serial.base, db_file, JobKeeper(keep=True)) assert result == 42 os.unlink(db_file)
def test_prov_02(): db_file = "prov1.json" A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(gather(*multiples)) result = run_single(C, serial.base, db_file) assert result == 42 os.unlink(db_file)
def test_global_prov(): a = add(4, 5) b = sub(a, 3) c = mul(b, 7) d = mul(b, 7) e = sub(c, 3) f = sub(c, 5) set_global_provenance(c._workflow, base()) assert c._workflow.prov is not None assert b._workflow.prov is not None assert c._workflow.prov != b._workflow.prov set_global_provenance(d._workflow, base()) set_global_provenance(e._workflow, base()) set_global_provenance(f._workflow, base()) assert c._workflow.prov == d._workflow.prov assert b._workflow.prov != e._workflow.prov assert f._workflow.prov != e._workflow.prov
def test_prov_04(): db_file = "prov3.json" A = add2(1, 1) B = sub(3, A) multiples = [mul(add2(i, B), A) for i in range(6)] C = accumulate(gather(*multiples)) result = run_parallel_opt(C, 4, serial.base, db_file) assert result == 42 os.unlink(db_file)
def test_xenon_42_simple(xenon_server): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = schedule(sum)(gather_all(multiples)) machine = Machine() worker_config = XenonJobConfig(verbose=True) result = run_xenon_simple(C, machine=machine, worker_config=worker_config) assert (result == 42)
def test_xenon_42_multi(xenon_server): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = schedule(sum)(gather_all(multiples)) machine = Machine() worker_config = XenonJobConfig(queue_name='multi', verbose=True) result = run_xenon(C, machine=machine, worker_config=worker_config, n_processes=2) assert (result == 42)
def test_worker_ten_online_jobs(xenon_server): registry = serial.base() def single_job(wf): job = next(iter(wf._workflow.nodes.values())) job_message = JobMessage(0, job) return (registry.to_json(job_message) + '\n').encode() m = Machine(scheduler_adaptor='local') scheduler = m.scheduler job_config = XenonJobConfig(verbose=True) executable, arguments = job_config.command_line() xjob_description = xenon.JobDescription(executable=str(executable), arguments=arguments) xjob, xstdout = scheduler.submit_interactive_job( xjob_description, [single_job(mul(10, i)) for i in range(10)]) scheduler.wait_until_done(xjob) result_json = "" for m in xstdout: if m.stdout: result_json += m.stdout.decode() if m.stderr: for l in m.stderr.decode().splitlines(): print("remote:", l) results = [registry.from_json(r) for r in result_json.splitlines()] print("results: ", end='') for r in results: print(r.value, end=' ') print() assert len(results) == 10 for i, result in enumerate(results): assert result.status == 'done' assert result.key == 0 assert result.msg is None assert result.value == i * 10 scheduler.close()
def test_xenon_42(): A = log_add(1, 1) B = sub(3, A) multiples = [mul(log_add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) xenon_config = XenonConfig(jobs_scheme='local') job_config = RemoteJobConfig(registry=serial.base, time_out=1000) with XenonKeeper() as Xe, NCDisplay() as display: result = run_xenon(C, Xe, "cache.json", 2, xenon_config, job_config, display=display) assert (result == 42)
def test_xenon_42(): A = log_add(1, 1) B = sub(3, A) multiples = [mul(log_add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) xenon_config = XenonConfig( jobs_scheme='local' ) job_config = RemoteJobConfig( registry=serial.base, time_out=1000 ) with XenonKeeper() as Xe, NCDisplay() as display: result = run_xenon( C, Xe, "cache.json", 2, xenon_config, job_config, display=display) assert(result == 42)
def test_truthfulness(): w = sub(4, add(3, 2)) a = cond(True, w, should_not_run()) b = cond(False, should_not_run(), w) result = run_single(mul(a, b)) assert result == 1
from noodles.tutorial import (add, sub, mul) from draw_workflow import draw_workflow u = add(5, 4) v = sub(u, 3) w = sub(u, 2) x = mul(v, w) draw_workflow("callgraph-a.pdf", x._workflow)
from noodles.tutorial import (add, sub, mul) from noodles.draw_workflow import draw_workflow u = add(5, 4) v = sub(u, 3) w = sub(u, 2) x = mul(v, w) draw_workflow("callgraph-a.pdf", x._workflow)
def test_42(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] return accumulate(noodles.gather(*multiples))
def truthfulness(): w = sub(4, add(3, 2)) a = cond(True, w, should_not_run()) b = cond(False, should_not_run(), w) return mul(a, b)
def foo(a, b, c): return mul(add(a, b), c)