def test_xenon_42_multi(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = schedule(sum)(gather_all(multiples)) machine = Machine( scheduler_adaptor='slurm', location='ssh://fs0.das5.cs.vu.nl/home/jhidding', credential=xenon.CertificateCredential( username='******', certfile='/home/johannes/.ssh/id_rsa'), jobs_properties={ 'xenon.adaptors.schedulers.ssh.strictHostKeyChecking': 'false' }) worker_config = XenonJobConfig( prefix=Path('/home/jhidding/.local/share/workon/mcfly'), working_dir='/home/jhidding/', time_out=1000000000000, verbose=False) # , options=['-C', 'TitanX', '--gres=gpu:1']) result = run_xenon(C, machine=machine, worker_config=worker_config, n_processes=2) print("The answer is:", result)
def test_broker_02(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) assert run_single(C) == 42
def test_broker_01(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(noodles.gather(*multiples)) assert run_parallel(C, 4) == 42
def test_prov_02(): db_file = "prov1.json" A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(gather(*multiples)) result = run_single(C, serial.base, db_file) assert result == 42 os.unlink(db_file)
def test_prov_03(): db_file = "prov2.json" A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = accumulate(gather(*multiples)) result = run_parallel(C, 4, serial.base, db_file, JobKeeper(keep=True)) assert result == 42 os.unlink(db_file)
def test_prov_00(): reg = serial.base() a = add(3, 4) b = sub(3, 4) c = add(3, 4) d = add(4, 3) enc = [reg.deep_encode(x._workflow.root_node) for x in [a, b, c, d]] key = [prov_key(o) for o in enc] assert key[0] == key[2] assert key[1] != key[0] assert key[3] != key[0]
def test_xenon_42_simple(xenon_server): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = schedule(sum)(gather_all(multiples)) machine = Machine() worker_config = XenonJobConfig(verbose=True) result = run_xenon_simple(C, machine=machine, worker_config=worker_config) assert (result == 42)
def test_xenon_42_multi(xenon_server): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] C = schedule(sum)(gather_all(multiples)) machine = Machine() worker_config = XenonJobConfig(queue_name='multi', verbose=True) result = run_xenon(C, machine=machine, worker_config=worker_config, n_processes=2) assert (result == 42)
def test_lift_01(): a = A() a.x = add(1, 2) a.y = sub(9, 11) b = f(lift(a)) result = run_single(b) assert result == 1
def test_prov_01(): reg = serial.base() a = add(3, 4) enc = reg.deep_encode(a._workflow.root_node) dec = reg.deep_decode(enc) result = run_job(0, dec) assert result.value == 7
def static_sum(values, limit_n=1000): """Example of static sum routine.""" if len(values) < limit_n: return sum(values) else: half = len(values) // 2 return add( static_sum(values[:half], limit_n), static_sum(values[half:], limit_n))
def dynamic_sum(values, limit_n=1000, acc=0, depth=4): """Example of dynamic sum.""" if len(values) < limit_n: return acc + sum(values) if depth > 0: half = len(values) // 2 return add( dynamic_sum(values[:half], limit_n, acc, depth=depth-1), dynamic_sum(values[half:], limit_n, 0, depth=depth-1)) return dynamic_sum(values[limit_n:], limit_n, acc + sum(values[:limit_n]), depth)
def test_get_intermediate_results(): r1 = add(1, 1) r2 = sub(3, r1) def foo(a, b, c): return mul(add(a, b), c) r3 = [foo(i, r2, r1) for i in range(6)] r4 = accumulate(noodles.gather_all(r3)) run_single(r4) assert noodles.result(r1) == 2 assert noodles.result(r2) == 1 assert [noodles.result(r) for r in r3] == [2, 4, 6, 8, 10, 12] assert noodles.result(r4) == 42
def test_attaching(): db = JobDB(':memory:', registry=serial.base) wf = add(1, 1) job = Job(wf._workflow, wf._workflow.root) key1, node1 = db.register(job) msg, value = db.add_job_to_db(key1, node1) assert msg == 'initialized' key2, node2 = db.register(job) msg, value = db.add_job_to_db(key2, node2) assert msg == 'attached' duplicates = db.store_result_in_db(ResultMessage(key1, 'done', 2, None)) assert duplicates == (key2, ) key3, node3 = db.register(job) msg, result = db.add_job_to_db(key3, node3) assert msg == 'retrieved' assert result.value == 2
def test_global_prov(): a = add(4, 5) b = sub(a, 3) c = mul(b, 7) d = mul(b, 7) e = sub(c, 3) f = sub(c, 5) set_global_provenance(c._workflow, base()) assert c._workflow.prov is not None assert b._workflow.prov is not None assert c._workflow.prov != b._workflow.prov set_global_provenance(d._workflow, base()) set_global_provenance(e._workflow, base()) set_global_provenance(f._workflow, base()) assert c._workflow.prov == d._workflow.prov assert b._workflow.prov != e._workflow.prov assert f._workflow.prov != e._workflow.prov
def test_worker_one_batch_job(xenon_server, tmpdir): tmpdir = Path(str(tmpdir)) infile = tmpdir / 'infile.json' outfile = tmpdir / 'outfile.json' wf = add(1, 1) job = next(iter(wf._workflow.nodes.values())) job_message = JobMessage(42, job) registry = serial.base() print(registry.to_json(job_message), file=infile.open('w')) m = Machine(scheduler_adaptor='local') scheduler = m.scheduler job_config = XenonJobConfig() executable, arguments = job_config.command_line() job_description = xenon.JobDescription(executable=str(executable), arguments=arguments, stdin=str(infile), stdout=str(outfile)) job = scheduler.submit_batch_job(job_description) scheduler.wait_until_done(job) result_json = [line.strip() for line in outfile.open()] assert len(result_json) == 1 result = registry.from_json(result_json[0]) assert result.status == 'done' assert result.key == 42 assert result.msg is None assert result.value == 2 scheduler.close()
def test_gather_all(): d = noodles.gather_all(add(x, 2*x) for x in range(10)) result = run_single(d) assert result == list(range(0, 30, 3))
def test_truthfulness(): w = sub(4, add(3, 2)) a = cond(True, w, should_not_run()) b = cond(False, should_not_run(), w) result = run_single(mul(a, b)) assert result == 1
from noodles.tutorial import (add, sub, mul) from noodles.draw_workflow import draw_workflow u = add(5, 4) v = sub(u, 3) w = sub(u, 2) x = mul(v, w) draw_workflow("callgraph-a.pdf", x._workflow)
def lift_ordered_dict(): x = OrderedDict() x['a'] = 1 x['b'] = add(1, 2) return g(noodles.lift(x))
def test_gather(): d = noodles.gather(*[add(x, x) for x in range(10)]) result = run_single(d) assert result == list(range(0, 20, 2))
def foo(a, b, c): return mul(add(a, b), c)
def fibonacci(n): if n < 2: return 1 else: return add(fibonacci(n - 1), fibonacci(n - 2))
def test_unpack_00(): a, b, c = unpack(f(), 3) d = run_single(add(a, add(b, c))) assert d == 6
def test_42(): A = add(1, 1) B = sub(3, A) multiples = [mul(add(i, B), A) for i in range(6)] return accumulate(noodles.gather(*multiples))
def truthfulness(): w = sub(4, add(3, 2)) a = cond(True, w, should_not_run()) b = cond(False, should_not_run(), w) return mul(a, b)
def lift_object(): a = A() a.x = add(1, 2) a.y = sub(9, 11) return f(noodles.lift(a))
def unpack(): a, b, c = noodles.unpack(f(), 3) return add(a, add(b, c))
from noodles.tutorial import (add, sub, mul) from draw_workflow import draw_workflow u = add(5, 4) v = sub(u, 3) w = sub(u, 2) x = mul(v, w) draw_workflow("callgraph-a.pdf", x._workflow)
def fib(n): if n < 2: return 1 else: return add(fib(n - 1), fib(n - 2))
from noodles import (schedule, run_single) from noodles.tutorial import (add) from noodles.draw_workflow import draw_workflow @schedule class A: def __init__(self, value): self.value = value @property def square(self): return self.value**2 @square.setter def square(self, sqr): self.value = sqr**(1 / 2) u = A(3) v = A(4) u.square = add(u.square, v.square) draw_workflow("pythagoras.pdf", u.value._workflow) print("⎷(3² + 4²) = ", run_single(u.value))
def gather(): return noodles.gather(*[add(x, x) for x in range(10)])
def gather_dict(): return noodles.gather_dict(a=1, b=add(1, 1), c=add(2, 3))
def test_gather_all(): return noodles.gather_all(add(x, 2 * x) for x in range(10))
def test_gather_dict(): d = noodles.gather_dict(a=1, b=add(1, 1), c=add(2, 3)) result = run_single(d) assert result == {'a': 1, 'b': 2, 'c': 5}