def test_xenon_42_multi():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine(
scheduler_adaptor='slurm',
location='ssh://fs0.das5.cs.vu.nl/home/jhidding',
credential=xenon.CertificateCredential(
username='******', certfile='/home/johannes/.ssh/id_rsa'),
jobs_properties={
'xenon.adaptors.schedulers.ssh.strictHostKeyChecking': 'false'
})
worker_config = XenonJobConfig(
prefix=Path('/home/jhidding/.local/share/workon/mcfly'),
working_dir='/home/jhidding/',
time_out=1000000000000,
verbose=False) # , options=['-C', 'TitanX', '--gres=gpu:1'])
result = run_xenon(C,
machine=machine,
worker_config=worker_config,
n_processes=2)
print("The answer is:", result)
def test_broker_02():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_single(C) == 42
def test_broker_01():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_parallel(C, 4) == 42
def test_broker_02():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_single(C) == 42
def test_broker_01():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_parallel(C, 4) == 42
def test_prov_02():
db_file = "prov1.json"
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(gather(*multiples))
result = run_single(C, serial.base, db_file)
assert result == 42
os.unlink(db_file)
def test_prov_03():
db_file = "prov2.json"
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(gather(*multiples))
result = run_parallel(C, 4, serial.base, db_file, JobKeeper(keep=True))
assert result == 42
os.unlink(db_file)
def test_prov_00():
reg = serial.base()
a = add(3, 4)
b = sub(3, 4)
c = add(3, 4)
d = add(4, 3)
enc = [reg.deep_encode(x._workflow.root_node) for x in [a, b, c, d]]
key = [prov_key(o) for o in enc]
assert key[0] == key[2]
assert key[1] != key[0]
assert key[3] != key[0]
def test_xenon_42_simple(xenon_server):
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine()
worker_config = XenonJobConfig(verbose=True)
result = run_xenon_simple(C, machine=machine, worker_config=worker_config)
assert (result == 42)
def test_xenon_42_multi(xenon_server):
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine()
worker_config = XenonJobConfig(queue_name='multi', verbose=True)
result = run_xenon(C,
machine=machine,
worker_config=worker_config,
n_processes=2)
assert (result == 42)
def test_lift_01():
a = A()
a.x = add(1, 2)
a.y = sub(9, 11)
b = f(lift(a))
result = run_single(b)
assert result == 1
def test_prov_01():
reg = serial.base()
a = add(3, 4)
enc = reg.deep_encode(a._workflow.root_node)
dec = reg.deep_decode(enc)
result = run_job(0, dec)
assert result.value == 7
def static_sum(values, limit_n=1000):
"""Example of static sum routine."""
if len(values) < limit_n:
return sum(values)
else:
half = len(values) // 2
return add(
static_sum(values[:half], limit_n),
static_sum(values[half:], limit_n))
def dynamic_sum(values, limit_n=1000, acc=0, depth=4):
"""Example of dynamic sum."""
if len(values) < limit_n:
return acc + sum(values)
if depth > 0:
half = len(values) // 2
return add(
dynamic_sum(values[:half], limit_n, acc, depth=depth-1),
dynamic_sum(values[half:], limit_n, 0, depth=depth-1))
return dynamic_sum(values[limit_n:], limit_n,
acc + sum(values[:limit_n]), depth)
def test_get_intermediate_results():
r1 = add(1, 1)
r2 = sub(3, r1)
def foo(a, b, c):
return mul(add(a, b), c)
r3 = [foo(i, r2, r1) for i in range(6)]
r4 = accumulate(noodles.gather_all(r3))
run_single(r4)
assert noodles.result(r1) == 2
assert noodles.result(r2) == 1
assert [noodles.result(r) for r in r3] == [2, 4, 6, 8, 10, 12]
assert noodles.result(r4) == 42
def test_attaching():
db = JobDB(':memory:', registry=serial.base)
wf = add(1, 1)
job = Job(wf._workflow, wf._workflow.root)
key1, node1 = db.register(job)
msg, value = db.add_job_to_db(key1, node1)
assert msg == 'initialized'
key2, node2 = db.register(job)
msg, value = db.add_job_to_db(key2, node2)
assert msg == 'attached'
duplicates = db.store_result_in_db(ResultMessage(key1, 'done', 2, None))
assert duplicates == (key2, )
key3, node3 = db.register(job)
msg, result = db.add_job_to_db(key3, node3)
assert msg == 'retrieved'
assert result.value == 2
def test_global_prov():
a = add(4, 5)
b = sub(a, 3)
c = mul(b, 7)
d = mul(b, 7)
e = sub(c, 3)
f = sub(c, 5)
set_global_provenance(c._workflow, base())
assert c._workflow.prov is not None
assert b._workflow.prov is not None
assert c._workflow.prov != b._workflow.prov
set_global_provenance(d._workflow, base())
set_global_provenance(e._workflow, base())
set_global_provenance(f._workflow, base())
assert c._workflow.prov == d._workflow.prov
assert b._workflow.prov != e._workflow.prov
assert f._workflow.prov != e._workflow.prov
def test_global_prov():
a = add(4, 5)
b = sub(a, 3)
c = mul(b, 7)
d = mul(b, 7)
e = sub(c, 3)
f = sub(c, 5)
set_global_provenance(c._workflow, base())
assert c._workflow.prov is not None
assert b._workflow.prov is not None
assert c._workflow.prov != b._workflow.prov
set_global_provenance(d._workflow, base())
set_global_provenance(e._workflow, base())
set_global_provenance(f._workflow, base())
assert c._workflow.prov == d._workflow.prov
assert b._workflow.prov != e._workflow.prov
assert f._workflow.prov != e._workflow.prov
def test_worker_one_batch_job(xenon_server, tmpdir):
tmpdir = Path(str(tmpdir))
infile = tmpdir / 'infile.json'
outfile = tmpdir / 'outfile.json'
wf = add(1, 1)
job = next(iter(wf._workflow.nodes.values()))
job_message = JobMessage(42, job)
registry = serial.base()
print(registry.to_json(job_message), file=infile.open('w'))
m = Machine(scheduler_adaptor='local')
scheduler = m.scheduler
job_config = XenonJobConfig()
executable, arguments = job_config.command_line()
job_description = xenon.JobDescription(executable=str(executable),
arguments=arguments,
stdin=str(infile),
stdout=str(outfile))
job = scheduler.submit_batch_job(job_description)
scheduler.wait_until_done(job)
result_json = [line.strip() for line in outfile.open()]
assert len(result_json) == 1
result = registry.from_json(result_json[0])
assert result.status == 'done'
assert result.key == 42
assert result.msg is None
assert result.value == 2
scheduler.close()
def test_gather_all():
d = noodles.gather_all(add(x, 2*x) for x in range(10))
result = run_single(d)
assert result == list(range(0, 30, 3))
def test_truthfulness():
w = sub(4, add(3, 2))
a = cond(True, w, should_not_run())
b = cond(False, should_not_run(), w)
result = run_single(mul(a, b))
assert result == 1
from noodles.tutorial import (add, sub, mul)
from noodles.draw_workflow import draw_workflow
u = add(5, 4)
v = sub(u, 3)
w = sub(u, 2)
x = mul(v, w)
draw_workflow("callgraph-a.pdf", x._workflow)
def lift_ordered_dict():
x = OrderedDict()
x['a'] = 1
x['b'] = add(1, 2)
return g(noodles.lift(x))
def test_gather():
d = noodles.gather(*[add(x, x) for x in range(10)])
result = run_single(d)
assert result == list(range(0, 20, 2))
def foo(a, b, c):
return mul(add(a, b), c)
def fibonacci(n):
if n < 2:
return 1
else:
return add(fibonacci(n - 1), fibonacci(n - 2))
def test_unpack_00():
a, b, c = unpack(f(), 3)
d = run_single(add(a, add(b, c)))
assert d == 6
def test_42():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
return accumulate(noodles.gather(*multiples))
def truthfulness():
w = sub(4, add(3, 2))
a = cond(True, w, should_not_run())
b = cond(False, should_not_run(), w)
return mul(a, b)
def lift_object():
a = A()
a.x = add(1, 2)
a.y = sub(9, 11)
return f(noodles.lift(a))
def unpack():
a, b, c = noodles.unpack(f(), 3)
return add(a, add(b, c))
from noodles.tutorial import (add, sub, mul)
from draw_workflow import draw_workflow
u = add(5, 4)
v = sub(u, 3)
w = sub(u, 2)
x = mul(v, w)
draw_workflow("callgraph-a.pdf", x._workflow)
def fib(n):
if n < 2:
return 1
else:
return add(fib(n - 1), fib(n - 2))
from noodles import (schedule, run_single)
from noodles.tutorial import (add)
from noodles.draw_workflow import draw_workflow
@schedule
class A:
def __init__(self, value):
self.value = value
@property
def square(self):
return self.value**2
@square.setter
def square(self, sqr):
self.value = sqr**(1 / 2)
u = A(3)
v = A(4)
u.square = add(u.square, v.square)
draw_workflow("pythagoras.pdf", u.value._workflow)
print("⎷(3² + 4²) = ", run_single(u.value))
def gather():
return noodles.gather(*[add(x, x) for x in range(10)])
def gather_dict():
return noodles.gather_dict(a=1, b=add(1, 1), c=add(2, 3))
def test_gather_all():
return noodles.gather_all(add(x, 2 * x) for x in range(10))
def test_truthfulness():
w = sub(4, add(3, 2))
a = cond(True, w, should_not_run())
b = cond(False, should_not_run(), w)
result = run_single(mul(a, b))
assert result == 1
def test_gather_dict():
d = noodles.gather_dict(a=1, b=add(1, 1), c=add(2, 3))
result = run_single(d)
assert result == {'a': 1, 'b': 2, 'c': 5}
