def test_worker_one_online_job(xenon_server):
wf = mul(6, 7)
job = next(iter(wf._workflow.nodes.values()))
job_message = JobMessage(1234, job)
registry = serial.base()
msg = registry.to_json(job_message)
m = Machine(scheduler_adaptor='local')
scheduler = m.scheduler
job_config = XenonJobConfig()
executable, arguments = job_config.command_line()
xjob_description = xenon.JobDescription(executable=str(executable),
arguments=arguments)
xjob, xstdout = scheduler.submit_interactive_job(xjob_description,
[msg.encode()])
scheduler.wait_until_done(xjob)
result_json = ''.join(m.stdout.decode() for m in xstdout if m.stdout)
assert len(result_json) > 0
result = registry.from_json(result_json)
assert result.status == 'done'
assert result.key == 1234
assert result.msg is None
assert result.value == 42
scheduler.close()
def test_xenon_42_multi():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine(
scheduler_adaptor='slurm',
location='ssh://fs0.das5.cs.vu.nl/home/jhidding',
credential=xenon.CertificateCredential(
username='******', certfile='/home/johannes/.ssh/id_rsa'),
jobs_properties={
'xenon.adaptors.schedulers.ssh.strictHostKeyChecking': 'false'
})
worker_config = XenonJobConfig(
prefix=Path('/home/jhidding/.local/share/workon/mcfly'),
working_dir='/home/jhidding/',
time_out=1000000000000,
verbose=False) # , options=['-C', 'TitanX', '--gres=gpu:1'])
result = run_xenon(C,
machine=machine,
worker_config=worker_config,
n_processes=2)
print("The answer is:", result)
def test_broker_02():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_single(C) == 42
def test_broker_01():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_parallel(C, 4) == 42
def test_broker_02():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_single(C) == 42
def test_broker_01():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
assert run_parallel(C, 4) == 42
def test_xenon_42():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
result = run_parallel_timing(C, 4, sys.stdout)
assert (result == 42)
def test_xenon_42():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
result = run_parallel_timing(C, 4, sys.stdout)
assert(result == 42)
def test_broker_logging():
A = log_add(1, 1)
B = sub(3, A)
multiples = [mul(log_add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
with NCDisplay(title="Running the test") as display:
assert run_parallel_with_display(C, 4, display) == 42
def test_broker_logging():
A = log_add(1, 1)
B = sub(3, A)
multiples = [mul(log_add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
with NCDisplay(title="Running the test") as display:
assert run_parallel_with_display(C, 4, display) == 42
def test_prov_03():
db_file = "prov2.json"
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(gather(*multiples))
result = run_parallel(C, 4, serial.base, db_file, JobKeeper(keep=True))
assert result == 42
os.unlink(db_file)
def test_prov_02():
db_file = "prov1.json"
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = accumulate(gather(*multiples))
result = run_single(C, serial.base, db_file)
assert result == 42
os.unlink(db_file)
def test_global_prov():
a = add(4, 5)
b = sub(a, 3)
c = mul(b, 7)
d = mul(b, 7)
e = sub(c, 3)
f = sub(c, 5)
set_global_provenance(c._workflow, base())
assert c._workflow.prov is not None
assert b._workflow.prov is not None
assert c._workflow.prov != b._workflow.prov
set_global_provenance(d._workflow, base())
set_global_provenance(e._workflow, base())
set_global_provenance(f._workflow, base())
assert c._workflow.prov == d._workflow.prov
assert b._workflow.prov != e._workflow.prov
assert f._workflow.prov != e._workflow.prov
def test_global_prov():
a = add(4, 5)
b = sub(a, 3)
c = mul(b, 7)
d = mul(b, 7)
e = sub(c, 3)
f = sub(c, 5)
set_global_provenance(c._workflow, base())
assert c._workflow.prov is not None
assert b._workflow.prov is not None
assert c._workflow.prov != b._workflow.prov
set_global_provenance(d._workflow, base())
set_global_provenance(e._workflow, base())
set_global_provenance(f._workflow, base())
assert c._workflow.prov == d._workflow.prov
assert b._workflow.prov != e._workflow.prov
assert f._workflow.prov != e._workflow.prov
def test_prov_04():
db_file = "prov3.json"
A = add2(1, 1)
B = sub(3, A)
multiples = [mul(add2(i, B), A) for i in range(6)]
C = accumulate(gather(*multiples))
result = run_parallel_opt(C, 4, serial.base, db_file)
assert result == 42
os.unlink(db_file)
def test_xenon_42_simple(xenon_server):
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine()
worker_config = XenonJobConfig(verbose=True)
result = run_xenon_simple(C, machine=machine, worker_config=worker_config)
assert (result == 42)
def test_xenon_42_multi(xenon_server):
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine()
worker_config = XenonJobConfig(queue_name='multi', verbose=True)
result = run_xenon(C,
machine=machine,
worker_config=worker_config,
n_processes=2)
assert (result == 42)
def test_worker_ten_online_jobs(xenon_server):
registry = serial.base()
def single_job(wf):
job = next(iter(wf._workflow.nodes.values()))
job_message = JobMessage(0, job)
return (registry.to_json(job_message) + '\n').encode()
m = Machine(scheduler_adaptor='local')
scheduler = m.scheduler
job_config = XenonJobConfig(verbose=True)
executable, arguments = job_config.command_line()
xjob_description = xenon.JobDescription(executable=str(executable),
arguments=arguments)
xjob, xstdout = scheduler.submit_interactive_job(
xjob_description, [single_job(mul(10, i)) for i in range(10)])
scheduler.wait_until_done(xjob)
result_json = ""
for m in xstdout:
if m.stdout:
result_json += m.stdout.decode()
if m.stderr:
for l in m.stderr.decode().splitlines():
print("remote:", l)
results = [registry.from_json(r) for r in result_json.splitlines()]
print("results: ", end='')
for r in results:
print(r.value, end=' ')
print()
assert len(results) == 10
for i, result in enumerate(results):
assert result.status == 'done'
assert result.key == 0
assert result.msg is None
assert result.value == i * 10
scheduler.close()
def test_xenon_42():
A = log_add(1, 1)
B = sub(3, A)
multiples = [mul(log_add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
xenon_config = XenonConfig(jobs_scheme='local')
job_config = RemoteJobConfig(registry=serial.base, time_out=1000)
with XenonKeeper() as Xe, NCDisplay() as display:
result = run_xenon(C,
Xe,
"cache.json",
2,
xenon_config,
job_config,
display=display)
assert (result == 42)
def test_xenon_42():
A = log_add(1, 1)
B = sub(3, A)
multiples = [mul(log_add(i, B), A) for i in range(6)]
C = accumulate(noodles.gather(*multiples))
xenon_config = XenonConfig(
jobs_scheme='local'
)
job_config = RemoteJobConfig(
registry=serial.base,
time_out=1000
)
with XenonKeeper() as Xe, NCDisplay() as display:
result = run_xenon(
C, Xe, "cache.json", 2, xenon_config, job_config,
display=display)
assert(result == 42)
def test_truthfulness():
w = sub(4, add(3, 2))
a = cond(True, w, should_not_run())
b = cond(False, should_not_run(), w)
result = run_single(mul(a, b))
assert result == 1
from noodles.tutorial import (add, sub, mul)
from draw_workflow import draw_workflow
u = add(5, 4)
v = sub(u, 3)
w = sub(u, 2)
x = mul(v, w)
draw_workflow("callgraph-a.pdf", x._workflow)
from noodles.tutorial import (add, sub, mul)
from noodles.draw_workflow import draw_workflow
u = add(5, 4)
v = sub(u, 3)
w = sub(u, 2)
x = mul(v, w)
draw_workflow("callgraph-a.pdf", x._workflow)
def test_42():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
return accumulate(noodles.gather(*multiples))
def truthfulness():
w = sub(4, add(3, 2))
a = cond(True, w, should_not_run())
b = cond(False, should_not_run(), w)
return mul(a, b)
def test_truthfulness():
w = sub(4, add(3, 2))
a = cond(True, w, should_not_run())
b = cond(False, should_not_run(), w)
result = run_single(mul(a, b))
assert result == 1
def foo(a, b, c):
return mul(add(a, b), c)
