def test_worker_one_online_job(xenon_server):
wf = mul(6, 7)
job = next(iter(wf._workflow.nodes.values()))
job_message = JobMessage(1234, job)
registry = serial.base()
msg = registry.to_json(job_message)
m = Machine(scheduler_adaptor='local')
scheduler = m.scheduler
job_config = XenonJobConfig()
executable, arguments = job_config.command_line()
xjob_description = xenon.JobDescription(executable=str(executable),
arguments=arguments)
xjob, xstdout = scheduler.submit_interactive_job(xjob_description,
[msg.encode()])
scheduler.wait_until_done(xjob)
result_json = ''.join(m.stdout.decode() for m in xstdout if m.stdout)
assert len(result_json) > 0
result = registry.from_json(result_json)
assert result.status == 'done'
assert result.key == 1234
assert result.msg is None
assert result.value == 42
scheduler.close()
def registry():
"""
This function pass to the noodles infrascture all the information
related to the Structure of the Package object that is schedule.
This *Registry* class contains hints that help Noodles to encode
and decode this Package object.
"""
return Registry(parent=serial.base() + arrays_to_hdf5())
def test_path():
a = files.Path("/usr/bin/python")
registry = serial.base() + files.registry()
encoded = registry.deep_encode(a)
b = registry.deep_decode(encoded)
assert a.path == b.path
def test_path():
a = Path("/usr/bin/python")
registry = serial.base()
encoded = registry.deep_encode(a)
b = registry.deep_decode(encoded)
assert a == b
def test_prov_01():
reg = serial.base()
a = add(3, 4)
enc = reg.deep_encode(a._workflow.root_node)
dec = reg.deep_decode(enc)
result = run_job(0, dec)
assert result.value == 7
def test_global_prov():
a = add(4, 5)
b = sub(a, 3)
c = mul(b, 7)
d = mul(b, 7)
e = sub(c, 3)
f = sub(c, 5)
set_global_provenance(c._workflow, base())
assert c._workflow.prov is not None
assert b._workflow.prov is not None
assert c._workflow.prov != b._workflow.prov
set_global_provenance(d._workflow, base())
set_global_provenance(e._workflow, base())
set_global_provenance(f._workflow, base())
assert c._workflow.prov == d._workflow.prov
assert b._workflow.prov != e._workflow.prov
assert f._workflow.prov != e._workflow.prov
def test_prov_00():
reg = serial.base()
a = add(3, 4)
b = sub(3, 4)
c = add(3, 4)
d = add(4, 3)
enc = [reg.deep_encode(x._workflow.root_node) for x in [a, b, c, d]]
key = [prov_key(o) for o in enc]
assert key[0] == key[2]
assert key[1] != key[0]
assert key[3] != key[0]
def test_global_prov():
a = add(4, 5)
b = sub(a, 3)
c = mul(b, 7)
d = mul(b, 7)
e = sub(c, 3)
f = sub(c, 5)
set_global_provenance(c._workflow, base())
assert c._workflow.prov is not None
assert b._workflow.prov is not None
assert c._workflow.prov != b._workflow.prov
set_global_provenance(d._workflow, base())
set_global_provenance(e._workflow, base())
set_global_provenance(f._workflow, base())
assert c._workflow.prov == d._workflow.prov
assert b._workflow.prov != e._workflow.prov
assert f._workflow.prov != e._workflow.prov
def registry():
return Registry(
parent=base() + numpy_registry(),
types={
DataFrame: SerStandardObject(
DataFrame, ['data', 'units', 'comments']),
DCS: SerStandardObject(
DCS, ['energy', 'angle', 'cs'])
},
hooks={
'<quantity>': SerQuantity('<quantity>'),
'<unit>': SerUnit('<unit>')
},
hook_fn=quantity_hook)
def registry():
"""
This function pass to the noodles infrascture all the information
related to the Structure of the Package object that is schedule.
This *Registry* class contains hints that help Noodles to encode
and decode this Package object.
"""
return Registry(parent=serial.base() + arrays_to_hdf5(),
types={
Package: AsDict(Package),
Path: SerPath(),
plams.Molecule: SerMolecule(),
Chem.Mol: SerMol(),
Result: SerStorable(Result),
Settings: SerSettings()
})
def test_worker_ten_online_jobs(xenon_server):
registry = serial.base()
def single_job(wf):
job = next(iter(wf._workflow.nodes.values()))
job_message = JobMessage(0, job)
return (registry.to_json(job_message) + '\n').encode()
m = Machine(scheduler_adaptor='local')
scheduler = m.scheduler
job_config = XenonJobConfig(verbose=True)
executable, arguments = job_config.command_line()
xjob_description = xenon.JobDescription(executable=str(executable),
arguments=arguments)
xjob, xstdout = scheduler.submit_interactive_job(
xjob_description, [single_job(mul(10, i)) for i in range(10)])
scheduler.wait_until_done(xjob)
result_json = ""
for m in xstdout:
if m.stdout:
result_json += m.stdout.decode()
if m.stderr:
for l in m.stderr.decode().splitlines():
print("remote:", l)
results = [registry.from_json(r) for r in result_json.splitlines()]
print("results: ", end='')
for r in results:
print(r.value, end=' ')
print()
assert len(results) == 10
for i, result in enumerate(results):
assert result.status == 'done'
assert result.key == 0
assert result.msg is None
assert result.value == i * 10
scheduler.close()
def registry():
"""
This function pass to the noodles infrascture all the information
related to the Structure of the Package object that is schedule.
This *Registry* class contains hints that help Noodles to encode
and decode this Package object.
"""
return Registry(
parent=serial.base() + arrays_to_hdf5(),
types={
Package: AsDict(Package),
Path: SerPath(),
plams.Molecule: SerMolecule(),
Chem.Mol: SerMol(),
Result: AsDict(Result),
Settings: SerSettings(),
plams.KFFile: SerReasonableObject(plams.KFFile),
plams.KFReader: SerReasonableObject(plams.KFReader),
np.floating: SerNumpyScalar(),
np.integer: SerNumpyScalar()
}
)
def test_worker_one_batch_job(xenon_server, tmpdir):
tmpdir = Path(str(tmpdir))
infile = tmpdir / 'infile.json'
outfile = tmpdir / 'outfile.json'
wf = add(1, 1)
job = next(iter(wf._workflow.nodes.values()))
job_message = JobMessage(42, job)
registry = serial.base()
print(registry.to_json(job_message), file=infile.open('w'))
m = Machine(scheduler_adaptor='local')
scheduler = m.scheduler
job_config = XenonJobConfig()
executable, arguments = job_config.command_line()
job_description = xenon.JobDescription(executable=str(executable),
arguments=arguments,
stdin=str(infile),
stdout=str(outfile))
job = scheduler.submit_batch_job(job_description)
scheduler.wait_until_done(job)
result_json = [line.strip() for line in outfile.open()]
assert len(result_json) == 1
result = registry.from_json(result_json[0])
assert result.status == 'done'
assert result.key == 42
assert result.msg is None
assert result.value == 2
scheduler.close()
def registry():
"""Serialisation registry for matrix testing backends."""
return serial.base() + arrays_to_string()
def registry2():
return serial.base() + serial.namedtuple()
def registry():
return serial.pickle() + serial.base()
def registry():
return serial.pickle() + serial.base()
def registry1():
return Registry(
parent=serial.base(),
types={
A: SerNamedTuple(A)
})
def registry():
return serial.base() + serial.numpy()
def registry():
return serial.base()
def registry():
reg = Registry(parent=serial.base())
reg[A] = AsDict(A)
return reg
def registry2():
return serial.base() + serial.namedtuple()
workdir = plams.config.get('default_jobmanager').workdir
except AttributeError as ex:
raise plams.PlamsError(
"Failed to initialize the PLAMS jobmanager") from ex
db_file = join(workdir, 'cache.db')
return run_parallel(wf,
n_threads=n_processes,
registry=registry,
db_file=db_file,
always_cache=always_cache,
echo_log=False)
#: A :class:`Registry` instance to-be returned by :func:`registry`.
REGISTRY: Registry = Registry(parent=serial.base() + arrays_to_hdf5(),
types={
Package:
SerReduce(Package),
Path:
SerPath(),
plams.Molecule:
SerMolecule(),
Chem.Mol:
SerMol(),
Result:
AsDict(Result),
Settings:
SerSettings(),
plams.KFFile:
SerReasonableObject(plams.KFFile),
def registry():
return serial.base() + arrays_to_hdf5()
def registry1():
return Registry(parent=serial.base(), types={A: SerNamedTuple(A)})
def registry():
return serial.base() + arrays_to_hdf5()