def test_job_description_attributes():
# attributes serialization
attrs = {'j1_name': 'j1', 'j1_var1': 'var1'}
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
attributes=attrs)
assert all((len(j.attributes) == len(attrs), j.attributes == attrs))
j_json = j.to_json()
j_clone = Job(**json.loads(j_json))
assert all(
(len(j_clone.attributes) == len(attrs), j_clone.attributes == attrs))
assert j.to_dict() == j_clone.to_dict()
# attributes wrong format
with pytest.raises(IllegalJobDescription):
Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
attributes="some_illegal_attributes")
with pytest.raises(IllegalJobDescription):
Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
attributes=["some_illegal_attributes", "more_illegal_attributes"])
def test_job_description_serialization():
# gpu cr
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date", "stdin": "in_file", "stdout": "out_file", "stderr": "err_file",
"modules": [ "python/3.6" ], "venv": [ "venv-3.6" ] },
"resources": { "numCores": { "min": 2, "max": 4}, "numNodes": { "min": 1, "max": 2}, "nodeCrs": { "gpu": 1 } } }"""
job = Job(**json.loads(jobd))
assert job, "Job with node consumable resources (gpu)"
job_dict = job.to_dict()
job_clone = json.loads(job.to_json())
assert job_dict == job_clone
def test_job_description_iterations():
# no iterations
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert not job.has_iterations
# valid iterations
jobd = """{ "name": "job1",
"iteration": { "start": 0, "stop": 10 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert all((job.has_iterations, job.iteration.iterations() == 10))
assert all(job.iteration.in_range(i) for i in range(10))
assert all(not job.iteration.in_range(i) for i in range(10, 20))
# valid iterations with default start
jobd = """{ "name": "job1",
"iteration": { "stop": 10 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert all((job.has_iterations, job.iteration.iterations() == 10))
assert all(job.iteration.in_range(i) for i in range(10))
assert all(not job.iteration.in_range(i) for i in range(10, 20))
# valid iterations
jobd = """{ "name": "job1",
"iteration": { "start": 5, "stop": 10 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert all((job.has_iterations, job.iteration.iterations() == 5))
assert all(job.iteration.in_range(i) for i in range(5, 10))
assert all(not job.iteration.in_range(i) for i in range(0, 5))
# not valid iteration type
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"iteration": [ 5, 10 ],
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# not valid iteration spec
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"iteration": { },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# not valid iteration spec
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"iteration": { "iterations": 5 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# not valid iteration spec
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"iteration": { "start": 5, "stop": 0, "step": 2 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# wrong iteration range
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"iteration": { "start": 5, "stop": 0 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# wrong iteration range
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"iteration": { "start": 5, "stop": 5 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# json serialization
jit = JobIteration(start=0, stop=10)
jit_json = jit.to_json()
jit_clone = JobIteration(**json.loads(jit_json))
jit.to_dict() == jit_clone.to_dict()
# string serialization
assert str(jit) == '{}-{}'.format(0, 10)
# serialization
jobd = """{ "name": "job1",
"iteration": { "start": 5, "stop": 10 },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert job, "Job with dependencies"
job_dict = job.to_dict()
job_clone = json.loads(job.to_json())
assert job_dict == job_clone
def test_job_description_dependencies():
# simple dependencies
jobd = """{ "name": "job1",
"dependencies": { "after": [ "job2" ] },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert all((len(job.dependencies.after) == 1, 'job2'
in job.dependencies.after))
# valid dependencies
jobd = """{ "name": "job1",
"dependencies": { "after": [ "job2", "job3" ] },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert all((len(job.dependencies.after) == 2, 'job2'
in job.dependencies.after, 'job3' in job.dependencies.after))
jobd = """{ "name": "job1",
"dependencies": { "after": [ "job2", "job3" ] },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert job.has_dependencies
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert not job.has_dependencies
jobd = """{ "name": "job1",
"dependencies": { "after": [ ] },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert not job.has_dependencies
# dependencies not as list
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"dependencies": { "after": "job2" },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# wrong keyword in job dependencies
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"dependencies": { "whenever": "job2" },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# wrong type of job dependencies
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"dependencies": { "after": { "job": "job2" } },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# wrong elements of job dependencies
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"dependencies": { "after": [ "job2", [ "job3", "job4" ] ] },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# wrong type of job dependencies
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"dependencies": [ { "after": [ "job2", [ "job3", "job4" ] ] } ],
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
Job(**json.loads(jobd))
# json serialization
jdep = JobDependencies(after=["job2", "job3"])
jdep_json = jdep.to_json()
assert all((jdep.has_dependencies, len(jdep.after) == 2, 'job2'
in jdep.after, 'job3' in jdep.after))
print("job dependencies as json: {}".format(jdep_json))
jdep_clone = JobDependencies(**json.loads(jdep_json))
jdep.to_dict() == jdep_clone.to_dict()
# serialization
jobd = """{ "name": "job1",
"dependencies": { "after": [ "job2", "job3" ] },
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert job, "Job with dependencies"
job_dict = job.to_dict()
job_clone = json.loads(job.to_json())
assert job_dict == job_clone