def test_job_description_attributes():
# attributes serialization
attrs = {'j1_name': 'j1', 'j1_var1': 'var1'}
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
attributes=attrs)
assert all((len(j.attributes) == len(attrs), j.attributes == attrs))
j_json = j.to_json()
j_clone = Job(**json.loads(j_json))
assert all(
(len(j_clone.attributes) == len(attrs), j_clone.attributes == attrs))
assert j.to_dict() == j_clone.to_dict()
# attributes wrong format
with pytest.raises(IllegalJobDescription):
Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
attributes="some_illegal_attributes")
with pytest.raises(IllegalJobDescription):
Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
attributes=["some_illegal_attributes", "more_illegal_attributes"])
def test_local_error_duplicate_name_job_separate_reqs(tmpdir):
file_path = tmpdir.join('jobs.json')
print('tmpdir: {}'.format(str(tmpdir)))
jobName = 'mdate'
jobs1 = [
job.to_dict() for job in [
Job(
jobName,
JobExecution('date',
wd=abspath(tmpdir.join('date.sandbox')),
stdout='date.out',
stderr='date.err'),
JobResources(numCores=ResourceSize(1)))
]
]
jobs2 = [
job.to_dict() for job in [
Job(
jobName,
JobExecution('sleep',
wd=abspath(tmpdir.join('sleep.sandbox')),
stdout='sleep.out',
stderr='sleep.err'),
JobResources(numCores=ResourceSize(1)))
]
]
reqs = [{
'request': 'submit',
'jobs': jobs1
}, {
'request': 'submit',
'jobs': jobs2
}, {
'request': 'control',
'command': 'finishAfterAllTasksDone'
}]
save_reqs_to_file(reqs, file_path)
print('jobs saved to file_path: {}'.format(str(file_path)))
sys.argv = [
'QCG-PilotJob', '--file', '--file-path',
str(file_path), '--nodes', '2', '--wd',
str(tmpdir), '--report-format', 'json'
]
QCGPMService().start()
# the first job (date) should execute
check_job_status_in_json([jobName],
workdir=str(tmpdir),
dest_state='SUCCEED')
assert all((isdir(abspath(tmpdir.join('date.sandbox'))),
exists(join(abspath(tmpdir.join('date.sandbox')), 'date.out')),
exists(join(abspath(tmpdir.join('date.sandbox')),
'date.err'))))
# the second job (sleep) due to the name clash should not execute
assert not isdir(abspath(tmpdir.join('sleep.sandbox')))
def test_joblist():
jlist = JobList()
# adding and removing jobs from list
jlist.add(
Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1)))
assert jlist.exist('j1')
assert jlist.get('j1').get_name() == 'j1'
jlist.add(
Job(name='j2',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1)))
assert all((jlist.exist('j1'), jlist.exist('j2')))
assert jlist.get('j2').get_name() == 'j2'
with pytest.raises(JobAlreadyExist):
jlist.add(
Job(name='j2',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1)))
with pytest.raises(JobAlreadyExist):
jlist.add(
Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1)))
jnames = jlist.jobs()
assert all((len(jnames) == 2, 'j1' in jnames, 'j2' in jnames))
jlist.remove('j2')
assert not jlist.exist('j2')
assert jlist.get('j2') is None
jnames = jlist.jobs()
assert all((len(jnames) == 1, 'j1' in jnames))
# try to add something which is not a job
with pytest.raises(Exception):
jlist.add('another job')
# parsing job iteration names
assert JobList.parse_jobname('j1') == ('j1', None)
assert JobList.parse_jobname('j1:1') == ('j1', 1)
with pytest.raises(ValueError):
assert JobList.parse_jobname('j1:2:1') == ('j1', '2:1')
with pytest.raises(ValueError):
assert JobList.parse_jobname('j1:') == ('j1', '')
def test_jobdescription_jobname():
# job name ok
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1))
assert j
# missing job name
with pytest.raises(IllegalJobDescription):
Job(name=None,
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1))
# illegal character in job name
with pytest.raises(IllegalJobDescription):
Job(name='j1:1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1))
def test_local_error_duplicate_name_job(tmpdir):
file_path = tmpdir.join('jobs.json')
print('tmpdir: {}'.format(str(tmpdir)))
jobName = 'mdate'
jobs = [
job.to_dict() for job in [
Job(
jobName,
JobExecution('date',
wd=abspath(tmpdir.join('date.sandbox')),
stdout='date.out',
stderr='date.err'),
JobResources(numCores=ResourceSize(1))),
Job(
jobName,
JobExecution('sleep',
wd=abspath(tmpdir.join('sleep.sandbox')),
stdout='sleep.out',
stderr='sleep.err'),
JobResources(numCores=ResourceSize(1)))
]
]
reqs = [{
'request': 'submit',
'jobs': jobs
}, {
'request': 'control',
'command': 'finishAfterAllTasksDone'
}]
save_reqs_to_file(reqs, file_path)
print('jobs saved to file_path: {}'.format(str(file_path)))
sys.argv = [
'QCG-PilotJob', '--file', '--file-path',
str(file_path), '--nodes', '2', '--wd',
str(tmpdir), '--report-format', 'json'
]
QCGPMService().start()
# no job should be executed due to the failed submit request with non-unique jobs inside
assert not isdir(abspath(tmpdir.join('date.sandbox')))
assert not isdir(abspath(tmpdir.join('sleep.sandbox')))
def test_local_error_job_desc():
# missing job execution
with pytest.raises(IllegalJobDescription):
Job('error_job',
JobExecution(None, stdout='date.out', stderr='date.err'),
JobResources(numCores=ResourceSize(1)))
# wrong format of arguments
with pytest.raises(IllegalJobDescription):
Job(
'error_job',
JobExecution('date',
args='this should be a list',
stdout='date.out',
stderr='date.err'),
JobResources(numCores=ResourceSize(1)))
# wrong format of environment
with pytest.raises(IllegalJobDescription):
Job(
'error_job',
JobExecution('date',
args=['arg1'],
env=['this shuld be a dict'],
stdout='date.out',
stderr='date.err'),
JobResources(numCores=ResourceSize(1)))
# missing execution definition
with pytest.raises(IllegalJobDescription):
Job('error_job', None, JobResources(numCores=ResourceSize(1)))
# missing resources definition
with pytest.raises(IllegalJobDescription):
Job(
'error_job',
JobExecution('date',
args=['arg1'],
env=['this shuld be a dict'],
stdout='date.out',
stderr='date.err'), None)
def test_local_simple_script_job(tmpdir):
file_path = tmpdir.join('jobs.json')
print('tmpdir: {}'.format(str(tmpdir)))
jobName = 'mdate_script'
jobs = [
job.to_dict() for job in [
Job(
jobName,
JobExecution(script='/bin/date\n/bin/hostname\n',
wd=abspath(tmpdir.join('date.sandbox')),
stdout='date.out',
stderr='date.err'),
JobResources(numCores=ResourceSize(1)))
]
]
reqs = [{
'request': 'submit',
'jobs': jobs
}, {
'request': 'control',
'command': 'finishAfterAllTasksDone'
}]
save_reqs_to_file(reqs, file_path)
print('jobs saved to file_path: {}'.format(str(file_path)))
sys.argv = [
'QCG-PilotJob', '--log', 'debug', '--file', '--file-path',
str(file_path), '--nodes', '2', '--wd',
str(tmpdir), '--report-format', 'json'
]
QCGPMService().start()
check_job_status_in_json([jobName],
workdir=str(tmpdir),
dest_state='SUCCEED')
assert all(
(isdir(abspath(tmpdir.join('date.sandbox'))),
exists(join(abspath(tmpdir.join('date.sandbox')), 'date.out')),
exists(join(abspath(tmpdir.join('date.sandbox')), 'date.err')),
stat(join(abspath(tmpdir.join('date.sandbox')), 'date.out')).st_size >
0, stat(join(abspath(tmpdir.join('date.sandbox')),
'date.err')).st_size == 0))
with pytest.raises(ValueError):
check_job_status_in_json([jobName + 'xxx'],
workdir=str(tmpdir),
dest_state='SUCCEED')
def test_local_workflows_error(tmpdir):
file_path = tmpdir.join('jobs.json')
print('tmpdir: {}'.format(str(tmpdir)))
jobs = [
job.to_dict() for job in [
Job('first',
JobExecution('sleep',
args=['2s'],
wd=abspath(tmpdir.join('first.sandbox')),
stdout='out',
stderr='err'),
JobResources(numCores=ResourceSize(1)),
dependencies=JobDependencies(after=['not-existing']))
]
]
reqs = [{
'request': 'submit',
'jobs': jobs
}, {
'request': 'control',
'command': 'finishAfterAllTasksDone'
}]
save_reqs_to_file(reqs, file_path)
print('jobs saved to file_path: {}'.format(str(file_path)))
sys.argv = [
'QCG-PilotJob', '--file', '--file-path',
str(file_path), '--nodes', '2', '--wd',
str(tmpdir), '--report-format', 'json'
]
QCGPMService().start()
assert not exists(abspath(tmpdir.join('first.sandbox')))
rmtree(str(tmpdir))
def test_local_workflows(tmpdir):
file_path = tmpdir.join('jobs.json')
print('tmpdir: {}'.format(str(tmpdir)))
jobs = [
job.to_dict() for job in [
Job(
'first',
JobExecution('sleep',
args=['2s'],
wd=abspath(tmpdir.join('first.sandbox')),
stdout='out',
stderr='err'),
JobResources(numCores=ResourceSize(1))),
Job('second',
JobExecution('sleep',
args=['1s'],
wd=abspath(tmpdir.join('second.sandbox')),
stdout='out',
stderr='err'),
JobResources(numCores=ResourceSize(1)),
dependencies=JobDependencies(after=['first'])),
Job('third',
JobExecution('date',
wd=abspath(tmpdir.join('third.sandbox')),
stdout='out',
stderr='err'),
JobResources(numCores=ResourceSize(1)),
dependencies=JobDependencies(after=['first', 'second']))
]
]
reqs = [{
'request': 'submit',
'jobs': jobs
}, {
'request': 'control',
'command': 'finishAfterAllTasksDone'
}]
save_reqs_to_file(reqs, file_path)
print('jobs saved to file_path: {}'.format(str(file_path)))
# the ammount of resources should be enough to theoretically start all three job's at once
sys.argv = [
'QCG-PilotJob', '--file', '--file-path',
str(file_path), '--nodes', '4', '--wd',
str(tmpdir), '--report-format', 'json'
]
QCGPMService().start()
jnames = ['first', 'second', 'third']
check_job_status_in_json(jnames, workdir=str(tmpdir), dest_state='SUCCEED')
for jname in jnames:
assert all((isdir(abspath(tmpdir.join('{}.sandbox'.format(jname)))),
exists(
join(abspath(tmpdir.join('{}.sandbox'.format(jname))),
'out')),
exists(
join(abspath(tmpdir.join('{}.sandbox'.format(jname))),
'err'))))
with open(join(find_single_aux_dir(str(tmpdir)), 'jobs.report'), 'r') as f:
job_stats = [json.loads(line) for line in f.readlines()]
assert len(job_stats) == len(jnames)
jstats = {}
for i in range(0, len(jnames)):
job = job_stats[i]
print('readed job stats: {}'.format(str(job)))
t = datetime.strptime(job['runtime']['rtime'], "%H:%M:%S.%f")
rtime = timedelta(hours=t.hour,
minutes=t.minute,
seconds=t.second,
microseconds=t.microsecond)
# find start executing time
exec_state = list(
filter(lambda st_en: st_en['state'] == 'EXECUTING',
job['history']))
assert len(exec_state) == 1
# find finish executing time
finish_state = list(
filter(lambda st_en: st_en['state'] == 'SUCCEED', job['history']))
assert len(finish_state) == 1
start_time = datetime.strptime(exec_state[0]['date'],
'%Y-%m-%dT%H:%M:%S.%f')
finish_time = datetime.strptime(finish_state[0]['date'],
'%Y-%m-%dT%H:%M:%S.%f')
jstats[job['name']] = {
'r_time': rtime,
's_time': start_time,
'f_time': finish_time
}
# assert second job started after the first one
assert jstats['second']['s_time'] > jstats['first']['f_time']
# assert third job started after the first and second ones
assert all((jstats['third']['s_time'] > jstats['first']['f_time'],
jstats['third']['s_time'] > jstats['second']['f_time']))
rmtree(str(tmpdir))
def test_slurmenv_simple_job():
if not in_slurm_allocation() or get_num_slurm_nodes() < 2:
pytest.skip(
'test not run in slurm allocation or allocation is smaller than 2 nodes'
)
resources, allocation = get_slurm_resources_binded()
resources_node_names = set(n.name for n in resources.nodes)
set_pythonpath_to_qcg_module()
tmpdir = str(tempfile.mkdtemp(dir=SHARED_PATH))
file_path = join(tmpdir, 'jobs.json')
print('tmpdir: {}'.format(tmpdir))
jobName = 'mdate'
jobs = [
job.to_dict() for job in [
Job(
jobName,
JobExecution('date',
wd=abspath(join(tmpdir, 'date.sandbox')),
stdout='date.out',
stderr='date.err'),
JobResources(numCores=ResourceSize(1)))
]
]
reqs = [{
'request': 'submit',
'jobs': jobs
}, {
'request': 'control',
'command': 'finishAfterAllTasksDone'
}]
save_reqs_to_file(reqs, file_path)
print('jobs saved to file_path: {}'.format(str(file_path)))
sys.argv = [
'QCG-PilotJob', '--log', 'debug', '--file', '--file-path',
str(file_path), '--wd', tmpdir, '--report-format', 'json'
]
QCGPMService().start()
jobEntries = check_job_status_in_json([jobName],
workdir=tmpdir,
dest_state='SUCCEED')
assert all(
(isdir(abspath(join(tmpdir, 'date.sandbox'))),
exists(join(abspath(join(tmpdir, 'date.sandbox')), 'date.out')),
exists(join(abspath(join(tmpdir, 'date.sandbox')), 'date.err')),
stat(join(abspath(join(tmpdir, 'date.sandbox')), 'date.out')).st_size
> 0))
# there can be some debugging messages in the stderr
# stat(join(abspath(join(tmpdir, 'date.sandbox')), 'date.err')).st_size == 0))
for jname, jentry in jobEntries.items():
assert all(('runtime' in jentry, 'allocation'
in jentry.get('runtime', {})))
jalloc = jentry['runtime']['allocation']
for jalloc_node in jalloc.split(','):
node_name = jalloc_node[:jalloc_node.index('[')]
print('{} in available nodes ({})'.format(
node_name, ','.join(resources_node_names)))
assert node_name in resources_node_names, '{} not in nodes ({}'.format(
node_name, ','.join(resources_node_names))
with pytest.raises(ValueError):
check_job_status_in_json([jobName + 'xxx'],
workdir=tmpdir,
dest_state='SUCCEED')
rmtree(tmpdir)
def test_slurmenv_many_nodes_many_cores():
if not in_slurm_allocation() or get_num_slurm_nodes() < 2:
pytest.skip(
'test not run in slurm allocation or allocation is smaller than 2 nodes'
)
resources, allocation = get_slurm_resources_binded()
resources_node_names = set(n.name for n in resources.nodes)
set_pythonpath_to_qcg_module()
tmpdir = str(tempfile.mkdtemp(dir=SHARED_PATH))
file_path = join(tmpdir, 'jobs.json')
print('tmpdir: {}'.format(tmpdir))
jobName = 'hostname'
jobwdir_base = 'hostname.sandbox'
cores_num = resources.nodes[0].free
nodes_num = resources.total_nodes
jobs = [
job.to_dict() for job in [
Job(
jobName,
JobExecution(exec='mpirun',
args=['--allow-run-as-root', 'hostname'],
wd=abspath(join(tmpdir, jobwdir_base)),
stdout='hostname.out',
stderr='hostname.err',
modules=['mpi/openmpi-x86_64']),
JobResources(numCores=ResourceSize(cores_num),
numNodes=ResourceSize(nodes_num)))
]
]
reqs = [{
'request': 'submit',
'jobs': jobs
}, {
'request': 'control',
'command': 'finishAfterAllTasksDone'
}]
save_reqs_to_file(reqs, file_path)
print('jobs saved to file_path: {}'.format(str(file_path)))
sys.argv = [
'QCG-PilotJob', '--log', 'debug', '--file', '--file-path',
str(file_path), '--wd', tmpdir, '--report-format', 'json'
]
QCGPMService().start()
jobEntries = check_job_status_in_json([jobName],
workdir=tmpdir,
dest_state='SUCCEED')
assert all(
(isdir(abspath(join(tmpdir, jobwdir_base))),
exists(join(abspath(join(tmpdir, jobwdir_base)), 'hostname.out')),
exists(join(abspath(join(tmpdir, jobwdir_base)), 'hostname.err')),
stat(join(abspath(join(tmpdir, jobwdir_base)),
'hostname.out')).st_size > 0))
job_nodes = []
allocated_cores = 0
for jname, jentry in jobEntries.items():
assert all(('runtime' in jentry, 'allocation'
in jentry.get('runtime', {})))
jalloc = jentry['runtime']['allocation']
for jalloc_node in jalloc.split(','):
node_name = jalloc_node[:jalloc_node.index('[')]
job_nodes.append(node_name)
print('{} in available nodes ({})'.format(
node_name, ','.join(resources_node_names)))
assert node_name in resources_node_names, '{} not in nodes ({}'.format(
node_name, ','.join(resources_node_names))
ncores = len(jalloc_node[jalloc_node.index('[') + 1:-1].split(':'))
print('#{} cores on node {}'.format(ncores, node_name))
allocated_cores += ncores
assert len(job_nodes) == nodes_num, str(job_nodes)
assert allocated_cores == nodes_num * cores_num, allocated_cores
# check if hostname is in stdout in two lines
with open(abspath(join(tmpdir, join(jobwdir_base, 'hostname.out'))),
'rt') as stdout_file:
stdout_content = [line.rstrip() for line in stdout_file.readlines()]
assert len(stdout_content) == nodes_num * cores_num, str(stdout_content)
assert all(hostname in job_nodes
for hostname in stdout_content), str(stdout_content)
with pytest.raises(ValueError):
check_job_status_in_json([jobName + 'xxx'],
workdir=tmpdir,
dest_state='SUCCEED')
rmtree(tmpdir)
def test_job_description_resources():
# a resource size
# range
rs = ResourceSize(min=4, max=5)
assert all((rs.exact is None, rs.scheduler is None, rs.min == 4,
rs.max == 5, not rs.is_exact()))
assert rs.range == (4, 5)
# exact
rs = ResourceSize(exact=4)
assert all(
(rs.exact == 4, rs.scheduler is None, rs.min is None, rs.max is None,
rs.is_exact()))
assert rs.range == (None, None)
# range with scheduler (just for tests0
rs = ResourceSize(min=4, max=5, scheduler="sched1")
assert all((rs.exact is None, rs.scheduler == "sched1", rs.min == 4,
rs.max == 5, not rs.is_exact()))
assert rs.range == (4, 5)
# exact with scheduler
with pytest.raises(IllegalResourceRequirements):
ResourceSize(exact=4, scheduler="sched1")
# no data
with pytest.raises(IllegalResourceRequirements):
ResourceSize()
# no required data
with pytest.raises(IllegalResourceRequirements):
ResourceSize(scheduler="shed1")
# range and exact
with pytest.raises(IllegalResourceRequirements):
ResourceSize(exact=4, min=2)
# range and exact
with pytest.raises(IllegalResourceRequirements):
ResourceSize(exact=4, max=2)
# illegal exact
with pytest.raises(IllegalResourceRequirements):
ResourceSize(exact=-1)
# illegal range
with pytest.raises(IllegalResourceRequirements):
ResourceSize(max=-2)
with pytest.raises(IllegalResourceRequirements):
ResourceSize(min=-2)
with pytest.raises(IllegalResourceRequirements):
ResourceSize(min=4, max=2)
# serialization with range
rs = ResourceSize(min=4, max=5, scheduler="sched1")
assert all((rs.exact is None, rs.scheduler == "sched1", rs.min == 4,
rs.max == 5, not rs.is_exact()))
assert rs.range == (4, 5)
rs_json = rs.to_json()
rs_clone = ResourceSize(**json.loads(rs_json))
assert all((rs_clone.exact is None, rs_clone.scheduler == "sched1",
rs_clone.min == 4, rs_clone.max == 5, not rs_clone.is_exact()))
assert rs_clone.range == (4, 5)
rs_clone.to_dict() == rs_clone.to_dict()
# serialization with exact
rs = ResourceSize(exact=2)
assert all(
(rs.exact == 2, rs.scheduler is None, rs.min is None, rs.max is None,
rs.is_exact()))
assert rs.range == (None, None)
rs_json = rs.to_json()
rs_clone = ResourceSize(**json.loads(rs_json))
assert all(
(rs_clone.exact == 2, rs_clone.scheduler is None, rs_clone.min is None,
rs_clone.max is None, rs_clone.is_exact()))
assert rs_clone.range == (None, None)
rs_clone.to_dict() == rs_clone.to_dict()
# number of cores as a number
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2 } }"""
job = Job(**json.loads(jobd))
assert job, "Simple job with integer number of cores"
# number of cores as an exact object
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "exact": 2 } } }"""
job = Job(**json.loads(jobd))
assert job, "Simple job with number of cores as an exact object"
# number of cores as a range object
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "min": 2, "max": 3 } } }"""
job = Job(**json.loads(jobd))
assert job, "Simple job with number of cores as a range object"
# number of cores as a range object, with one of the boundary
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "min": 2 } } }"""
job = Job(**json.loads(jobd))
assert job, "Simple job with number of cores as a range object with only min boundary"
# number of cores as a range object, with one of the boundary
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "max": 2 } } }"""
job = Job(**json.loads(jobd))
assert job, "Simple job with number of cores as a range object with only max boundary"
# empty resources element
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { } }"""
Job(**json.loads(jobd))
# no cores specification
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { } } }"""
Job(**json.loads(jobd))
# no nodes specification
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numNodes": { } } }"""
Job(**json.loads(jobd))
# illegal type of resources specification
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": [ "numNodes" ] }"""
Job(**json.loads(jobd))
# illegal type of cores specification
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": [ 1 ] } }"""
Job(**json.loads(jobd))
# illegal type of nodes specification
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numNodes": [ 1 ] } }"""
Job(**json.loads(jobd))
# exact number with range
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "exact": 2, "min": 1, "max": 3 } } }"""
job = Job(**json.loads(jobd))
# exact number with one of the range boundary
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "exact": 2, "min": 1 } } }"""
job = Job(**json.loads(jobd))
# number of cores negative
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": -2 } }"""
job = Job(**json.loads(jobd))
# 'max' greater than 'min' in range object
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "min": 4, "max": 3 } } }"""
job = Job(**json.loads(jobd))
# range boundary negative
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "min": -2 } } }"""
job = Job(**json.loads(jobd))
# range boundary negative
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": { "min": -4, "max": -1 } } }"""
job = Job(**json.loads(jobd))
# general crs
assert not JobResources(numCores=1).has_crs
jr = JobResources(numCores=1, nodeCrs={'gpu': 1})
assert all((jr.has_crs, len(jr.crs) == 1, CRType.GPU
in jr.crs, jr.crs[CRType.GPU] == 1))
# crs without cores count
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "nodeCrs": { "gpu": 1 } } }"""
job = Job(**json.loads(jobd))
# gpu cr
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "gpu": 1 } } }"""
job = Job(**json.loads(jobd))
assert job, "Job with node consumable resources (gpu)"
# mem cr
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "mem": 1 } } }"""
job = Job(**json.loads(jobd))
assert job, "Job with node consumable resources (mem)"
# gpu cr with non-uniform letter case
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "GpU": 1 } } }"""
job = Job(**json.loads(jobd))
assert job, "Job with node consumable resources (gpu)"
# mem cr with non-uniform letter case
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "meM": 1 } } }"""
job = Job(**json.loads(jobd))
assert job, "Job with node consumable resources (mem)"
# many crs with non-uniform letter case
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "meM": 1, "gPU": 2 } } }"""
job = Job(**json.loads(jobd))
assert job, "Job with node consumable resources (mem)"
# unknown cr
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "strange_cr": 1 } } }"""
Job(**json.loads(jobd))
# gpu cr without integer value
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "gpu": "1" } } }"""
Job(**json.loads(jobd))
# mem cr without integer value
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "mem": "one" } } }"""
Job(**json.loads(jobd))
# gpu cr with negative integer value
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "gpu": -1 } } }"""
Job(**json.loads(jobd))
# gpu cr with 0
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "gpu": 0 } } }"""
Job(**json.loads(jobd))
# repeating gpu cr
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "gpu": 1, "GpU": 2 } } }"""
Job(**json.loads(jobd))
# repeating gpu cr
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": { "gpu": 1, "mem": 2, "GpU": 2 } } }"""
Job(**json.loads(jobd))
# wrong format of cr's
with pytest.raises(IllegalJobDescription):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "nodeCrs": [ ] } }"""
Job(**json.loads(jobd))
# num cores & nodes as integers
jr = JobResources(numCores=2)
assert all((jr.cores.is_exact(), jr.cores.exact == 2))
jr = JobResources(numNodes=3)
assert all((jr.nodes.is_exact(), jr.nodes.exact == 3))
# min number of cores
assert JobResources(numCores=ResourceSize(
exact=2)).get_min_num_cores() == 2
assert JobResources(numCores=ResourceSize(min=4)).get_min_num_cores() == 4
assert JobResources(
numCores=ResourceSize(min=3, max=6)).get_min_num_cores() == 3
assert JobResources(
numCores=ResourceSize(exact=2),
numNodes=ResourceSize(exact=1)).get_min_num_cores() == 2
assert JobResources(numCores=ResourceSize(exact=2),
numNodes=ResourceSize(min=4)).get_min_num_cores() == 8
assert JobResources(numCores=ResourceSize(exact=2),
numNodes=ResourceSize(min=4,
max=5)).get_min_num_cores() == 8
assert JobResources(
numCores=ResourceSize(min=3, max=6),
numNodes=ResourceSize(exact=2)).get_min_num_cores() == 6
assert JobResources(numCores=ResourceSize(min=3, max=6),
numNodes=ResourceSize(min=4,
max=6)).get_min_num_cores() == 12
# walltime in job resources
jr = JobResources(numCores=ResourceSize(exact=2), wt="10m")
assert jr.wt == timedelta(minutes=10)
# errors
with pytest.raises(IllegalResourceRequirements):
JobResources(numCores=ResourceSize(exact=2), wt="")
with pytest.raises(IllegalResourceRequirements):
JobResources(numCores=ResourceSize(exact=2), wt="0")
with pytest.raises(IllegalResourceRequirements):
JobResources(numCores=ResourceSize(exact=2), wt="2")
with pytest.raises(IllegalResourceRequirements):
JobResources(numCores=ResourceSize(exact=2), wt="2d")
with pytest.raises(IllegalResourceRequirements):
JobResources(numCores=ResourceSize(exact=2), wt="two hours")
with pytest.raises(IllegalResourceRequirements):
JobResources(numCores=ResourceSize(exact=2), wt="-10")
with pytest.raises(IllegalResourceRequirements):
JobResources(numCores=ResourceSize(exact=2), wt="-10s")
# job resources serialization
jr = JobResources(numCores=ResourceSize(min=2, max=6, scheduler="sched1"),
numNodes=ResourceSize(exact=4),
wt="10m",
nodeCrs={"gpu": 2})
jr_json = jr.to_json()
jr_clone = JobResources(**json.loads(jr_json))
jr.to_dict() == jr_clone.to_dict()
# walltime
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "wt": "10m" } }"""
job = Job(**json.loads(jobd))
assert job.resources.wt == timedelta(minutes=10)
# walltime 2
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "wt": "24h" } }"""
job = Job(**json.loads(jobd))
assert job.resources.wt == timedelta(hours=24)
# walltime 3
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "wt": "24h10m5s" } }"""
job = Job(**json.loads(jobd))
assert job.resources.wt == timedelta(hours=24, minutes=10, seconds=5)
# missing walltime value
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "wt": "" } }"""
job = Job(**json.loads(jobd))
print('job walltime: {}'.format(str(job.resources.wt)))
# wrong walltime format walltime
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "wt": "2d" } }"""
Job(**json.loads(jobd))
# wrong walltime format walltime 2
with pytest.raises(IllegalResourceRequirements):
jobd = """{ "name": "job1",
"execution": { "exec": "/bin/date" },
"resources": { "numCores": 2, "wt": "2" } }"""
Job(**json.loads(jobd))
def test_job_description_subjobs():
# iteration job
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
# iteration names
assert j.get_name() == 'j1'
assert all(
j.get_name(it) == '{}:{}'.format(j.get_name(), it) for it in range(10))
# iteration states (initial)
assert j.state() == JobState.QUEUED
assert all(j.state(it) == JobState.QUEUED for it in range(10))
assert all(j.str_state(it) == JobState.QUEUED.name for it in range(10))
# iteration runtimes
for it in range(10):
j.append_runtime({'host': 'local.{}'.format(it)}, it)
assert all(
j.runtime(it).get('host') == 'local.{}'.format(it) for it in range(10))
# whole job success
for it in range(10):
j.set_state(JobState.SUCCEED, it, 'job {} succeed'.format(it))
assert j.state() == JobState.SUCCEED
assert all(
j.messages(it) == 'job {} succeed'.format(it) for it in range(10))
# whole job fail (one of the iteration failed)
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
for it in range(9):
j.set_state(JobState.SUCCEED, it, 'job {} succeed'.format(it))
j.set_state(JobState.FAILED, 9, 'job 9 failed')
assert j.state() == JobState.FAILED
assert all(
j.messages(it) == 'job {} succeed'.format(it) for it in range(9))
assert j.messages(9) == 'job 9 failed'
# whole job fail (one of the iteration canceled)
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
for it in range(9):
j.set_state(JobState.SUCCEED, it, 'job {} succeed'.format(it))
j.set_state(JobState.CANCELED, 9, 'job 9 canceled')
assert j.state() == JobState.FAILED
assert all(
j.messages(it) == 'job {} succeed'.format(it) for it in range(9))
assert j.messages(9) == 'job 9 canceled'
# whole job fail (one of the iteration omitted)
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
for it in range(9):
j.set_state(JobState.SUCCEED, it, 'job {} succeed'.format(it))
j.set_state(JobState.OMITTED, 9, 'job 9 omitted')
assert j.state() == JobState.FAILED
assert all(
j.messages(it) == 'job {} succeed'.format(it) for it in range(9))
assert j.messages(9) == 'job 9 omitted'
# not all iterations finished
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
for it in range(9):
j.set_state(JobState.SUCCEED, it, 'job {} succeed'.format(it))
assert j.state() == JobState.QUEUED
assert all(
j.messages(it) == 'job {} succeed'.format(it) for it in range(9))
# whole job fail (just one succeed
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
for it in range(9):
j.set_state(JobState.FAILED, it, 'job {} failed'.format(it))
j.set_state(JobState.CANCELED, 9, 'job 9 succeed')
assert j.state() == JobState.FAILED
assert all(j.messages(it) == 'job {} failed'.format(it) for it in range(9))
assert j.messages(9) == 'job 9 succeed'
# many messages per iteration
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
for it in range(10):
j.set_state(JobState.EXECUTING, it, 'job {} executing'.format(it))
assert j.state() == JobState.QUEUED
assert all(j.state(it) == JobState.EXECUTING for it in range(10))
assert all(j.str_state(it) == JobState.EXECUTING.name for it in range(10))
for it in range(10):
j.set_state(JobState.SUCCEED, it, 'job {} finished'.format(it))
assert j.state() == JobState.SUCCEED
assert all(j.state(it) == JobState.SUCCEED for it in range(10))
assert all(j.str_state(it) == JobState.SUCCEED.name for it in range(10))
assert all(
j.messages(it) == 'job {it} executing\njob {it} finished'.format(it=it)
for it in range(10))
# messages for job
j = Job(name='j1',
execution=JobExecution(exec='/bin/date'),
resources=JobResources(numCores=1),
iteration=JobIteration(stop=10))
assert j.has_iterations
j.set_state(JobState.EXECUTING, iteration=None, err_msg='job executing')
assert all((j.state() == JobState.EXECUTING,
j.str_state() == JobState.EXECUTING.name))
assert j.messages() == 'job executing'
for it in range(10):
j.set_state(JobState.EXECUTING, it, 'job {} executing'.format(it))
j.set_state(JobState.FAILED, iteration=None, err_msg='job failed')
assert all(
(j.state() == JobState.FAILED, j.str_state() == JobState.FAILED.name))
assert j.messages() == 'job executing\njob failed'
# failed job will not change state if once set
for it in range(10):
j.set_state(JobState.SUCCEED, it, 'job {} finished'.format(it))
assert all(
(j.state() == JobState.FAILED, j.str_state() == JobState.FAILED.name))
assert all(j.str_state(it) == JobState.SUCCEED.name for it in range(10))
assert all(
j.messages(it) == 'job {it} executing\njob {it} finished'.format(it=it)
for it in range(10))
def test_scheduler_allocate_job_gpus():
nCores = [8, 8, 8, 8]
nGpus = [2, 4, 4, 4]
nCrs = [{
CRType.GPU: CRBind(CRType.GPU, list(range(nGpu)))
} for nGpu in nGpus]
r = Resources(ResourcesType.LOCAL, [
Node("n.{}".format(i), total_cores=nCores[i], used=0, crs=nCrs[i])
for i in range(len(nCores))
])
s = Scheduler(r)
assert s
assert all((len(r.nodes) == len(nCores),
[r.nodes[i] == "n.{}".format(i) for i in range(len(nCores))]))
assert all([
r.nodes[i].total == nCores[i] and r.nodes[i].used == 0
and r.nodes[i].free == nCores[i] for i in range(len(nCores))
])
assert all([len(r.nodes[i].crs) == 1 and CRType.GPU in r.nodes[i].crs and r.nodes[i].crs[CRType.GPU].total_count == nGpus[i] and \
r.nodes[i].crs[CRType.GPU].used == 0 and r.nodes[i].crs[CRType.GPU].available == nGpus[i] for i in range(len(nCores))])
assert all((r.total_cores == sum(nCores), r.free_cores == sum(nCores),
r.used_cores == 0))
# allocate half of the first node's cpus and one gpu
job1_r_c = 4
job1_r_n = 1
job1_r_g = 1
job1_r = JobResources(numCores=job1_r_c,
numNodes=job1_r_n,
nodeCrs={'gpu': job1_r_g})
assert job1_r
a1 = s.allocate_job(job1_r)
assert a1
assert a1 and all(
(a1.cores == job1_r_c * job1_r_n, len(
a1.nodes) == job1_r_n, a1.nodes[0].node.name
== "n.0", a1.nodes[0].cores == [str(cid) for cid in range(job1_r_c)]))
assert a1.nodes[0].crs and ((CRType.GPU in a1.nodes[0].crs,
a1.nodes[0].crs[CRType.GPU].count == job1_r_g,
a1.nodes[0].crs[CRType.GPU].instances == list(
range(job1_r_g))))
assert all(
(r.total_cores == sum(nCores), r.free_cores == sum(nCores) - job1_r_c,
r.used_cores == job1_r_c))
# try to allocate half of the node's cpus but with two gpu's - allocation should not fit to the first node
job2_r_c = 4
job2_r_n = 1
job2_r_g = 2
job2_r = JobResources(numCores=job2_r_c,
numNodes=job2_r_n,
nodeCrs={'gpu': job2_r_g})
assert job2_r
a2 = s.allocate_job(job2_r)
assert a2
assert a2 and all(
(a2.cores == job2_r_c * job2_r_n, len(
a2.nodes) == job2_r_n, a2.nodes[0].node.name
== "n.1", a2.nodes[0].cores == [str(cid) for cid in range(job2_r_c)]))
assert a2.nodes[0].crs and ((CRType.GPU in a2.nodes[0].crs,
a2.nodes[0].crs[CRType.GPU].count == job2_r_g,
a2.nodes[0].crs[CRType.GPU].instances == list(
range(job2_r_g))))
assert all((r.total_cores == sum(nCores),
r.free_cores == sum(nCores) - job1_r_c - job2_r_c,
r.used_cores == job1_r_c + job2_r_c))
# try to allocate node with exceeding gpu amount on node
job3_r_c = 1
job3_r_n = 1
job3_r_g = max(nGpus) + 2
job3_r = JobResources(numCores=job3_r_c,
numNodes=job3_r_n,
nodeCrs={'gpu': job3_r_g})
assert job3_r
with pytest.raises(NotSufficientResources):
s.allocate_job(job3_r)
# try to allocate node with exceeding total gpus amount
job4_r_c = 1
job4_r_n = 4
job4_r_g = max(nGpus)
job4_r = JobResources(numCores=job4_r_c,
numNodes=job4_r_n,
nodeCrs={'gpu': job4_r_g})
assert job4_r
with pytest.raises(NotSufficientResources):
s.allocate_job(job4_r)
# allocate gpus across many nodes
job5_r_c = 2
job5_r_n = 3
job5_r_g = 2
job5_r = JobResources(numCores=job5_r_c,
numNodes=job5_r_n,
nodeCrs={'gpu': job5_r_g})
assert job5_r
a5 = s.allocate_job(job5_r)
assert a5 and all(
(a5.cores == job5_r_c * job5_r_n, len(a5.nodes)
== job5_r_n, a5.nodes[0].node.name == "n.1", a5.nodes[0].cores == [
str(cid) for cid in range(job2_r_c, job2_r_c + job5_r_c)
], a5.nodes[1].node.name == "n.2", a5.nodes[1].cores
== [str(cid) for cid in range(job5_r_c)], a5.nodes[2].node.name
== "n.3", a5.nodes[2].cores == [str(cid)
for cid in range(job5_r_c)])), str(a5)
assert a5.nodes[0].crs and ((CRType.GPU in a5.nodes[0].crs,
a5.nodes[0].crs[CRType.GPU].count == job5_r_g,
a5.nodes[0].crs[CRType.GPU].instances == list(
range(job5_r_g))))
assert a5.nodes[1].crs and ((CRType.GPU in a5.nodes[1].crs,
a5.nodes[1].crs[CRType.GPU].count == job5_r_g,
a5.nodes[1].crs[CRType.GPU].instances == list(
range(job5_r_g))))
assert a5.nodes[2].crs and ((CRType.GPU in a5.nodes[2].crs,
a5.nodes[2].crs[CRType.GPU].count == job5_r_g,
a5.nodes[2].crs[CRType.GPU].instances == list(
range(job5_r_g))))
assert all((r.total_cores == sum(nCores), r.free_cores == sum(nCores) -
job1_r_c - job2_r_c - job5_r_c * job5_r_n,
r.used_cores == job1_r_c + job2_r_c + job5_r_c * job5_r_n))
# allocate cpu's across many nodes
job6_r_c = 16
job6_r = JobResources(numCores=job6_r_c)
assert job6_r
a6 = s.allocate_job(job6_r)
assert a6 and all(
(a6.cores == job6_r_c, len(a6.nodes) == 4, a6.nodes[0].node.name
== "n.0", a6.nodes[0].cores == [
str(cid) for cid in range(job1_r_c, nCores[0])
], a6.nodes[1].node.name == "n.1", a6.nodes[1].cores == [
str(cid) for cid in range(job2_r_c + job5_r_c, nCores[1])
], a6.nodes[2].node.name == "n.2", a6.nodes[2].cores == [
str(cid) for cid in range(job5_r_c, nCores[2])
], a6.nodes[3].node.name == "n.3", a6.nodes[3].cores
== [str(cid) for cid in range(job5_r_c, job5_r_c + 4)])), str(a6)
assert ([a6.nodes[i].crs is None for i in range(4)])
assert all(
(r.total_cores == sum(nCores), r.free_cores == sum(nCores) - job1_r_c -
job2_r_c - job5_r_c * job5_r_n - job6_r_c,
r.used_cores == job1_r_c + job2_r_c + job5_r_c * job5_r_n + job6_r_c))
# release all allocations
a1.release()
a2.release()
a5.release()
a6.release()
assert all((len(r.nodes) == len(nCores),
[r.nodes[i] == "n.{}".format(i) for i in range(len(nCores))]))
assert all([
r.nodes[i].total == nCores[i] and r.nodes[i].used == 0
and r.nodes[i].free == nCores[i] for i in range(len(nCores))
])
assert all([len(r.nodes[i].crs) == 1 and CRType.GPU in r.nodes[i].crs and r.nodes[i].crs[CRType.GPU].total_count == nGpus[i] and \
r.nodes[i].crs[CRType.GPU].used == 0 and r.nodes[i].crs[CRType.GPU].available == nGpus[i] for i in range(len(nCores))])
assert all((r.total_cores == sum(nCores), r.free_cores == sum(nCores),
r.used_cores == 0))