def test_JobListInit(self):
jl = JobList()
try:
jl.add('job1')
self.fail("Invalid object added to job list")
except AssertionError:
pass
jName1 = 'job1'
j1 = Job(jName1, JobExecution('/bin/date'), JobResources(numCores=2))
jl.add(j1)
self.assertTrue(jl.get(jName1) is not None)
self.assertTrue(jl.exist(jName1))
try:
jl.add(j1)
self.fail('Non-uniqe job added to job list')
except JobAlreadyExist:
pass
self.assertTrue(jl.get(jName1) is not None)
self.assertTrue(jl.exist(jName1))
jName2 = 'job2'
jl.add(Job('job2', JobExecution('/bin/date'),
JobResources(numCores=1)))
self.assertTrue(jl.get(jName2) is not None)
self.assertTrue(jl.exist(jName2))
def test_JobImport(self):
jDesc = '''{
"name": "msleep2",
"execution": {
"exec": "/usr/bin/sleep",
"args": [
"5s"
],
"env": {},
"wd": "/home/kieras/app-scheduler/src/repo/test-sandbox/sleep2.sandbox",
"stdout": "sleep2.stdout",
"stderr": "sleep2.stderr"
},
"resources": {
"numCores": {
"exact": 2
}
}
}'''
j = Job(**json.loads(jDesc))
self.assertIsNotNone(j)
jDesc_copy = j.toJSON()
jDict = json.loads(jDesc)
jDictCopy = json.loads(jDesc_copy)
self.assertEqual(jDict, jDictCopy)
def test_JobVars(self):
res = self.createLocalResources()
manager = Manager(res, self.config)
scriptFile = join(self.testSandbox, 'script.sh')
if exists(scriptFile):
os.remove(scriptFile)
with open(scriptFile, 'w') as f:
f.write('''
#!/bin/env bash
echo "*** environment ***"
env
''')
os.chmod(scriptFile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
jobs = [
Job(
'job1',
JobExecution(
'bash',
args=[scriptFile],
wd='${root_wd}/job1',
stdout='${nnodes}.${ncores}-${jname}.stdout',
stderr='${nnodes}.${ncores}-${jname}.stderr',
),
JobResources(numNodes=ResourceSize(1),
numCores=ResourceSize(2)))
]
startTime = datetime.datetime.now()
asyncio.get_event_loop().run_until_complete(
asyncio.gather(self.__schedule(jobs, manager)))
asyncio.get_event_loop().close()
duration = datetime.datetime.now() - startTime
self.assertTrue(duration.total_seconds() > 0
and duration.total_seconds() < 4)
job1_wd = join(self.testSandbox, 'job1')
self.assertTrue(os.path.exists(job1_wd))
self.assertTrue(
os.path.exists(os.path.join(job1_wd, '1.2-job1.stdout')))
self.assertTrue(
os.path.exists(os.path.join(job1_wd, '1.2-job1.stderr')))
stderrStat = os.stat(os.path.join(job1_wd, '1.2-job1.stderr'))
self.assertTrue(stderrStat.st_size == 0)
with open(os.path.join(job1_wd, '1.2-job1.stdout'), 'r', 1) as file, \
mmap.mmap(file.fileno(), 0, prot=mmap.PROT_READ) as s:
self.assertTrue(s.find('QCG_PM_NTASKS=2'.encode('UTF-8')) != -1)
self.assertTrue(
s.find('QCG_PM_TASKS_PER_NODE=2'.encode('UTF-8')) != -1)
self.assertTrue(s.find('QCG_PM_NNODES=1'.encode('UTF-8')) != -1)
self.assertTrue(s.find('QCG_PM_NPROCS=2'.encode('UTF-8')) != -1)
def __setupJobs(self):
self.jobs = [
Job(
'msleep1',
JobExecution(
'/usr/bin/sleep',
args=['5s'],
wd=abspath(join(self.testSandbox, 'sleep1.sandbox')),
stdout='sleep1.stdout',
stderr='sleep1.stderr',
), JobResources(numCores=ResourceSize(2))),
Job(
'msleep2',
JobExecution(
'/usr/bin/sleep',
args=['5s'],
wd=abspath(join(self.testSandbox, 'sleep2.sandbox')),
stdout='sleep2.stdout',
stderr='sleep2.stderr',
), JobResources(numCores=ResourceSize(2))),
Job(
'mscript',
JobExecution(
'/usr/bin/bash',
args=[self.scriptFile],
wd=abspath(join(self.testSandbox, 'script.sandbox')),
stdout='script.stdout',
stderr='script.stderr',
), JobResources(numCores=ResourceSize(2))),
Job(
'msleep3',
JobExecution(
'/usr/bin/sleep',
args=['5s'],
wd=abspath(join(self.testSandbox, 'sleep3.sandbox')),
stdout='sleep3.stdout',
stderr='sleep3.stderr',
), JobResources(numCores=ResourceSize(1)))
]
def test_JobInitAndJson(self):
jName = 'job1'
j = Job(jName, JobExecution('/bin/date'), JobResources(numCores=2))
self.assertIsNotNone(j)
self.assertIsNotNone(j.execution)
self.assertEqual(j.name, jName)
self.assertEqual(j.state, JobState.QUEUED)
self.assertEqual(len(j.history), 1)
self.assertEqual(j.hasDependencies(), False)
j.state = JobState.EXECUTING
self.assertEqual(j.state, JobState.EXECUTING)
self.assertEqual(len(j.history), 2)
j_json = j.toJSON()
j_copy = Job(**json.loads(j_json))
self.assertIsNotNone(j_copy)
j_json2 = j_copy.toJSON()
self.assertEqual(j_json, j_json2)
self.compareJobs(j, j_copy)
def test_JobComplex(self):
jDesc = '''{
"name": "mscript",
"execution": {
"exec": "/usr/bin/bash",
"args": [
"/home/kieras/app-scheduler/src/repo/test-sandbox/script.sh"
],
"env": {
"VAR1": "VAR1_VALUE"
},
"wd": "/home/kieras/app-scheduler/src/repo/test-sandbox/script.sandbox",
"stdout": "script.stdout",
"stderr": "script.stderr"
},
"resources": {
"numCores": {
"exact": 2
}
},
"files": {
"stageIn": [
"in1",
"in2"
],
"stageOut": [
"out1",
"out2",
"out3"
]
},
"dependencies": {
"after": [
"job1",
"job2"
]
}
}'''
j = Job(**json.loads(jDesc))
self.assertIsNotNone(j)
jDesc_copy = j.toJSON()
jDict = json.loads(jDesc)
jDictCopy = json.loads(jDesc_copy)
self.assertEqual(jDict, jDictCopy)
self.assertEqual(j.name, "mscript")
self.assertEqual(j.execution.exec, "/usr/bin/bash")
self.assertEqual(len(j.execution.args), 1)
self.assertEqual(
j.execution.args[0],
"/home/kieras/app-scheduler/src/repo/test-sandbox/script.sh")
self.assertEqual(
j.execution.wd,
"/home/kieras/app-scheduler/src/repo/test-sandbox/script.sandbox")
self.assertEqual(len(j.execution.env), 1)
self.assertEqual(j.execution.env['VAR1'], 'VAR1_VALUE')
self.assertEqual(j.resources.hasCores(), True)
self.assertEqual(j.resources.cores.exact, 2)
self.assertEqual(j.resources.hasNodes(), False)
self.assertEqual(j.hasDependencies(), True)
self.assertEqual(len(j.dependencies.after), 2)
self.assertEqual(j.hasFiles(), True)
self.assertEqual(len(j.files.stageIn), 2)
self.assertEqual(len(j.files.stageOut), 3)
def __init__(self, reqData, env=None):
self.jobs = []
assert reqData is not None
if 'jobs' not in reqData or not reqData['jobs'] or not isinstance(reqData['jobs'], list):
raise InvalidRequest('Wrong submit request - missing jobs data')
newJobs = []
vars = {
'rcnt': str(SubmitReq.REQ_CNT),
'uniq': str(uuid.uuid4()),
'sname': 'local',
'date': str(datetime.datetime.today()),
'time': str(datetime.time()),
'dateTime': str(datetime.datetime.now())
}
SubmitReq.REQ_CNT += 1
logging.debug("request data contains %d jobs" % (len(reqData['jobs'])))
for reqJob in reqData['jobs']:
if not isinstance(reqJob, dict):
raise InvalidRequest('Wrong submit request - wrong job data')
haveIterations = False
start = 0
end = 1
# look for 'iterate' directive
if 'iterate' in reqJob:
if not isinstance(reqJob['iterate'], list) or len(reqJob['iterate']) != 2:
raise InvalidRequest('Wrong format of iterative directive: not a two-element list')
(start, end) = reqJob['iterate'][0:2]
if start > end:
raise InvalidRequest('Wrong format of iterative directive: start index larger then stop one')
vars['uniq'] = str(uuid.uuid4())
vars['its'] = end - start
vars['it_start'] = start
vars['it_stop'] = end
haveIterations = True
del reqJob['iterate']
logging.debug("request job params: start(%d), end(%d), haveIters(%s)" %
(start, end, haveIterations))
if 'resources' not in env or env['resources'] is None:
raise InvalidRequest(
'Wrong submit request - failed to resolve split-into without resource information')
numCoresPlans = []
# look for 'split-into' in resources->numCores
if 'resources' in reqJob and 'numCores' in reqJob['resources']:
if 'split-into' in reqJob['resources']['numCores']:
numCoresPlans = IterScheduler.GetScheduler('split-into').Schedule(reqJob['resources']['numCores'],
end - start, env['resources'].totalCores)
elif 'scheduler' in reqJob['resources']['numCores']:
Scheduler = IterScheduler.GetScheduler(reqJob['resources']['numCores']['scheduler'])
del reqJob['resources']['numCores']['scheduler']
numCoresPlans = Scheduler.Schedule(reqJob['resources']['numCores'],
end - start, env['resources'].totalCores)
numNodesPlans = []
# look for 'split-into' in resources->numNodes
if 'resources' in reqJob and 'numNodes' in reqJob['resources']:
if 'split-into' in reqJob['resources']['numNodes']:
numNodesPlans = IterScheduler.GetScheduler('split-into').Schedule(reqJob['resources']['numNodes'],
end - start, env['resources'].totalNodes)
elif 'scheduler' in reqJob['resources']['numNodes']:
Scheduler = IterScheduler.GetScheduler(reqJob['resources']['numNodes']['scheduler'])
del reqJob['resources']['numNodes']['scheduler']
numNodesPlans = Scheduler.Schedule(reqJob['resources']['numNodes'],
end - start, env['resources'].totalNodes)
# default value for missing 'resources' definition
if 'resources' not in reqJob:
reqJob['resources'] = { 'numCores': { 'exact': 1 } }
for idx in range(start, end):
if haveIterations:
vars['it'] = idx
try:
reqJob_vars = self.__replaceVariables(reqJob, vars)
varsStep2 = {
'jname': reqJob_vars['name']
}
logging.debug("replacing jname variable with %s" % (reqJob['name']))
reqJob_vars = self.__replaceVariables(reqJob_vars, varsStep2)
if numCoresPlans is not None and len(numCoresPlans) > idx - start:
reqJob_vars['resources']['numCores'] = numCoresPlans[idx - start]
if numNodesPlans is not None and len(numNodesPlans) > idx - start:
reqJob_vars['resources']['numNodes'] = numNodesPlans[idx - start]
newJobs.append(Job(**reqJob_vars))
except Exception as e:
logging.exception('Wrong submit request')
raise InvalidRequest('Wrong submit request - problem with variables') from e
logging.debug("appending %d jobs to request job list" % (len(newJobs)))
self.jobs.extend(newJobs)
def test_ExecutorSimple(self):
res = self.createLocalResources()
manager = Manager(res)
testSandbox = 'test-sandbox'
if exists(testSandbox):
shutil.rmtree(testSandbox)
os.makedirs(testSandbox)
for dir in [
'hostname.sandbox', 'env.sandbox', 'sleep.sandbox',
'script.sandbox'
]:
dPath = join(testSandbox, dir)
if exists(dPath):
shutil.rmtree(dPath)
scriptFile = abspath(join(testSandbox, 'script.sh'))
if exists(scriptFile):
os.remove(scriptFile)
with open(scriptFile, 'w') as f:
f.write('''
#!/bin/bash
echo "*** environment ***"
env
echo "*** info ***"
echo "host: `hostname --fqdn`"
echo "cwd: `pwd`"
echo "date: `date`"
echo "account: `id`"
echo "taskset: `taskset -p $$`"
''')
hostnameStdinFile = abspath(join(testSandbox, 'hostname.stdin'))
if exists(hostnameStdinFile):
os.remove(hostnameStdinFile)
with open(hostnameStdinFile, 'w') as f:
f.write('some host name')
jobs = [
Job(
'job1',
JobExecution(
'/usr/bin/wc',
args=['-m'],
wd=abspath(join(testSandbox, 'hostname.sandbox')),
stdin=hostnameStdinFile,
stdout='hostname.stdout',
stderr='hostname.stderr',
), JobResources(numCores=ResourceSize(2))),
Job(
'job2',
JobExecution(
'/usr/bin/env',
wd=abspath(join(testSandbox, 'env.sandbox')),
stdout='env.stdout',
stderr='env.stderr',
), JobResources(numCores=ResourceSize(1))),
Job(
'sleep',
JobExecution(
'/usr/bin/sleep',
args=['2s'],
wd=abspath(join(testSandbox, 'sleep.sandbox')),
stdout='sleep.stdout',
stderr='sleep.stderr',
), JobResources(numCores=ResourceSize(1))),
Job(
'script',
JobExecution(
'/usr/bin/bash',
args=[scriptFile],
wd=abspath(join(testSandbox, 'script.sandbox')),
stdout='script.stdout',
stderr='script.stderr',
), JobResources(numCores=ResourceSize(1)))
]
startTime = datetime.datetime.now()
asyncio.get_event_loop().run_until_complete(
asyncio.gather(self.__schedule(jobs, manager)))
asyncio.get_event_loop().close()
duration = datetime.datetime.now() - startTime
self.assertTrue(duration.total_seconds() > 2
and duration.total_seconds() < 6)
for job in jobs:
self.assertTrue(os.path.exists(job.execution.wd))
self.assertTrue(
os.path.exists(
os.path.join(job.execution.wd, job.execution.stdout)))
self.assertTrue(
os.path.exists(
os.path.join(job.execution.wd, job.execution.stderr)))
stderrStat = os.stat(
os.path.join(job.execution.wd, job.execution.stderr))
self.assertTrue(stderrStat.st_size == 0)