def test_get_env(self):
with TestAreaContext("job_manager_get_env"):
with open("x.py", "w") as f:
f.write("#!/usr/bin/env python\n")
f.write("import os\n")
f.write("assert(os.environ['KEY_ONE'] == 'FirstValue')\n")
f.write("assert(os.environ['KEY_TWO'] == 'SecondValue')\n")
f.write("assert(os.environ['PATH104'] == 'NewPath')\n")
f.write("assert(os.environ['KEY_THREE'] == 'FourthValue:ThirdValue')\n")
f.write("assert(os.environ['KEY_FOUR'] == 'FifthValue:SixthValue:ThirdValue:FourthValue')\n")
os.chmod("x.py", stat.S_IEXEC + stat.S_IREAD)
executable = "./x.py"
joblist = {"name" : "TEST_GET_ENV1",
"executable" : executable,
"stdout" : "outfile.stdout",
"stderr" : "outfile.stderr",
"argList" : [] }
data = {"umask" : "0000",
"global_environment" : {"KEY_ONE" : "FirstValue", "KEY_TWO" : "SecondValue", "KEY_THREE" : "ThirdValue", "KEY_FOUR" : "ThirdValue:FourthValue" },
"global_update_path" : {"PATH104" : "NewPath", "KEY_THREE" : "FourthValue", "KEY_FOUR" : "FifthValue:SixthValue"},
"DATA_ROOT" : "/path/to/data",
"jobList" : [joblist, joblist]}
jobs_file = os.path.join(os.getcwd(), "jobs.json")
with open(jobs_file, "w") as f:
f.write(json.dumps(data))
jobm = JobManager()
exit_status, msg = jobm.runJob(jobm[0])
self.assertEqual(exit_status, 0)
exit_status, msg = jobm.runJob(jobm[1])
self.assertEqual(exit_status, 0)
def test1(self):
with TestAreaContext("job_manager_runtime_int_kw"):
executable = "echo"
job_0 = {
"name": "JOB_1",
"executable": executable,
"stdout": "outfile.stdout.1",
"stderr": None,
"argList": ['a_file', '5.12'],
"min_arg": 1,
"max_arg": 2,
"arg_types": ['STRING', 'RUNTIME_INT']
}
data = {
"umask": "0000",
"DATA_ROOT": "/path/to/data",
"jobList": [job_0]
}
jobs_file = os.path.join(os.getcwd(), "jobs.json")
with open(jobs_file, "w") as f:
f.write(json.dumps(data))
jobm = JobManager()
exit_status, msg = jobm.runJob(jobm[0])
self.assertEqual(exit_status, 1)
def _test_run_multiple_OK(self, create_jobs):
with TestAreaContext("mkdir"):
joblist = []
dir_list = ["1", "2", "3", "4", "5"]
for d in dir_list:
job = {
"name": "MKDIR",
"executable": "/bin/mkdir",
"stdout": "mkdir_out",
"stderr": "mkdir_err",
"argList": ["-p", "-v", d]
}
joblist.append(job)
create_jobs(joblist)
jobm = JobManager()
for (index, job) in enumerate(jobm):
exit_status, msg = jobm.runJob(job)
self.assertEqual(exit_status, 0)
self.assertTrue(os.path.isdir(dir_list[index]))
self.assertTrue(os.path.isfile("mkdir_out.%d" % index))
self.assertTrue(os.path.isfile("mkdir_err.%d" % index))
self.assertEqual(0, os.path.getsize("mkdir_err.%d" % index))
def test_exec_env(self):
with TestAreaContext("exec_env_test"):
with open("exec_env.py", "w") as f:
f.write("""#!/usr/bin/env python\n
import os
import json
with open("exec_env_exec_env.json") as f:
exec_env = json.load(f)
assert exec_env["TEST_ENV"] == "123"
assert exec_env["NOT_SET"] is None
""")
os.chmod("exec_env.py", stat.S_IEXEC + stat.S_IREAD)
with open("EXEC_ENV", "w") as f:
f.write("EXECUTABLE exec_env.py\n")
f.write("EXEC_ENV TEST_ENV 123\n")
f.write("EXEC_ENV NOT_SET")
ext_job = ExtJob("EXEC_ENV", False)
job_list = ExtJoblist()
job_list.add_job("EXEC_ENV", ext_job)
forward_model = ForwardModel(job_list)
forward_model.add_job("EXEC_ENV")
global_args = SubstitutionList()
env_varlist = EnvironmentVarlist()
forward_model.formatted_fprintf( "run_id", None, "data_root", global_args, 0, env_varlist)
jobm = JobManager(json_file = "jobs.json")
job0 = jobm[0]
exec_env = job0.get("exec_env")
self.assertEqual(len(exec_env), 2)
exit_status, msg = jobm.runJob(job0)
self.assertEqual(exit_status, 0)
def _test_verify_executable(self, create_jobs):
with TestAreaContext(gen_area_name("no_executable", create_jobs)):
with self.assertRaises(IOError):
fname = "this/is/not/a/file"
executable = os.path.join(os.getcwd(), fname)
joblist = [{
"name": "TEST_EXECUTABLE_NOT_FOUND",
"executable": executable,
"stdout": "mkdir_out",
"stderr": "mkdir_err",
"argList": []
}]
create_jobs(joblist)
jobm = JobManager()
jobm.runJob(jobm[0])
with TestAreaContext(gen_area_name("file_not_exec", create_jobs)):
with self.assertRaises(IOError):
fname = "not_executable"
with open(fname, "w") as f:
f.write("#!/bin/sh\n")
f.write("exit 1\n")
executable = os.path.join(os.getcwd(), fname)
joblist = [{
"name": "TEST_JOB",
"executable": executable,
"stdout": "mkdir_out",
"stderr": "mkdir_err",
"argList": []
}]
create_jobs(joblist)
jobm = JobManager()
jobm.runJob(jobm[0])
with TestAreaContext(gen_area_name("unix_cmd", create_jobs)):
executable = "ls"
self.assertFalse(os.path.isfile(executable))
joblist = [{
"name": "TEST_UNIX_CMD_FROM_PATH",
"executable": executable,
"stdout": "mkdir_out",
"stderr": "mkdir_err",
"argList": []
}]
create_jobs(joblist)
jobm = JobManager()
jobm.runJob(jobm[0])
def test_run_job(self):
with TestAreaContext(gen_area_name("run_job_fail", create_jobs_json)):
with open("run.sh", "w") as f:
f.write("#!/bin/sh\n")
f.write("exit 1\n")
st = os.stat("run.sh")
os.chmod("run.sh", st.st_mode | stat.S_IEXEC)
executable = os.path.join(os.getcwd(), "run.sh")
joblist = [{"name" : "TEST_JOB",
"executable" : executable,
"argList" : ["A","B"]}]
create_jobs_json(joblist)
jobm = JobManager()
self.assertTrue(os.path.isfile(executable))
exit_status, msg = jobm.runJob(jobm[0])
self.assertEqual(exit_status, 1)
def test_run_multiple_fail(self):
with TestAreaContext(gen_area_name("exit", create_jobs_json)):
joblist = []
for index in range(1, 6):
job = {"name" : "exit",
"executable" : "/bin/bash",
"stdout" : "exit_out",
"stderr": "exit_err",
# produces something on stderr, and exits with exit_code=index
"argList": ["-c", "echo \"failed with %s\" 1>&2 ; exit %s" % (index, index)]}
joblist.append(job)
create_jobs_json(joblist)
jobm = JobManager()
for (index, job) in enumerate(jobm):
exit_status, msg = jobm.runJob(job)
self.assertEqual(exit_status, index + 1)
self.assertTrue(os.path.getsize(
"exit_err.%d" % index) > 0)
def _test_run_multiple_fail(self, create_jobs):
with TestAreaContext(gen_area_name("mkdir", create_jobs)):
joblist = []
dir_list = ["1", "2", "3", "4", "5"]
for d in dir_list:
job = {
"name": "MKDIR",
"executable": "/bin/mkdir",
"stdout": "mkdir_out",
"stderr": "mkdir_err",
"argList": ["-p", "-v", "read-only/%s" % d]
}
joblist.append(job)
create_jobs(joblist)
jobm = JobManager()
os.mkdir("read-only")
os.chmod("read-only", stat.S_IRUSR + stat.S_IXUSR)
for (index, job) in enumerate(jobm):
exit_status, msg = jobm.runJob(job)
self.assertEqual(exit_status, 1)
self.assertTrue(os.path.getsize("mkdir_err.%d" % index) > 0)