def test_get_env(self): with TestAreaContext("job_manager_get_env"): with open("x.py", "w") as f: f.write("#!/usr/bin/env python\n") f.write("import os\n") f.write("assert(os.environ['KEY_ONE'] == 'FirstValue')\n") f.write("assert(os.environ['KEY_TWO'] == 'SecondValue')\n") f.write("assert(os.environ['PATH104'] == 'NewPath')\n") f.write("assert(os.environ['KEY_THREE'] == 'FourthValue:ThirdValue')\n") f.write("assert(os.environ['KEY_FOUR'] == 'FifthValue:SixthValue:ThirdValue:FourthValue')\n") os.chmod("x.py", stat.S_IEXEC + stat.S_IREAD) executable = "./x.py" joblist = {"name" : "TEST_GET_ENV1", "executable" : executable, "stdout" : "outfile.stdout", "stderr" : "outfile.stderr", "argList" : [] } data = {"umask" : "0000", "global_environment" : {"KEY_ONE" : "FirstValue", "KEY_TWO" : "SecondValue", "KEY_THREE" : "ThirdValue", "KEY_FOUR" : "ThirdValue:FourthValue" }, "global_update_path" : {"PATH104" : "NewPath", "KEY_THREE" : "FourthValue", "KEY_FOUR" : "FifthValue:SixthValue"}, "DATA_ROOT" : "/path/to/data", "jobList" : [joblist, joblist]} jobs_file = os.path.join(os.getcwd(), "jobs.json") with open(jobs_file, "w") as f: f.write(json.dumps(data)) jobm = JobManager() exit_status, msg = jobm.runJob(jobm[0]) self.assertEqual(exit_status, 0) exit_status, msg = jobm.runJob(jobm[1]) self.assertEqual(exit_status, 0)
def test1(self): with TestAreaContext("job_manager_runtime_int_kw"): executable = "echo" job_0 = { "name": "JOB_1", "executable": executable, "stdout": "outfile.stdout.1", "stderr": None, "argList": ['a_file', '5.12'], "min_arg": 1, "max_arg": 2, "arg_types": ['STRING', 'RUNTIME_INT'] } data = { "umask": "0000", "DATA_ROOT": "/path/to/data", "jobList": [job_0] } jobs_file = os.path.join(os.getcwd(), "jobs.json") with open(jobs_file, "w") as f: f.write(json.dumps(data)) jobm = JobManager() exit_status, msg = jobm.runJob(jobm[0]) self.assertEqual(exit_status, 1)
def _test_run_multiple_OK(self, create_jobs): with TestAreaContext("mkdir"): joblist = [] dir_list = ["1", "2", "3", "4", "5"] for d in dir_list: job = { "name": "MKDIR", "executable": "/bin/mkdir", "stdout": "mkdir_out", "stderr": "mkdir_err", "argList": ["-p", "-v", d] } joblist.append(job) create_jobs(joblist) jobm = JobManager() for (index, job) in enumerate(jobm): exit_status, msg = jobm.runJob(job) self.assertEqual(exit_status, 0) self.assertTrue(os.path.isdir(dir_list[index])) self.assertTrue(os.path.isfile("mkdir_out.%d" % index)) self.assertTrue(os.path.isfile("mkdir_err.%d" % index)) self.assertEqual(0, os.path.getsize("mkdir_err.%d" % index))
def test_exec_env(self): with TestAreaContext("exec_env_test"): with open("exec_env.py", "w") as f: f.write("""#!/usr/bin/env python\n import os import json with open("exec_env_exec_env.json") as f: exec_env = json.load(f) assert exec_env["TEST_ENV"] == "123" assert exec_env["NOT_SET"] is None """) os.chmod("exec_env.py", stat.S_IEXEC + stat.S_IREAD) with open("EXEC_ENV", "w") as f: f.write("EXECUTABLE exec_env.py\n") f.write("EXEC_ENV TEST_ENV 123\n") f.write("EXEC_ENV NOT_SET") ext_job = ExtJob("EXEC_ENV", False) job_list = ExtJoblist() job_list.add_job("EXEC_ENV", ext_job) forward_model = ForwardModel(job_list) forward_model.add_job("EXEC_ENV") global_args = SubstitutionList() env_varlist = EnvironmentVarlist() forward_model.formatted_fprintf( "run_id", None, "data_root", global_args, 0, env_varlist) jobm = JobManager(json_file = "jobs.json") job0 = jobm[0] exec_env = job0.get("exec_env") self.assertEqual(len(exec_env), 2) exit_status, msg = jobm.runJob(job0) self.assertEqual(exit_status, 0)
def _test_verify_executable(self, create_jobs): with TestAreaContext(gen_area_name("no_executable", create_jobs)): with self.assertRaises(IOError): fname = "this/is/not/a/file" executable = os.path.join(os.getcwd(), fname) joblist = [{ "name": "TEST_EXECUTABLE_NOT_FOUND", "executable": executable, "stdout": "mkdir_out", "stderr": "mkdir_err", "argList": [] }] create_jobs(joblist) jobm = JobManager() jobm.runJob(jobm[0]) with TestAreaContext(gen_area_name("file_not_exec", create_jobs)): with self.assertRaises(IOError): fname = "not_executable" with open(fname, "w") as f: f.write("#!/bin/sh\n") f.write("exit 1\n") executable = os.path.join(os.getcwd(), fname) joblist = [{ "name": "TEST_JOB", "executable": executable, "stdout": "mkdir_out", "stderr": "mkdir_err", "argList": [] }] create_jobs(joblist) jobm = JobManager() jobm.runJob(jobm[0]) with TestAreaContext(gen_area_name("unix_cmd", create_jobs)): executable = "ls" self.assertFalse(os.path.isfile(executable)) joblist = [{ "name": "TEST_UNIX_CMD_FROM_PATH", "executable": executable, "stdout": "mkdir_out", "stderr": "mkdir_err", "argList": [] }] create_jobs(joblist) jobm = JobManager() jobm.runJob(jobm[0])
def test_run_job(self): with TestAreaContext(gen_area_name("run_job_fail", create_jobs_json)): with open("run.sh", "w") as f: f.write("#!/bin/sh\n") f.write("exit 1\n") st = os.stat("run.sh") os.chmod("run.sh", st.st_mode | stat.S_IEXEC) executable = os.path.join(os.getcwd(), "run.sh") joblist = [{"name" : "TEST_JOB", "executable" : executable, "argList" : ["A","B"]}] create_jobs_json(joblist) jobm = JobManager() self.assertTrue(os.path.isfile(executable)) exit_status, msg = jobm.runJob(jobm[0]) self.assertEqual(exit_status, 1)
def test_run_multiple_fail(self): with TestAreaContext(gen_area_name("exit", create_jobs_json)): joblist = [] for index in range(1, 6): job = {"name" : "exit", "executable" : "/bin/bash", "stdout" : "exit_out", "stderr": "exit_err", # produces something on stderr, and exits with exit_code=index "argList": ["-c", "echo \"failed with %s\" 1>&2 ; exit %s" % (index, index)]} joblist.append(job) create_jobs_json(joblist) jobm = JobManager() for (index, job) in enumerate(jobm): exit_status, msg = jobm.runJob(job) self.assertEqual(exit_status, index + 1) self.assertTrue(os.path.getsize( "exit_err.%d" % index) > 0)
def _test_run_multiple_fail(self, create_jobs): with TestAreaContext(gen_area_name("mkdir", create_jobs)): joblist = [] dir_list = ["1", "2", "3", "4", "5"] for d in dir_list: job = { "name": "MKDIR", "executable": "/bin/mkdir", "stdout": "mkdir_out", "stderr": "mkdir_err", "argList": ["-p", "-v", "read-only/%s" % d] } joblist.append(job) create_jobs(joblist) jobm = JobManager() os.mkdir("read-only") os.chmod("read-only", stat.S_IRUSR + stat.S_IXUSR) for (index, job) in enumerate(jobm): exit_status, msg = jobm.runJob(job) self.assertEqual(exit_status, 1) self.assertTrue(os.path.getsize("mkdir_err.%d" % index) > 0)