def test_run_multiple_fail_only_runs_one(self):
joblist = []
for index in range(1, 6):
job = {
"name":
"exit",
"executable":
"/bin/bash",
"stdout":
"exit_out",
"stderr":
"exit_err",
# produces something on stderr, and exits with
# exit_code=index
"argList": [
"-c",
'echo "failed with {}" 1>&2 ; exit {}'.format(
index, index),
],
}
joblist.append(job)
create_jobs_json(joblist)
jobm = JobRunner()
statuses = [s for s in list(jobm.run([])) if isinstance(s, Exited)]
self.assertEqual(len(statuses), 1)
for i, status in enumerate(statuses):
self.assertEqual(status.exit_code, i + 1)
def test_run_one_job_with_an_integer_arg_is_actually_a_fractional(self):
executable = "echo"
job_0 = {
"name": "JOB_1",
"executable": executable,
"stdout": "outfile.stdout.1",
"stderr": None,
"argList": ['a_file', '5.12'],
"min_arg": 1,
"max_arg": 2,
"arg_types": ['STRING', 'RUNTIME_INT']
}
data = {
"umask": "0000",
"DATA_ROOT": "/path/to/data",
"jobList": [job_0]
}
jobs_file = os.path.join(os.getcwd(), "jobs.json")
with open(jobs_file, "w") as f:
f.write(json.dumps(data))
runner = JobRunner()
statuses = list(runner.run([]))
starts = [e for e in statuses if isinstance(e, Start)]
self.assertEqual(1, len(starts), "There should be 1 start message")
self.assertFalse(starts[0].success(),
"job should not start with success")
def test_run_multiple_ok(self):
joblist = []
dir_list = ["1", "2", "3", "4", "5"]
for d in dir_list:
job = {
"name": "MKDIR",
"executable": "/bin/mkdir",
"stdout": "mkdir_out",
"stderr": "mkdir_err",
"argList": ["-p", "-v", d],
}
joblist.append(job)
create_jobs_json(joblist)
jobm = JobRunner()
statuses = [s for s in list(jobm.run([])) if isinstance(s, Exited)]
self.assertEqual(len(statuses), 5)
for status in statuses:
self.assertEqual(status.exit_code, 0)
for index, dir_number in enumerate(dir_list):
self.assertTrue(os.path.isdir(dir_list[index]))
self.assertTrue(os.path.isfile("mkdir_out.%d" % index))
self.assertTrue(os.path.isfile("mkdir_err.%d" % index))
self.assertEqual(0, os.path.getsize("mkdir_err.%d" % index))
def main(args):
# If run_path is defined, enter into that directory
if len(args) > 1:
run_path = args[1]
if not os.path.exists(run_path):
sys.exit("No such directory: {}".format(run_path))
os.chdir(run_path)
jobs_to_run = []
if len(args) > 2:
jobs_to_run = args[2:]
reporters = []
is_interactive_run = len(args) > 2
if is_interactive_run:
reporters.append(reporting.Interactive())
else:
reporters.append(reporting.File())
reporters.append(reporting.Network())
job_runner = JobRunner()
for job_status in job_runner.run(jobs_to_run):
for reporter in reporters:
reporter.report(job_status)
if isinstance(job_status, Finish) and not job_status.success():
pgid = os.getpgid(os.getpid())
os.killpg(pgid, signal.SIGKILL)
def test_run_one_job_with_an_integer_arg_is_actually_a_fractional(self):
executable = "echo"
job_0 = {
"name": "JOB_1",
"executable": executable,
"stdout": "outfile.stdout.1",
"stderr": None,
"argList": ["a_file", "5.12"],
"min_arg": 1,
"max_arg": 2,
"arg_types": ["STRING", "RUNTIME_INT"],
}
data = {
"umask": "0000",
"DATA_ROOT": "/path/to/data",
"jobList": [job_0]
}
runner = JobRunner(data)
statuses = list(runner.run([]))
starts = [e for e in statuses if isinstance(e, Start)]
self.assertEqual(1, len(starts), "There should be 1 start message")
self.assertFalse(starts[0].success(),
"job should not start with success")
def test_run_given_one_job_with_missing_file_and_one_file_present(self):
with open("a_file", "w") as f:
f.write("Hello")
executable = "echo"
job_1 = {
"name": "JOB_0",
"executable": executable,
"stdout": "outfile.stdout.0",
"stderr": None,
"argList": ['some_file'],
"min_arg": 1,
"max_arg": 1,
"arg_types": ['RUNTIME_FILE']
}
job_0 = {
"name": "JOB_1",
"executable": executable,
"stdout": "outfile.stdout.1",
"stderr": None,
"argList": ['5', 'a_file'],
"min_arg": 1,
"max_arg": 2,
"arg_types": ['RUNTIME_INT', 'RUNTIME_FILE']
}
data = {
"umask": "0000",
"DATA_ROOT": "/path/to/data",
"jobList": [job_0, job_1]
}
jobs_file = os.path.join(os.getcwd(), "jobs.json")
with open(jobs_file, "w") as f:
f.write(json.dumps(data))
runner = JobRunner()
statuses = list(runner.run([]))
starts = [e for e in statuses if isinstance(e, Start)]
self.assertEqual(2, len(starts), "There should be 2 start messages")
self.assertTrue(starts[0].success(),
"first job should start with success")
self.assertFalse(starts[1].success(),
"second job should not start with success")
exits = [e for e in statuses if isinstance(e, Exited)]
self.assertEqual(1, len(exits), "There should be 1 exit message")
self.assertTrue(exits[0].success(),
"first job should exit with success")
self.assertEqual(Finish, type(statuses[-1]),
"last message should be Finish")
self.assertFalse(statuses[-1].success(),
"Finish status should not be success")
def main(args):
parser = argparse.ArgumentParser(
description=
"Run all the jobs specified in jobs.json, or specify the names of the jobs to run."
)
parser.add_argument("run_path",
nargs="?",
help="Path where jobs.json is located")
parser.add_argument(
"job",
nargs="*",
help=
"One or more jobs to be executed from the jobs.json file. If no jobs are specified, all jobs will be executed.",
)
parsed_args = parser.parse_args(args[1:])
# If run_path is defined, enter into that directory
if parsed_args.run_path is not None:
if not os.path.exists(parsed_args.run_path):
sys.exit("No such directory: {}".format(parsed_args.run_path))
os.chdir(parsed_args.run_path)
ee_id = None
try:
with open(JOBS_FILE, "r") as json_file:
jobs_data = json.load(json_file)
ee_id = jobs_data.get("ee_id")
ee_token = jobs_data.get("ee_token")
ee_cert_path = jobs_data.get("ee_cert_path")
evaluator_url = jobs_data.get("dispatch_url")
except ValueError as e:
raise IOError("Job Runner cli failed to load JSON-file.{}".format(
str(e)))
is_interactive_run = len(parsed_args.job) > 0
reporters = _setup_reporters(is_interactive_run, ee_id, evaluator_url,
ee_token, ee_cert_path)
job_runner = JobRunner(jobs_data)
for job_status in job_runner.run(parsed_args.job):
logger.info(f"Job status: {job_status}")
for reporter in reporters:
reporter.report(job_status)
if isinstance(job_status, Finish) and not job_status.success():
pgid = os.getpgid(os.getpid())
os.killpg(pgid, signal.SIGKILL)
def test_run_output_rename(self):
job = {
"name": "TEST_JOB",
"executable": "/bin/mkdir",
"stdout": "out",
"stderr": "err",
}
joblist = [job, job, job, job, job]
jobm = JobRunner(create_jobs_json(joblist))
for status in enumerate(jobm.run([])):
if isinstance(status, Start):
self.assertEqual(status.job.std_err, "err.{}".format(status.job.index))
self.assertEqual(status.job.std_out, "out.{}".format(status.job.index))