def test_env_varlist(self):
varlist_string = "global_environment"
update_string = "global_update_path"
first = "FIRST"
second = "SECOND"
third = "THIRD"
first_value = "TheFirstValue"
second_value = "TheSecondValue"
third_value = "$FIRST:$SECOND"
third_value_correct = "%s:%s" % (first_value, second_value)
varlist = EnvironmentVarlist()
varlist[first] = first_value
varlist[second] = second_value
varlist[third] = third_value
self.assertEqual(len(varlist), 3)
with TestAreaContext("python/job_queue/env_varlist"):
forward_model = self.set_up_forward_model([])
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask, varlist)
config = load_configs(self.JOBS_JSON_FILE)
env_config = config[varlist_string]
self.assertEqual(first_value, env_config[first])
self.assertEqual(second_value, env_config[second])
self.assertEqual(third_value_correct, env_config[third])
update_config = config[update_string]
def test_transfer_arg_types(self):
with TestAreaContext(
"python/job_queue/forward_model_transfer_arg_types"):
with open("FWD_MODEL", "w") as f:
f.write("EXECUTABLE ls\n")
f.write("MIN_ARG 2\n")
f.write("MAX_ARG 6\n")
f.write("ARG_TYPE 0 INT\n")
f.write("ARG_TYPE 1 FLOAT\n")
f.write("ARG_TYPE 2 STRING\n")
f.write("ARG_TYPE 3 BOOL\n")
f.write("ARG_TYPE 4 RUNTIME_FILE\n")
f.write("ARG_TYPE 5 RUNTIME_INT\n")
job = ExtJob("FWD_MODEL", True)
ext_joblist = ExtJoblist()
ext_joblist.add_job(job.name(), job)
forward_model = ForwardModel(ext_joblist)
forward_model.add_job("FWD_MODEL")
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask,
EnvironmentVarlist())
config = load_configs(self.JOBS_JSON_FILE)
printed_job = config["jobList"][0]
self.assertEqual(printed_job["min_arg"], 2)
self.assertEqual(printed_job["max_arg"], 6)
self.assertEqual(printed_job["arg_types"], [
"INT", "FLOAT", "STRING", "BOOL", "RUNTIME_FILE", "RUNTIME_INT"
])
def run_all(self):
forward_model = self.set_up_forward_model(range(len(joblist)))
umask = 0
run_id = "run_all"
global_args = SubstitutionList()
varlist = EnvironmentVarlist()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask, varlist)
self.verify_json_dump(range(len(joblist)), global_args, umask, run_id)
def test_no_jobs(self):
with TestAreaContext("python/job_queue/forward_model_no_jobs"):
forward_model = self.set_up_forward_model([])
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
varlist = EnvironmentVarlist()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask, varlist)
self.verify_json_dump([], global_args, umask, run_id)
def test_one_job(self):
with TestAreaContext("python/job_queue/forward_model_one_job"):
for i in range(len(joblist)):
forward_model = self.set_up_forward_model([i])
run_id = "test_one_job"
umask = 11
global_args = SubstitutionList()
varlist = EnvironmentVarlist()
forward_model.formatted_fprintf(run_id, os.getcwd(),
"data_root", global_args,
umask, varlist)
self.verify_json_dump([i], global_args, umask, run_id)
def test_status_file(self):
with TestAreaContext("status_json"):
forward_model = self.set_up_forward_model()
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
varlist = EnvironmentVarlist()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask, varlist)
s = '{"start_time": null, "jobs": [{"status": "Success", "start_time": 1519653419.0, "end_time": 1519653419.0, "name": "SQUARE_PARAMS", "error": null, "current_memory_usage": 2000, "max_memory_usage": 3000}], "end_time": null, "run_id": ""}'
with open("status.json", "w") as f:
f.write(s)
status = ForwardModelStatus.try_load("")
for job in status.jobs:
self.assertTrue(isinstance(job.start_time, datetime.datetime))
self.assertTrue(isinstance(job.end_time, datetime.datetime))