示例#1
0
    def test_transfer_arg_types(self):
        with TestAreaContext(
                "python/job_queue/forward_model_transfer_arg_types"):
            with open("FWD_MODEL", "w") as f:
                f.write("EXECUTABLE ls\n")
                f.write("MIN_ARG 2\n")
                f.write("MAX_ARG 6\n")
                f.write("ARG_TYPE 0 INT\n")
                f.write("ARG_TYPE 1 FLOAT\n")
                f.write("ARG_TYPE 2 STRING\n")
                f.write("ARG_TYPE 3 BOOL\n")
                f.write("ARG_TYPE 4 RUNTIME_FILE\n")
                f.write("ARG_TYPE 5 RUNTIME_INT\n")

            job = ExtJob("FWD_MODEL", True)

            ext_joblist = ExtJoblist()
            ext_joblist.add_job(job.name(), job)
            forward_model = ForwardModel(ext_joblist)
            forward_model.add_job("FWD_MODEL")

            run_id = "test_no_jobs_id"
            umask = 4
            global_args = SubstitutionList()

            forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
                                            global_args, umask,
                                            EnvironmentVarlist())
            config = load_configs(self.JOBS_JSON_FILE)
            printed_job = config["jobList"][0]
            self.assertEqual(printed_job["min_arg"], 2)
            self.assertEqual(printed_job["max_arg"], 6)
            self.assertEqual(printed_job["arg_types"], [
                "INT", "FLOAT", "STRING", "BOOL", "RUNTIME_FILE", "RUNTIME_INT"
            ])
示例#2
0
    def test_valid_args(self):
        arg_types = [ContentTypeEnum.CONFIG_FLOAT, ContentTypeEnum.CONFIG_INT, ContentTypeEnum.CONFIG_BOOL, ContentTypeEnum.CONFIG_STRING]
        run_arg_types = [ContentTypeEnum.CONFIG_RUNTIME_INT, ContentTypeEnum.CONFIG_RUNTIME_INT]

        arg_list = ["5.6", "65", "True", "car"]
        self.assertTrue( ExtJob.valid_args(arg_types, arg_list) )
        arg_list2 = ["True", "True", "8", "car"]
        self.assertFalse( ExtJob.valid_args(arg_types, arg_list2) )

        run_arg_list = ["Trjue", "76"]
        self.assertTrue( ExtJob.valid_args(run_arg_types, run_arg_list) )
        self.assertFalse( ExtJob.valid_args(run_arg_types, run_arg_list, True) )
示例#3
0
def _generate_job(name, executable, target_file, error_file, start_file,
                  stdout, stderr, stdin, environment, arglist,
                  max_running_minutes, max_running, license_root_path,
                  private):
    config_file = (DEFAULT_NAME if name is None else name)
    environment = (None if environment is None else environment.keys() +
                   environment.values())

    values = [
        str_none_sensitive(max_running), stdin, stdout, stderr, executable,
        target_file, error_file, start_file,
        None if arglist is None else " ".join(arglist),
        None if environment is None else " ".join(environment),
        str_none_sensitive(max_running_minutes)
    ]

    conf = open(config_file, "w")
    for key, val in zip(ext_job_keywords, values):
        if val is not None:
            conf.write("%s %s\n" % (key, val))
    conf.close()

    exec_file = open(executable, "w")
    exec_file.close()

    ext_job = ExtJob(config_file, private, name, license_root_path)
    os.unlink(config_file)
    os.unlink(executable)

    return ext_job
示例#4
0
    def _make_ensemble_builder(tmpdir, num_reals, num_jobs):
        builder = create_ensemble_builder()
        with tmpdir.as_cwd():
            ext_job_list = []
            for job_index in range(0, num_jobs):
                ext_job_config = Path(tmpdir) / f"EXT_JOB_{job_index}"
                with open(ext_job_config, "w") as f:
                    f.write(f"EXECUTABLE ext_{job_index}.py\n")

                ext_job_exec = Path(tmpdir) / f"ext_{job_index}.py"
                with open(ext_job_exec, "w") as f:
                    f.write(
                        "#!/usr/bin/env python\n"
                        'if __name__ == "__main__":\n'
                        f'    print("stdout from {job_index}")\n'
                    )

                ext_job_list.append(
                    ExtJob(str(ext_job_config), False, name=f"ext_job_{job_index}")
                )

            for iens in range(0, num_reals):
                run_path = Path(tmpdir / f"real_{iens}")
                os.mkdir(run_path)

                with open(run_path / "jobs.json", "w") as f:
                    json.dump(
                        {
                            "jobList": [
                                _dump_ext_job(ext_job, index)
                                for index, ext_job in enumerate(ext_job_list)
                            ],
                            "umask": "0022",
                        },
                        f,
                    )

                step = (
                    create_step_builder()
                    .set_id("0")
                    .set_job_name("job dispatch")
                    .set_job_script("job_dispatch.py")
                    .set_max_runtime(10000)
                    .set_run_arg(Mock(iens=iens))
                    .set_done_callback(lambda _: True)
                    .set_exit_callback(lambda _: True)
                    # the first callback_argument is expected to be a run_arg
                    # from the run_arg, the queue wants to access the iens prop
                    .set_callback_arguments([])
                    .set_run_path(str(run_path))
                    .set_num_cpu(1)
                    .set_name("dummy step")
                    .set_dummy_io()
                )

                for index, job in enumerate(ext_job_list):
                    step.add_job(
                        create_legacy_job_builder()
                        .set_id(index)
                        .set_name(f"dummy job {index}")
                        .set_ext_job(job)
                    )

                builder.add_realization(
                    create_realization_builder()
                    .active(True)
                    .set_iens(iens)
                    .add_step(step)
                )

        analysis_config = Mock()
        analysis_config.get_stop_long_running = Mock(return_value=False)

        ecl_config = Mock()
        ecl_config.assert_restart = Mock()

        builder.set_legacy_dependencies(
            queue_config,
            analysis_config,
        )
        return builder
示例#5
0
    def test_load_forward_model(self):
        with self.assertRaises(IOError):
            job = ExtJob("CONFIG_FILE", True)

        with TestAreaContext("python/job_queue/forward_model1"):
            create_valid_config("CONFIG")
            job = ExtJob("CONFIG", True)
            self.assertEqual(job.name(), "CONFIG")
            self.assertEqual(job.get_stdout_file(), None)
            self.assertEqual(job.get_stderr_file(), None)

            self.assertEqual(job.get_executable(),
                             os.path.join(os.getcwd(), "script.sh"))
            self.assertTrue(os.access(job.get_executable(), os.X_OK))

            self.assertEqual(job.min_arg, -1)

            job = ExtJob("CONFIG", True, name="Job")
            self.assertEqual(job.name(), "Job")
            pfx = "ExtJob("
            self.assertEqual(pfx, repr(job)[:len(pfx)])

        with TestAreaContext("python/job_queue/forward_model1a"):
            create_upgraded_valid_config("CONFIG")
            job = ExtJob("CONFIG", True)
            self.assertEqual(job.min_arg, 2)
            self.assertEqual(job.max_arg, 7)
            argTypes = job.arg_types
            self.assertEqual(
                argTypes,
                [
                    ContentTypeEnum.CONFIG_INT,
                    ContentTypeEnum.CONFIG_FLOAT,
                    ContentTypeEnum.CONFIG_STRING,
                    ContentTypeEnum.CONFIG_BOOL,
                    ContentTypeEnum.CONFIG_RUNTIME_FILE,
                    ContentTypeEnum.CONFIG_RUNTIME_INT,
                    ContentTypeEnum.CONFIG_STRING,
                ],
            )

        with TestAreaContext("python/job_queue/forward_model2"):
            create_config_missing_executable("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)

        with TestAreaContext("python/job_queue/forward_model3"):
            create_config_missing_EXECUTABLE("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)

        with TestAreaContext("python/job_queue/forward_model4"):
            create_config_executable_directory("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)

        with TestAreaContext("python/job_queue/forward_model5"):
            create_config_foreign_file("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)
示例#6
0
    def test_load_forward_model(self):
        with self.assertRaises(IOError):
            job = ExtJob("CONFIG_FILE", True)

        with TestAreaContext("python/job_queue/forward_model1"):
            create_valid_config("CONFIG")
            job = ExtJob("CONFIG", True)
            self.assertEqual(job.name(), "CONFIG")

            self.assertEqual(job.get_executable(),
                             os.path.join(os.getcwd(), "script.sh"))
            self.assertTrue(os.access(job.get_executable(), os.X_OK))

            job = ExtJob("CONFIG", True, name="Job")
            self.assertEqual(job.name(), "Job")
            pfx = 'ExtJob('
            self.assertEqual(pfx, repr(job)[:len(pfx)])

        with TestAreaContext("python/job_queue/forward_model2"):
            create_config_missing_executable("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)

        with TestAreaContext("python/job_queue/forward_model3"):
            create_config_missing_EXECUTABLE("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)

        with TestAreaContext("python/job_queue/forward_model4"):
            create_config_executable_directory("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)

        with TestAreaContext("python/job_queue/forward_model5"):
            create_config_foreign_file("CONFIG")
            with self.assertRaises(ValueError):
                job = ExtJob("CONFIG", True)