def task_def_to_gen_params(self, task_def: Task, build_variant: str) -> ResmokeGenTaskParams:
"""
Build parameters for how a task should be generated based on its task definition.
:param task_def: Task definition in evergreen project config.
:param build_variant: Name of Build Variant being generated.
:return: Parameters for how task should be generated.
"""
run_func = task_def.generate_resmoke_tasks_command
run_vars = run_func.get("vars", {})
repeat_suites = 1
if self.evg_expansions.resmoke_repeat_suites:
repeat_suites = self.evg_expansions.resmoke_repeat_suites
return ResmokeGenTaskParams(
use_large_distro=run_vars.get("use_large_distro"),
require_multiversion_setup=task_def.require_multiversion_setup(),
require_multiversion_version_combo=task_def.require_multiversion_version_combo(),
repeat_suites=repeat_suites,
resmoke_args=run_vars.get("resmoke_args"),
resmoke_jobs_max=run_vars.get("resmoke_jobs_max"),
large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"),
config_location=self.evg_expansions.config_location(),
dependencies=self.determine_task_dependencies(task_def),
)
def get_evg_task_config(task: Task, build_variant_config: Variant) -> Dict[str, Any]:
"""
Look up task config of the task to be generated.
:param task: Task to get info for.
:param build_variant_config: Config of build variant to collect task info from.
:return: Task configuration values.
"""
LOGGER.info("Calculating evg_task_config values for task", task=task.name)
task_vars = {}
for run_task_func in POSSIBLE_RUN_TASK_FUNCS:
task_def = task.find_func_command(run_task_func)
if task_def:
task_vars = task_def.get("vars", {})
break
task_vars.update({"suite": task.get_suite_name()})
task_name = task.name[:-4] if task.name.endswith("_gen") else task.name
return {
"task_name": task_name,
"build_variant": build_variant_config.name,
**task_vars,
"large_distro_name": build_variant_config.expansion("large_distro_name"),
}
def task_def_to_fuzzer_params(self, task_def: Task,
build_variant: str) -> FuzzerGenTaskParams:
"""
Build parameters for how a fuzzer task should be generated based on its task definition.
:param task_def: Task definition in evergreen project config.
:param build_variant: Name of Build Variant being generated.
:return: Parameters for how a fuzzer task should be generated.
"""
variant = self.evg_project_config.get_variant(build_variant)
run_vars = task_def.generate_resmoke_tasks_command.get("vars", {})
run_vars = {
k: translate_run_var(v, variant)
for k, v in run_vars.items()
}
return FuzzerGenTaskParams(
task_name=remove_gen_suffix(task_def.name),
variant=build_variant,
suite=run_vars.get("suite"),
num_files=int(run_vars.get("num_files")),
num_tasks=int(run_vars.get("num_tasks")),
resmoke_args=run_vars.get("resmoke_args"),
npm_command=run_vars.get("npm_command", "jstestfuzz"),
jstestfuzz_vars=run_vars.get("jstestfuzz_vars", ""),
continue_on_failure=run_vars.get("continue_on_failure"),
resmoke_jobs_max=run_vars.get("resmoke_jobs_max"),
should_shuffle=run_vars.get("should_shuffle"),
timeout_secs=run_vars.get("timeout_secs"),
require_multiversion_setup=task_def.require_multiversion_setup(),
use_large_distro=run_vars.get("use_large_distro", False),
large_distro_name=self.get_build_variant_expansion(
build_variant, "large_distro_name"),
config_location=self.evg_expansions.config_location(),
)
def generate_build_variant(self, build_variant_config: Variant,
changed_files: Set[str],
builder: EvgConfigBuilder) -> None:
"""
Generate the selected tasks on the specified build variant.
:param build_variant_config: Configuration of build variant to generate.
:param changed_files: List of file changes to determine what to run.
:param builder: Builder to create new configuration.
"""
build_variant_name = build_variant_config.name
LOGGER.info("Generating build variant",
build_variant=build_variant_name)
task_configs = self.get_task_config(build_variant_config,
changed_files)
for task_config in task_configs.values():
task_def = Task(task_config)
test_filter = None
if "selected_tests_to_run" in task_config:
test_filter = partial(
filter_set, input_set=task_config["selected_tests_to_run"])
split_params = SuiteSplitParameters(
build_variant=build_variant_name,
task_name=task_config["task_name"],
suite_name=task_config.get("suite", task_config["task_name"]),
filename=task_config.get("suite", task_config["task_name"]),
test_file_filter=test_filter,
is_asan=build_variant_config.is_asan_build(),
)
gen_params = ResmokeGenTaskParams(
use_large_distro=task_config.get("use_large_distro", False),
large_distro_name=task_config.get("large_distro_name"),
require_multiversion_setup=task_def.require_multiversion_setup(
),
require_multiversion_version_combo=False,
repeat_suites=task_config.get("repeat_suites", 1),
resmoke_args=task_config.get("resmoke_args", ""),
resmoke_jobs_max=task_config.get("resmoke_jobs_max"),
config_location=self.evg_expansions.get_config_location(),
dependencies={
depends_on["name"]
for depends_on in task_def.depends_on
},
)
builder.generate_suite(split_params, gen_params)
def build_mock_task(name, run_vars=None):
config = {
"name":
name, "commands": [
{"func": "do setup"},
{
"func": "generate resmoke tasks",
"vars": run_vars if run_vars else {},
},
]
}
return Task(config)
def build_mock_task(name, run_vars=None, depends_on=None):
config = {
"name":
name,
"commands": [
{
"func": "do setup"
},
{
"func": "generate resmoke tasks",
"vars": run_vars if run_vars else {},
},
]
}
if depends_on is not None:
config["depends_on"] = depends_on
return Task(config)
def get_evg_task_config(task: Task,
build_variant_config: Variant) -> Dict[str, Any]:
"""
Look up task config of the task to be generated.
:param task: Task to get info for.
:param build_variant_config: Config of build variant to collect task info from.
:return: Task configuration values.
"""
LOGGER.info("Calculating evg_task_config values for task",
task=task.name)
task_vars = {}
for run_task_func in POSSIBLE_RUN_TASK_FUNCS:
task_def = task.find_func_command(run_task_func)
if task_def:
task_vars = task_def["vars"]
break
suite_name = ResmokeArgs.get_arg(task_vars["resmoke_args"], "suites")
if suite_name:
task_vars.update({"suite": suite_name})
# the suites argument will run all tests in a suite even when individual
# tests are specified in resmoke_args, so we remove it
resmoke_args_without_suites = ResmokeArgs.remove_arg(
task_vars["resmoke_args"], "suites")
task_vars["resmoke_args"] = resmoke_args_without_suites
task_name = task.name[:-4] if task.name.endswith("_gen") else task.name
return {
"task_name":
task_name,
"build_variant":
build_variant_config.name,
**task_vars,
"large_distro_name":
build_variant_config.expansion("large_distro_name"),
}