def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str],
repos: List[Repo]) -> Dict[str, str]:
# pylint: disable=too-many-locals
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param repos: List of repos containing changed files.
:return: Dict of files and file contents for generated tasks.
"""
config_dict_of_suites_and_tasks = {}
task_id = selected_tests_variant_expansions[TASK_ID_EXPANSION]
revision_map = generate_revision_map_from_manifest(repos, task_id, evg_api)
changed_files = find_changed_files_in_repos(repos, revision_map)
changed_files = {
_remove_repo_path_prefix(file_path)
for file_path in changed_files
}
LOGGER.info("Found changed files", files=changed_files)
shrub_project = ShrubProject()
for build_variant_config in evg_conf.get_required_variants():
shrub_build_variant = BuildVariant(build_variant_config.name)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions,
build_variant_config, changed_files)
remove_task_configs_already_in_build(
task_configs, evg_api, build_variant_config,
selected_tests_variant_expansions["version_id"])
for task_config in task_configs.values():
Suite.reset_current_index()
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_build_variant,
config_options,
config_dict_of_suites_and_tasks)
shrub_project.add_build_variant(shrub_build_variant)
config_dict_of_suites_and_tasks[
"selected_tests_config.json"] = shrub_project.json()
return config_dict_of_suites_and_tasks
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str], repos: List[Repo],
origin_build_variants: List[str]) -> Dict[str, str]:
# pylint: disable=too-many-locals
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param repos: List of repos containing changed files.
:param origin_build_variants: Build variants to collect task info from.
:return: Dict of files and file contents for generated tasks.
"""
config_dict_of_suites_and_tasks = {}
changed_files = find_changed_files_in_repos(repos)
changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files}
LOGGER.debug("Found changed files", files=changed_files)
shrub_project = ShrubProject()
for build_variant in origin_build_variants:
shrub_build_variant = BuildVariant(build_variant)
build_variant_config = evg_conf.get_variant(build_variant)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions, build_variant_config,
changed_files)
for task_config in task_configs.values():
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_build_variant, config_options,
config_dict_of_suites_and_tasks)
shrub_project.add_build_variant(shrub_build_variant)
config_dict_of_suites_and_tasks["selected_tests_config.json"] = shrub_project.json()
return config_dict_of_suites_and_tasks
def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock):
evg_conf_mock = get_evergreen_config()
create_tests_by_task_mock.return_value = {}
expansions_file_data = get_expansions_data()
buildvariant_map = {
"enterprise-rhel-62-64-bit-inmem": "enterprise-rhel-62-64-bit-inmem-required",
"enterprise-rhel-62-64-bit-majority-read-concern-off":
"enterprise-rhel-62-64-bit-majority-read-concern-off-required",
} # yapf: disable
shrub_config = ShrubProject()
evergreen_api = MagicMock()
repo = MagicMock()
under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
buildvariant_map, [repo], evg_conf_mock)
self.assertEqual(shrub_config.as_dict(), EMPTY_PROJECT)
def test_evg_config_has_timeouts_for_repeated_suites(self):
options = self.generate_mock_options()
options.repeat_suites = 5
suites = self.generate_mock_suites(3)
build_variant = BuildVariant("variant")
generator = under_test.EvergreenConfigGenerator(
suites, options, MagicMock())
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
self.assertEqual(len(config["tasks"]), len(suites) + 1)
command1 = config["tasks"][0]["commands"][2]
self.assertIn(" --repeatSuites=5 ", command1["vars"]["resmoke_args"])
self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"])
timeout_cmd = config["tasks"][0]["commands"][0]
self.assertEqual("timeout.update", timeout_cmd["command"])
expected_timeout = under_test.calculate_timeout(
suites[0].max_runtime, 3) * 5
self.assertEqual(expected_timeout,
timeout_cmd["params"]["timeout_secs"])
expected_exec_timeout = under_test.calculate_timeout(
suites[0].get_runtime(), 3) * 5
self.assertEqual(expected_exec_timeout,
timeout_cmd["params"]["exec_timeout_secs"])
def run(self):
"""Generate resmoke suites that run within a target execution time and write to disk."""
LOGGER.debug("config options", config_options=self.config_options)
if not should_tasks_be_generated(self.evergreen_api,
self.config_options.task_id):
LOGGER.info(
"Not generating configuration due to previous successful generation."
)
return
suites = self.get_suites()
LOGGER.debug("Creating suites",
num_suites=len(suites),
task=self.config_options.task,
dir=self.config_options.generated_config_dir)
config_dict_of_suites = self.generate_suites_config(suites)
shrub_config = ShrubProject.empty()
shrub_config.add_build_variant(self.generate_task_config(suites))
config_dict_of_suites[self.config_options.task +
".json"] = shrub_config.json()
write_file_dict(self.config_options.generated_config_dir,
config_dict_of_suites)
def gen_lint_config():
targets = ["src", "test"]
max_hosts = 5
task_group_name = "lint group"
variant_name = "lint variant"
def define_task(target):
name = f"make-lint-{target}"
return Task(name, [FunctionCall("run-make", {"target": name})])
task_group = TaskGroup(
task_group_name,
[define_task(target) for target in targets],
max_hosts=max_hosts,
setup_group=[
git_get_project("src").set_type(CommandType.SYSTEM),
FunctionCall("set-up-credentials"),
],
teardown_group=[
FunctionCall("attach-test-results"),
FunctionCall("remove-test-results"),
],
)
variant = BuildVariant(variant_name).add_task_group(task_group)
project = ShrubProject({variant})
return project
def test_evg_config_is_created_without_multiversion(self):
build_variant = BuildVariant("build variant")
options = self._create_options_mock()
under_test.create_fuzzer_task(options, build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
self.assertEqual(options.num_tasks, len(config["tasks"]))
self.assertEqual("setup jstestfuzz",
config["tasks"][0]["commands"][1]["func"])
command1 = config["tasks"][0]["commands"][2]
self.assertIn(str(options.num_files),
command1["vars"]["jstestfuzz_vars"])
self.assertIn(options.npm_command, command1["vars"]["npm_command"])
self.assertEqual("run jstestfuzz", command1["func"])
buildvariant = config["buildvariants"][0]
self.assertEqual(options.variant, buildvariant["name"])
self.assertEqual(options.num_tasks, len(buildvariant["tasks"]))
self.assertEqual(
1, len(buildvariant["display_tasks"][0]["execution_tasks"]))
self.assertEqual(under_test.GEN_PARENT_TASK,
buildvariant["display_tasks"][0]["name"])
self.assertIn(options.name + "_gen",
buildvariant["display_tasks"][0]["execution_tasks"])
self.assertEqual(
options.num_tasks,
len(buildvariant["display_tasks"][1]["execution_tasks"]))
self.assertEqual(options.name,
buildvariant["display_tasks"][1]["name"])
def test_generate_evg_tasks_one_test_changed(self, create_tests_by_task_mock):
evg_conf_mock = get_evergreen_config()
create_tests_by_task_mock.return_value = {
"aggregation_mongos_passthrough": {
"display_task_name": "aggregation_mongos_passthrough",
"resmoke_args":
"--suites=aggregation_mongos_passthrough --storageEngine=wiredTiger",
"tests": ["jstests/aggregation/bugs/ifnull.js"],
"use_multiversion": None
}
} # yapf: disable
expansions_file_data = get_expansions_data()
buildvariant_map = {
"enterprise-rhel-62-64-bit-inmem": "enterprise-rhel-62-64-bit-inmem-required",
"enterprise-rhel-62-64-bit-majority-read-concern-off":
"enterprise-rhel-62-64-bit-majority-read-concern-off-required",
} # yapf: disable
shrub_config = ShrubProject.empty()
evergreen_api = MagicMock()
repo = MagicMock()
evergreen_api.test_stats_by_project.return_value = [
MagicMock(test_file="dir/test2.js", avg_duration_pass=10)
]
under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
buildvariant_map, [repo], evg_conf_mock)
generated_config = shrub_config.as_dict()
self.assertEqual(len(generated_config["buildvariants"]), 2)
first_generated_build_variant = generated_config["buildvariants"][0]
self.assertIn(first_generated_build_variant["name"], buildvariant_map.values())
self.assertEqual(first_generated_build_variant["display_tasks"][0]["name"], "burn_in_tests")
self.assertEqual(
first_generated_build_variant["display_tasks"][0]["execution_tasks"][0],
f"burn_in:aggregation_mongos_passthrough_0_{first_generated_build_variant['name']}")
def main(expansions_file: str = "expansions.yml",
output_file: str = "powercycle_tasks.json") -> None:
"""Generate multiple powercycle tasks to run in evergreen."""
config = make_config(expansions_file)
build_variant = BuildVariant(config.build_variant)
for task_name in config.task_names:
if "skip_compile" in task_name:
commands, task_dependency = get_skip_compile_setup_commands()
else:
commands, task_dependency = get_setup_commands()
commands.extend([
FunctionCall("set up remote credentials", config.remote_credentials_vars),
BuiltInCommand("timeout.update", config.timeout_params),
FunctionCall("set up EC2 instance", config.set_up_ec2_instance_vars),
FunctionCall("run powercycle test", config.run_powercycle_vars),
])
build_variant.display_task(
task_name, {
Task(
name_generated_task(task_name, index, config.num_tasks, config.build_variant),
commands, task_dependency)
for index in range(config.num_tasks)
}, distros=[config.distro])
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
write_file(output_file, shrub_project.json())
def create_generate_tasks_file(tests_by_task: Dict,
generate_config: GenerateConfig,
repeat_config: RepeatConfig,
evg_api: Optional[EvergreenApi],
task_prefix: str = 'burn_in',
include_gen_task: bool = True) -> str:
"""
Create an Evergreen generate.tasks file to run the given tasks and tests.
:param tests_by_task: Dictionary of tests and tasks to run.
:param generate_config: Information about how burn_in should generate tasks.
:param repeat_config: Information about how burn_in should repeat tests.
:param evg_api: Evergreen api.
:param task_prefix: Prefix to start generated task's name with.
:param include_gen_task: Should the generating task be included in the display task.
:returns: Configuration to pass to 'generate.tasks'.
"""
build_variant = BuildVariant(generate_config.run_build_variant)
create_generate_tasks_config(build_variant,
tests_by_task,
generate_config,
repeat_config,
evg_api,
include_gen_task=include_gen_task,
task_prefix=task_prefix)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
return shrub_project.json()
def _generate_evg_tasks(evergreen_api: EvergreenApi,
shrub_project: ShrubProject,
task_expansions: Dict[str, Any],
build_variant_map: Dict[str, str], repos: List[Repo],
evg_conf: EvergreenProjectConfig,
install_dir: str) -> None:
"""
Generate burn in tests tasks for a given shrub config and group of build variants.
:param evergreen_api: Evergreen.py object.
:param shrub_project: Shrub config object that the build variants will be built upon.
:param task_expansions: Dictionary of expansions for the running task.
:param build_variant_map: Map of base buildvariants to their generated buildvariant.
:param repos: Git repositories.
"""
for build_variant, run_build_variant in build_variant_map.items():
config_options = _get_config_options(task_expansions, build_variant,
run_build_variant)
task_id = task_expansions[TASK_ID_EXPANSION]
change_detector = EvergreenFileChangeDetector(task_id, evergreen_api,
os.environ)
changed_tests = change_detector.find_changed_tests(repos)
tests_by_task = create_tests_by_task(build_variant, evg_conf,
changed_tests, install_dir)
if tests_by_task:
shrub_build_variant = _generate_evg_build_variant(
evg_conf.get_variant(build_variant), run_build_variant,
task_expansions["build_variant"])
gen_config = GenerateConfig(
build_variant,
config_options.project,
run_build_variant,
config_options.distro,
include_gen_task=False).validate(evg_conf)
repeat_config = RepeatConfig(
repeat_tests_min=config_options.repeat_tests_min,
repeat_tests_max=config_options.repeat_tests_max,
repeat_tests_secs=config_options.repeat_tests_secs)
burn_in_generator = GenerateBurnInExecutor(gen_config,
repeat_config,
evergreen_api)
burn_in_generator.generate_tasks_for_variant(
tests_by_task, shrub_build_variant)
shrub_project.add_build_variant(shrub_build_variant)
def burn_in(task_expansions: Dict[str, Any], evg_conf: EvergreenProjectConfig,
evergreen_api: RetryingEvergreenApi, repos: Iterable[Repo]):
"""Execute Main program."""
shrub_project = ShrubProject.empty()
build_variant_map = _create_evg_build_variant_map(task_expansions,
evg_conf)
_generate_evg_tasks(evergreen_api, shrub_project, task_expansions,
build_variant_map, repos, evg_conf)
write_file_to_dir(CONFIG_DIRECTORY, CONFIG_FILE, shrub_project.json())
def test_evg_config_is_created_with_multiversion(self):
build_variant = BuildVariant("build variant")
options = self._create_options_mock()
options.use_multiversion = "/data/multiversion"
under_test.create_fuzzer_task(options, build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
self.assertEqual("do multiversion setup", config["tasks"][0]["commands"][2]["func"])
self.assertEqual("/data/multiversion",
config["tasks"][0]["commands"][5]["vars"]["task_path_suffix"])
def run(self) -> None:
"""Generate multiversion suites that run within a specified target execution time."""
if not generate_resmoke.should_tasks_be_generated(self.evg_api, self.options.task_id):
LOGGER.info("Not generating configuration due to previous successful generation.")
return
build_variant = BuildVariant(self.options.variant)
self.generate_evg_tasks(build_variant)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
write_file_to_dir(CONFIG_DIR, f"{self.task}.json", shrub_project.json())
def _generate_evg_tasks(evergreen_api: EvergreenApi,
shrub_project: ShrubProject,
task_expansions: Dict[str, Any],
build_variant_map: Dict[str,
str], repos: Iterable[Repo],
evg_conf: EvergreenProjectConfig) -> None:
"""
Generate burn in tests tasks for a given shrub config and group of build variants.
:param evergreen_api: Evergreen.py object.
:param shrub_project: Shrub config object that the build variants will be built upon.
:param task_expansions: Dictionary of expansions for the running task.
:param build_variant_map: Map of base buildvariants to their generated buildvariant.
:param repos: Git repositories.
"""
for build_variant, run_build_variant in build_variant_map.items():
config_options = _get_config_options(task_expansions, build_variant,
run_build_variant)
changed_tests = find_changed_tests(repos)
tests_by_task = create_tests_by_task(build_variant, evg_conf,
changed_tests)
if tests_by_task:
shrub_build_variant = _generate_evg_build_variant(
evg_conf.get_variant(build_variant), run_build_variant,
task_expansions["build_variant"])
gen_config = GenerateConfig(
build_variant, config_options.project, run_build_variant,
config_options.distro).validate(evg_conf)
repeat_config = RepeatConfig(
repeat_tests_min=config_options.repeat_tests_min,
repeat_tests_max=config_options.repeat_tests_max,
repeat_tests_secs=config_options.repeat_tests_secs)
create_generate_tasks_config(shrub_build_variant,
tests_by_task,
gen_config,
repeat_config,
evergreen_api,
include_gen_task=False)
shrub_project.add_build_variant(shrub_build_variant)
def test_with_large_distro(self):
build_variant = BuildVariant("build variant")
options = self._create_options_mock()
options.large_distro_name = "large build variant"
options.use_large_distro = True
under_test.create_fuzzer_task(options, build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
for variant in config["buildvariants"]:
for task in variant["tasks"]:
self.assertEqual(task["distros"], [options.large_distro_name])
def test_evg_config_does_not_overwrite_repeat_resmoke_arg_with_repeatSuites_default(self):
options = self.generate_mock_options()
options.resmoke_args = "resmoke_args --repeat=5"
suites = self.generate_mock_suites(1)
build_variant = BuildVariant("variant")
generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock())
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
command1 = config["tasks"][0]["commands"][2]
self.assertIn("--repeat=5", command1["vars"]["resmoke_args"])
self.assertNotIn("--repeatSuites=1", command1["vars"]["resmoke_args"])
def validate_task_generation_limit(shrub_project: ShrubProject) -> bool:
"""
Determine if this shrub configuration generates less than the limit.
:param shrub_project: Shrub configuration to validate.
:return: True if the configuration is under the limit.
"""
tasks_to_create = len(shrub_project.all_tasks())
if tasks_to_create > MAX_SHRUB_TASKS_FOR_SINGLE_TASK:
LOGGER.warning("Attempting to create more tasks than max, aborting",
tasks=tasks_to_create,
max=MAX_SHRUB_TASKS_FOR_SINGLE_TASK)
return False
return True
def test_timeout_info_not_included_if_use_default_timeouts_set(self):
suite_without_timing_info = 1
options = self.generate_mock_options()
suites = self.generate_mock_suites(3)
options.use_default_timeouts = True
build_variant = BuildVariant("variant")
generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock())
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
timeout_cmd = config["tasks"][suite_without_timing_info]["commands"][0]
self.assertNotIn("command", timeout_cmd)
self.assertEqual("do setup", timeout_cmd["func"])
def test_suites_without_enough_info_should_not_include_timeouts(self):
suite_without_timing_info = 1
options = self.generate_mock_options()
suites = self.generate_mock_suites(3)
suites[suite_without_timing_info].should_overwrite_timeout.return_value = False
build_variant = BuildVariant("variant")
generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock())
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
timeout_cmd = config["tasks"][suite_without_timing_info]["commands"][0]
self.assertNotIn("command", timeout_cmd)
self.assertEqual("do setup", timeout_cmd["func"])
def test_evg_config_can_use_large_distro(self):
options = self.generate_mock_options()
options.use_large_distro = "true"
options.large_distro_name = "large distro name"
suites = self.generate_mock_suites(3)
build_variant = BuildVariant("variant")
generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock())
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
self.assertEqual(len(config["tasks"]), len(suites) + 1)
self.assertEqual(options.large_distro_name,
config["buildvariants"][0]["tasks"][0]["distros"][0])
def test_evg_config_is_created(self):
options = self.generate_mock_options()
suites = self.generate_mock_suites(3)
build_variant = BuildVariant("variant")
generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock())
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
self.assertEqual(len(config["tasks"]), len(suites) + 1)
command1 = config["tasks"][0]["commands"][2]
self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"])
self.assertIn(" --originSuite=suite", command1["vars"]["resmoke_args"])
self.assertIn(options.run_multiple_jobs, command1["vars"]["run_multiple_jobs"])
self.assertEqual("run generated tests", command1["func"])
def create_generate_tasks_configuration(self, tests_by_task: Dict[str, TaskInfo]) -> str:
"""
Create the configuration with the configuration to generate the burn_in tasks.
:param tests_by_task: Dictionary of tasks and test to generate.
:return: Configuration to use to create generated tasks.
"""
build_variant = BuildVariant(self.generate_config.run_build_variant)
self.add_config_for_build_variant(build_variant, tests_by_task)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
return shrub_project.json()
def burn_in(task_expansions: Dict[str, Any], evg_conf: EvergreenProjectConfig,
evergreen_api: RetryingEvergreenApi, repos: List[Repo]):
"""
Execute main program.
:param task_expansions: Dictionary of expansions for the running task.
:param evg_conf: Evergreen configuration.
:param evergreen_api: Evergreen.py object.
:param repos: Git repositories.
"""
shrub_project = ShrubProject.empty()
build_variant_map = _create_evg_build_variant_map(task_expansions,
evg_conf)
_generate_evg_tasks(evergreen_api, shrub_project, task_expansions,
build_variant_map, repos, evg_conf)
write_file_to_dir(CONFIG_DIRECTORY, CONFIG_FILE, shrub_project.json())
def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None:
"""
Execute the given tests in the given tasks.
:param tests_by_task: Dictionary of tasks to run with tests to run in each.
"""
build_variant = BuildVariant(self.generate_config.run_build_variant)
self.generate_tasks_for_variant(tests_by_task, build_variant)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
assert self.generate_tasks_file is not None
if self.generate_tasks_file:
write_file(self.generate_tasks_file, shrub_project.json())
def test_no_suites_or_tasks_are_generated(self, generate_subsuites_mock,
selected_tests_config_options_mock):
generate_subsuites_mock.return_value.generate_suites_config.return_value = {}
def generate_task_config(shrub_config, suites):
pass
generate_subsuites_mock.return_value.generate_task_config.side_effect = generate_task_config
build_variant = BuildVariant("variant")
config_dict_of_suites_and_tasks = {}
under_test._update_config_with_task(
MagicMock(), build_variant, config_options=MagicMock(),
config_dict_of_suites_and_tasks=config_dict_of_suites_and_tasks)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
self.assertEqual(config_dict_of_suites_and_tasks, {})
self.assertEqual(shrub_project.as_dict(), empty_build_variant("variant"))
def test_build_variant_without_large_distro_defined_can_be_ignored(self):
options = self.generate_mock_options()
options.use_large_distro = "true"
options.large_distro_name = None
suites = self.generate_mock_suites(3)
build_variant = BuildVariant("variant")
generate_config = under_test.GenerationConfiguration(
build_variant_large_distro_exceptions={"variant"})
generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock(),
generate_config)
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
self.assertEqual(len(config["tasks"]), len(suites) + 1)
self.assertIsNone(config["buildvariants"][0]["tasks"][0].get("distros"))
def burn_in(task_expansions: Dict[str, Any], evg_conf: EvergreenProjectConfig,
evergreen_api: RetryingEvergreenApi, repos: List[Repo],
install_dir: str):
"""
Execute main program.
:param task_expansions: Dictionary of expansions for the running task.
:param evg_conf: Evergreen configuration.
:param evergreen_api: Evergreen.py object.
:param repos: Git repositories.
:param install_dir: path to bin directory of a testable installation
"""
shrub_project = ShrubProject.empty()
build_variant_map = _create_evg_build_variant_map(task_expansions)
_generate_evg_tasks(evergreen_api, shrub_project, task_expansions,
build_variant_map, repos, evg_conf, install_dir)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
write_file_to_dir(CONFIG_DIRECTORY, CONFIG_FILE, shrub_project.json())
def test_multiversion_path_is_used(self):
n_tasks = 1
n_tests = 1
build_variant = BuildVariant("variant")
gen_config = MagicMock(run_build_variant="variant", distro=None)
repeat_config = MagicMock()
tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
first_task = "task_0_gen"
multiversion_path = "multiversion_path"
tests_by_task[first_task]["use_multiversion"] = multiversion_path
create_generate_tasks_config(build_variant, tests_by_task, gen_config,
repeat_config, None)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
evg_config_dict = shrub_project.as_dict()
tasks = evg_config_dict["tasks"]
self.assertEqual(n_tasks * n_tests, len(tasks))
self.assertEqual(multiversion_path,
tasks[0]["commands"][2]["vars"]["task_path_suffix"])
def test_evg_config_is_created_with_diff_task_and_suite(self):
options = self.generate_mock_options()
options.task = "task"
options.display_task_name = "display task"
options.generate_display_task.return_value = DisplayTaskDefinition("task")
suites = self.generate_mock_suites(3)
build_variant = BuildVariant("variant")
generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock())
generator.generate_config(build_variant)
shrub_project = ShrubProject.empty().add_build_variant(build_variant)
config = shrub_project.as_dict()
self.assertEqual(len(config["tasks"]), len(suites) + 1)
display_task = config["buildvariants"][0]["display_tasks"][0]
self.assertEqual(options.display_task_name, display_task["name"])
task = config["tasks"][0]
self.assertIn(options.variant, task["name"])
self.assertIn(options.suite, task["commands"][2]["vars"]["resmoke_args"])