def main():
"""Gather information about how tags are used in evergreen tasks."""
options = parse_command_line()
evg_config = evergreen.parse_evergreen_file(options.evergreen_file)
if options.list_tags:
list_all_tags(evg_config)
if options.tasks_for_tag:
list_tasks_with_tag(evg_config, options.tasks_for_tag)
def _check_etc_evergreen_for_bypass(path, build_variant):
"""
Check if changes to etc/evergreen can be allowed to bypass compile.
:param path: Path to etc/evergreen file.
:param build_variant: Build variant to check.
:return: True if changes can bypass compile.
"""
variant_before = _get_original_etc_evergreen(path).get_variant(
build_variant)
variant_after = parse_evergreen_file(path).get_variant(build_variant)
for expansion in EXPANSIONS_TO_CHECK:
if variant_before.expansion(expansion) != variant_after.expansion(
expansion):
return False
return True
def dependencies(binder: inject.Binder) -> None:
binder.bind(EvgExpansions, evg_expansions)
binder.bind(
SuiteSplitConfig,
evg_expansions.build_suite_split_config(start_date, end_date))
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(GenTaskOptions,
evg_expansions.build_evg_config_gen_options())
binder.bind(EvergreenApi,
RetryingEvergreenApi.get_api(config_file=evg_api_config))
binder.bind(EvergreenProjectConfig,
parse_evergreen_file(evg_project_config))
binder.bind(GenerationConfiguration,
GenerationConfiguration.from_yaml_file())
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
def main():
"""Gather information about how tags are used in evergreen tasks."""
options = parse_command_line()
evg_config = evergreen.parse_evergreen_file(options.evergreen_file)
if options.list_tags:
list_all_tags(evg_config)
if options.list_variants_and_tasks:
list_all_variants_and_tasks(evg_config)
if options.list_tasks:
list_all_tasks(evg_config, options.list_tasks)
if options.tasks_for_tag:
list_tasks_with_tag(evg_config, options.tasks_for_tag,
options.tasks_for_tag_filter)
def main():
"""Execute Main program."""
options, args = parse_command_line()
resmoke_cmd = _set_resmoke_cmd(options, args)
# Load the dict of tests to run.
if options.test_list_file:
tests_by_task = _load_tests_file(options.test_list_file)
# If there are no tests to run, carry on.
if tests_by_task is None:
test_results = {"failures": 0, "results": []}
_write_json_file(test_results, options.report_file)
sys.exit(0)
# Run the executor finder.
else:
# Parse the Evergreen project configuration file.
evergreen_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
changed_tests = find_changed_tests(options.branch, options.base_commit,
options.max_revisions, options.buildvariant,
options.check_evergreen)
exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE)
changed_tests = filter_tests(changed_tests, exclude_tests)
if changed_tests:
suites = resmokelib.suitesconfig.get_suites(suite_files=SUITE_FILES,
test_files=changed_tests)
tests_by_executor = create_executor_list(suites, exclude_suites)
tests_by_task = create_task_list(evergreen_conf, options.buildvariant,
tests_by_executor, exclude_tasks)
else:
print("No new or modified tests found.")
tests_by_task = {}
if options.test_list_outfile:
_write_json_file(tests_by_task, options.test_list_outfile)
if options.generate_tasks_file:
create_generate_tasks_file(options, tests_by_task)
else:
run_tests(options.no_exec, tests_by_task, resmoke_cmd, options.report_file)
def _create_evg_buildvariant_map(expansions_file_data):
"""
Generate relationship of base buildvariant to generated buildvariant.
:param expansions_file_data: Config data file to use.
:return: Map of base buildvariants to their generated buildvariants.
"""
burn_in_tags_gen_variant = expansions_file_data["build_variant"]
evergreen_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
burn_in_tags_gen_variant_config = evergreen_conf.get_variant(
burn_in_tags_gen_variant)
burn_in_tag_buildvariants = burn_in_tags_gen_variant_config.expansions.get(
"burn_in_tag_buildvariants")
if burn_in_tag_buildvariants:
return {
base_variant: f"{base_variant}-required"
for base_variant in burn_in_tag_buildvariants.split(" ")
}
return {}
def main(evergreen_api, repo):
"""Execute Main program."""
parser = argparse.ArgumentParser(description=main.__doc__)
parser.add_argument(
"--expansion-file",
dest="expansion_file",
type=str,
help="Location of expansions file generated by evergreen.")
cmd_line_options = parser.parse_args()
expansions_file_data = read_config.read_config_file(
cmd_line_options.expansion_file)
shrub_config = Configuration()
evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
build_variant_map = _create_evg_build_variant_map(expansions_file_data,
evg_conf)
_generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
build_variant_map, repo, evg_conf)
_write_to_file(shrub_config)
def dependencies(binder: inject.Binder) -> None:
binder.bind(EvgExpansions, evg_expansions)
binder.bind(EvergreenApi,
RetryingEvergreenApi.get_api(config_file=evg_api_config))
binder.bind(EvergreenProjectConfig,
parse_evergreen_file(EVERGREEN_FILE))
binder.bind(SelectedTestsClient,
SelectedTestsClient.from_file(selected_tests_config))
binder.bind(
SuiteSplitConfig,
evg_expansions.build_suite_split_config(start_date, end_date))
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options())
binder.bind(
GenerationConfiguration,
GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE))
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
def main(
verbose: bool,
expansion_file: str,
evg_api_config: str,
build_variant: str,
selected_tests_config: str,
):
"""
Select tasks to be run based on changed files in a patch build.
:param verbose: Log extra debug information.
:param expansion_file: Configuration file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param build_variant: Build variant to query tasks from.
:param selected_tests_config: Location of config file to connect to elected-tests service.
"""
_configure_logging(verbose)
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
selected_tests_service = SelectedTestsService.from_file(
selected_tests_config)
repo = Repo(".")
changed_files = find_changed_files(repo)
buildscripts.resmokelib.parser.set_options()
LOGGER.debug("Found changed files", files=changed_files)
related_test_files = _find_related_test_files(selected_tests_service,
changed_files)
LOGGER.debug("related test files found",
related_test_files=related_test_files)
if related_test_files:
tests_by_task = create_task_list_for_tests(related_test_files,
build_variant, evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
config_dict_of_generated_tasks = _generate_shrub_config(
evg_api, evg_conf, expansion_file, tests_by_task, build_variant)
write_file_dict(SELECTED_TESTS_CONFIG_DIR,
config_dict_of_generated_tasks)
def validate_options(parser, options):
"""Validate command line options."""
if options.repeat_tests_max:
if options.repeat_tests_secs is None:
parser.error("Must specify --repeatTestsSecs with --repeatTestsMax")
if options.repeat_tests_min and options.repeat_tests_min > options.repeat_tests_max:
parser.error("--repeatTestsSecsMin is greater than --repeatTestsMax")
if options.repeat_tests_min and options.repeat_tests_secs is None:
parser.error("Must specify --repeatTestsSecs with --repeatTestsMin")
if options.repeat_tests_num and options.repeat_tests_secs:
parser.error("Cannot specify --repeatTests and --repeatTestsSecs")
if options.test_list_file is None and options.buildvariant is None:
parser.error("Must specify --buildVariant to find changed tests")
if options.buildvariant:
evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
if not evg_conf.get_variant(options.buildvariant):
parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format(
options.buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names))))
def main():
# type: () -> None
"""Execute Main Entry point."""
parser = argparse.ArgumentParser(description='MongerDB CryptD Check Tool.')
parser.add_argument('file', type=str, help="etc/evergreen.yml file")
parser.add_argument('--variant',
type=str,
help="Build variant to check for")
args = parser.parse_args()
expected_variants = read_variable_from_yml(args.file, MONGOCRYPTD_VARIANTS)
if not expected_variants:
print("ERROR: Could not find node %s in file '%s'" %
(MONGOCRYPTD_VARIANTS, args.file),
file=sys.stderr)
sys.exit(1)
evg_config = parse_evergreen_file(args.file)
if can_validation_be_skipped(evg_config, args.variant):
print(f"Skipping validation on buildvariant {args.variant}")
sys.exit(0)
if args.variant not in expected_variants:
print("ERROR: Expected to find variant %s in list %s" %
(args.variant, expected_variants),
file=sys.stderr)
print(
"ERROR: Please add the build variant %s to the %s list in '%s'" %
(args.variant, MONGOCRYPTD_VARIANTS, args.file),
file=sys.stderr)
sys.exit(1)
sys.exit(0)
def main(
verbose: bool,
expansion_file: str,
evg_api_config: str,
selected_tests_config: str,
):
"""
Select tasks to be run based on changed files in a patch build.
:param verbose: Log extra debug information.
:param expansion_file: Configuration file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param selected_tests_config: Location of config file to connect to elected-tests service.
"""
_configure_logging(verbose)
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
selected_tests_service = SelectedTestsService.from_file(
selected_tests_config)
repo = Repo(".")
changed_files = find_changed_files(repo)
buildscripts.resmokelib.parser.set_options()
LOGGER.debug("Found changed files", files=changed_files)
selected_tests_variant_expansions = read_config.read_config_file(
expansion_file)
origin_build_variants = selected_tests_variant_expansions[
"selected_tests_buildvariants"].split(" ")
config_dict_of_suites_and_tasks = run(evg_api, evg_conf,
selected_tests_service,
selected_tests_variant_expansions,
changed_files, origin_build_variants)
write_file_dict(SELECTED_TESTS_CONFIG_DIR, config_dict_of_suites_and_tasks)
def main():
"""Execute Main program."""
values, args = parse_command_line()
# If a resmoke.py command wasn't passed in, use a simple version.
if not args:
args = ["python", "buildscripts/resmoke.py", "--repeat=2"]
# Load the dict of tests to run.
if values.test_list_file:
tests_by_task = _load_tests_file(values.test_list_file)
# If there are no tests to run, carry on.
if tests_by_task is None:
test_results = {"failures": 0, "results": []}
_write_report_file(test_results, values.report_file)
sys.exit(0)
# Run the executor finder.
else:
# Parse the Evergreen project configuration file.
evergreen_conf = evergreen.parse_evergreen_file(values.evergreen_file)
if values.buildvariant is None:
print "Option buildVariant must be specified to find changed tests.\n", \
"Select from the following: \n" \
"\t", "\n\t".join(sorted(evergreen_conf.variant_names))
sys.exit(1)
changed_tests = find_changed_tests(values.branch, values.base_commit, values.max_revisions,
values.buildvariant, values.check_evergreen)
exclude_suites, exclude_tasks, exclude_tests = find_exclude_tests(values.selector_file)
changed_tests = filter_tests(changed_tests, exclude_tests)
# If there are no changed tests, exit cleanly.
if not changed_tests:
print "No new or modified tests found."
if values.test_list_outfile is not None:
_write_report_file({}, values.test_list_outfile)
sys.exit(0)
suites = resmokelib.suitesconfig.get_suites(
suite_files=values.suite_files.split(","), test_files=changed_tests)
tests_by_executor = create_executor_list(suites, exclude_suites)
tests_by_task = create_task_list(evergreen_conf, values.buildvariant, tests_by_executor,
exclude_tasks)
if values.test_list_outfile is not None:
_write_report_file(tests_by_task, values.test_list_outfile)
# If we're not in noExec mode, run the tests.
if not values.no_exec:
test_results = {"failures": 0, "results": []}
for task in sorted(tests_by_task):
resmoke_cmd = copy.deepcopy(args)
resmoke_cmd.extend(shlex.split(tests_by_task[task]["resmoke_args"]))
resmoke_cmd.extend(tests_by_task[task]["tests"])
try:
subprocess.check_call(resmoke_cmd, shell=False)
except subprocess.CalledProcessError as err:
print "Resmoke returned an error with task:", task
_save_report_data(test_results, values.report_file, task)
_write_report_file(test_results, values.report_file)
sys.exit(err.returncode)
_save_report_data(test_results, values.report_file, task)
_write_report_file(test_results, values.report_file)
sys.exit(0)
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, evg_api_config, verbose, task_id, revision,
build_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
enable_logging(verbose)
burn_in_config = BurnInConfig(build_variant=build_variant,
build_id=build_id,
revision=revision)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
generate_config.validate(evg_conf)
gen_task_options = GenTaskOptions(
create_misc_suite=False,
is_patch=True,
generated_config_dir=DEFAULT_CONFIG_DIR,
use_default_timeouts=False,
)
split_task_options = SuiteSplitConfig(
evg_project=project,
target_resmoke_time=60,
max_sub_suites=100,
max_tests_per_suite=1,
start_date=datetime.utcnow(),
end_date=datetime.utcnow(),
default_to_fallback=True,
)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
def dependencies(binder: inject.Binder) -> None:
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
binder.bind(SuiteSplitConfig, split_task_options)
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(EvergreenProjectConfig, evg_conf)
binder.bind(GenTaskOptions, gen_task_options)
binder.bind(EvergreenApi, evg_api)
binder.bind(GenerationConfiguration,
GenerationConfiguration.from_yaml_file())
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
binder.bind(EvergreenFileChangeDetector,
EvergreenFileChangeDetector(task_id, evg_api))
binder.bind(BurnInConfig, burn_in_config)
inject.configure(dependencies)
burn_in_orchestrator = MultiversionBurnInOrchestrator() # pylint: disable=no-value-for-parameter
burn_in_orchestrator.generate_tests(repos, generate_config,
generate_tasks_file)
def setUpClass(cls):
cls.conf = _evergreen.parse_evergreen_file(TEST_FILE_PATH,
evergreen_binary=None)
def test_invalid_path(self):
invalid_path = "non_existing_file"
with self.assertRaises(IOError):
_evergreen.parse_evergreen_file(invalid_path,
evergreen_binary=None)
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, no_exec, repeat_tests_num, repeat_tests_min,
repeat_tests_max, repeat_tests_secs, resmoke_args, local_mode,
evg_api_config, verbose, use_multiversion, task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
burn_in_tests detects jstests that are new or changed since the last git command and then
runs those tests in a loop to validate their reliability.
The `--repeat-*` arguments allow configuration of how burn_in_tests repeats tests. Tests can
either be repeated a specified number of times with the `--repeat-tests` option, or they can
be repeated for a certain time period with the `--repeat-tests-secs` option.
When the `--use-multiversion` flag is set to True, burn_in_tests will run new or changed tests
against the appropriate generated multiversion suites. The purpose of these tests are to signal
bugs in the generated multiversion suites as these tasks are excluded from the required build
variants and are only run in certain daily build variants. As such, we only expect the burn-in
multiversion tests to be run once for each binary version configuration, and `--repeat-*`
arguments should be None when `--use-multiversion` is True.
There are two modes that burn_in_tests can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param no_exec: Just perform test discover, do not execute the tests.
:param repeat_tests_num: Repeat each test this number of times.
:param repeat_tests_min: Repeat each test at least this number of times.
:param repeat_tests_max: Once this number of repetitions has been reached, stop repeating.
:param repeat_tests_secs: Continue repeating tests for this number of seconds.
:param resmoke_args: Arguments to pass through to resmoke.
:param local_mode: Don't call out to the evergreen API (used for testing).
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
# When --
repeat_config = RepeatConfig(repeat_tests_secs=repeat_tests_secs,
repeat_tests_min=repeat_tests_min,
repeat_tests_max=repeat_tests_max,
repeat_tests_num=repeat_tests_num).validate(use_multiversion) # yapf: disable
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id).validate(evg_conf, use_multiversion,
local_mode) # yapf: disable
evg_api = _get_evg_api(evg_api_config, local_mode)
repo = Repo(".")
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
tests_by_task = create_tests_by_task(build_variant, repo, evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
if use_multiversion:
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_TAG)
# We expect the number of suites with MULTIVERSION_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include BURN_IN_CONFIG_KEY
# as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key_and_value(
MULTIVERSION_CONFIG_KEY, True)
assert len(multiversion_tasks) == len(multiversion_suites)
json_config = create_generate_tasks_file(
tests_by_task,
generate_config,
repeat_config,
evg_api,
use_multiversion=use_multiversion)
_write_json_file(json_config, generate_tasks_file)
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")
def get_evergreen_config() -> EvergreenProjectConfig:
return _evergreen.parse_evergreen_file(TEST_FILE_PATH,
evergreen_binary=None)
def check_variant(buildvariant, parser):
"""Check if the buildvariant is found in the evergreen file."""
evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
if not evg_conf.get_variant(buildvariant):
parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format(
buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names))))
def get_evergreen_config():
return _evergreen.parse_evergreen_file(TEST_FILE_PATH,
evergreen_binary=None)
def get_evergreen_config(config_file_path):
evergreen_home = os.path.expanduser(os.path.join("~", "evergreen"))
if os.path.exists(evergreen_home):
return parse_evergreen_file(config_file_path, evergreen_home)
return parse_evergreen_file(config_file_path)
def main(build_variant: str, run_build_variant: str, distro: str, project: str,
generate_tasks_file: str, repeat_tests_num: Optional[int],
repeat_tests_min: Optional[int], repeat_tests_max: Optional[int],
repeat_tests_secs: Optional[int], evg_api_config: str, verbose: bool,
task_id: str):
"""
Run new or changed tests in repeated mode to validate their stability.
burn_in_tests detects jstests that are new or changed since the last git command and then
runs those tests in a loop to validate their reliability.
The `--origin-rev` argument allows users to specify which revision should be used as the last
git command to compare against to find changed files. If the `--origin-rev` argument is provided,
we find changed files by comparing your latest changes to this revision. If not provided, we
find changed test files by comparing your latest changes to HEAD. The revision provided must
be a revision that exists in the mongodb repository.
The `--repeat-*` arguments allow configuration of how burn_in_tests repeats tests. Tests can
either be repeated a specified number of times with the `--repeat-tests` option, or they can
be repeated for a certain time period with the `--repeat-tests-secs` option.
Specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param repeat_tests_num: Repeat each test this number of times.
:param repeat_tests_min: Repeat each test at least this number of times.
:param repeat_tests_max: Once this number of repetitions has been reached, stop repeating.
:param repeat_tests_secs: Continue repeating tests for this number of seconds.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
:param task_id: Id of evergreen task being run in.
"""
_configure_logging(verbose)
repeat_config = RepeatConfig(repeat_tests_secs=repeat_tests_secs,
repeat_tests_min=repeat_tests_min,
repeat_tests_max=repeat_tests_max,
repeat_tests_num=repeat_tests_num) # yapf: disable
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
generate_config.validate(evg_conf)
burn_in(task_id, build_variant, generate_config, repeat_config, evg_api,
evg_conf, repos, generate_tasks_file)
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, evg_api_config, verbose, task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
generate_config.validate(evg_conf)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
change_detector = EvergreenFileChangeDetector(task_id, evg_api)
changed_tests = change_detector.find_changed_tests(repos)
tests_by_task = create_tests_by_task(generate_config.build_variant,
evg_conf, changed_tests)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
build_variant = create_multiversion_generate_tasks_config(
tests_by_task, evg_api, generate_config)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
write_file(generate_tasks_file, shrub_project.json())
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, no_exec, resmoke_args, evg_api_config, verbose,
task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param no_exec: Just perform test discover, do not execute the tests.
:param resmoke_args: Arguments to pass through to resmoke.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
repeat_config = RepeatConfig() # yapf: disable
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
if generate_tasks_file:
generate_config.validate(evg_conf)
evg_api = _get_evg_api(evg_api_config, False)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
tests_by_task = create_tests_by_task(generate_config.build_variant, repos,
evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
evg_config = Configuration()
evg_config = create_multiversion_generate_tasks_config(
evg_config, tests_by_task, evg_api, generate_config)
json_config = evg_config.to_map()
tasks_to_create = len(json_config.get('tasks', []))
if tasks_to_create > MAX_TASKS_TO_CREATE:
LOGGER.warning(
"Attempting to create more tasks than max, aborting",
tasks=tasks_to_create,
max=MAX_TASKS_TO_CREATE)
sys.exit(1)
_write_json_file(json_config, generate_tasks_file)
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")
def main():
"""Execute Main program."""
values, args = parse_command_line()
# If a resmoke.py command wasn't passed in, use a simple version.
if not args:
args = ["python", "buildscripts/resmoke.py", "--repeatSuites=2"]
# Load the dict of tests to run.
if values.test_list_file:
tests_by_task = _load_tests_file(values.test_list_file)
# If there are no tests to run, carry on.
if tests_by_task is None:
test_results = {"failures": 0, "results": []}
_write_report_file(test_results, values.report_file)
sys.exit(0)
# Run the executor finder.
else:
# Parse the Evergreen project configuration file.
evergreen_conf = evergreen.parse_evergreen_file(values.evergreen_file)
if values.buildvariant is None:
print "Option buildVariant must be specified to find changed tests.\n", \
"Select from the following: \n" \
"\t", "\n\t".join(sorted(evergreen_conf.variant_names))
sys.exit(1)
changed_tests = find_changed_tests(values.branch, values.base_commit, values.max_revisions,
values.buildvariant, values.check_evergreen)
exclude_suites, exclude_tasks, exclude_tests = find_exclude_tests(values.selector_file)
changed_tests = filter_tests(changed_tests, exclude_tests)
# If there are no changed tests, exit cleanly.
if not changed_tests:
print "No new or modified tests found."
if values.test_list_outfile is not None:
_write_report_file({}, values.test_list_outfile)
sys.exit(0)
suites = resmokelib.suitesconfig.get_suites(
suite_files=values.suite_files.split(","), test_files=changed_tests)
tests_by_executor = create_executor_list(suites, exclude_suites)
tests_by_task = create_task_list(evergreen_conf, values.buildvariant, tests_by_executor,
exclude_tasks)
if values.test_list_outfile is not None:
_write_report_file(tests_by_task, values.test_list_outfile)
# If we're not in noExec mode, run the tests.
if not values.no_exec:
test_results = {"failures": 0, "results": []}
for task in sorted(tests_by_task):
resmoke_cmd = copy.deepcopy(args)
resmoke_cmd.extend(shlex.split(tests_by_task[task]["resmoke_args"]))
resmoke_cmd.extend(tests_by_task[task]["tests"])
try:
subprocess.check_call(resmoke_cmd, shell=False)
except subprocess.CalledProcessError as err:
print "Resmoke returned an error with task:", task
_save_report_data(test_results, values.report_file, task)
_write_report_file(test_results, values.report_file)
sys.exit(err.returncode)
_save_report_data(test_results, values.report_file, task)
_write_report_file(test_results, values.report_file)
sys.exit(0)
def test_invalid_path(self):
invalid_path = "non_existing_file"
with self.assertRaises(IOError):
_evergreen.parse_evergreen_file(invalid_path, evergreen_binary=None)
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, no_exec, repeat_tests_num, repeat_tests_min,
repeat_tests_max, repeat_tests_secs, resmoke_args, local_mode,
evg_api_config, verbose):
"""
Run new or changed tests in repeated mode to validate their stability.
burn_in_tests detects jstests that are new or changed since the last git command and then
runs those tests in a loop to validate their reliability.
The `--repeat-*` arguments allow configuration of how burn_in_tests repeats tests. Tests can
either be repeated a specified number of times with the `--repeat-tests` option, or they can
be repeated for a certain time period with the `--repeat-tests-secs` option.
There are two modes that burn_in_tests can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param no_exec: Just perform test discover, do not execute the tests.
:param repeat_tests_num: Repeat each test this number of times.
:param repeat_tests_min: Repeat each test at least this number of times.
:param repeat_tests_max: Once this number of repetitions has been reached, stop repeating.
:param repeat_tests_secs: Continue repeating tests for this number of seconds.
:param resmoke_args: Arguments to pass through to resmoke.
:param local_mode: Don't call out to the evergreen API (used for testing).
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
repeat_config = RepeatConfig(repeat_tests_secs=repeat_tests_secs,
repeat_tests_min=repeat_tests_min,
repeat_tests_max=repeat_tests_max,
repeat_tests_num=repeat_tests_num) # yapf: disable
repeat_config.validate()
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project) # yapf: disable
generate_config.validate(evg_conf)
evg_api = _get_evg_api(evg_api_config, local_mode)
repo = Repo(".")
burn_in(repeat_config, generate_config, resmoke_args, generate_tasks_file,
no_exec, evg_conf, repo, evg_api)
def setUpClass(cls):
cls.conf = _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None)
def check_variant(buildvariant, parser):
"""Check if the buildvariant is found in the evergreen file."""
evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
if not evg_conf.get_variant(buildvariant):
parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format(
buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names))))
