def main(resmoke_report_file: str, evg_api_config: str, project_id: str,
build_variant: str, task_name: str) -> None:
"""Compare resmoke tests runtime with historic stats."""
enable_logging(verbose=False)
current_test_infos = parse_resmoke_report(resmoke_report_file)
current_stats_map = make_stats_map([
_TestData(test_info.test_file,
test_info.end_time - test_info.start_time)
for test_info in current_test_infos
])
historic_stats = get_historic_stats(evg_api_config, project_id,
list(current_stats_map.keys()),
task_name, build_variant)
historic_stats_map = make_stats_map([
_TestData(test_stats.test_file, test_stats.avg_duration_pass)
for test_stats in historic_stats
])
failed = False
for test, stats in current_stats_map.items():
current_mean = mean(stats)
if current_mean < IGNORE_LESS_THAN_SECS:
continue
historic_test_stats = historic_stats_map.get(test)
if historic_test_stats:
historic_max = max(historic_test_stats)
target = historic_max * HISTORIC_MAX_MULTIPLIER
if current_mean > target:
LOGGER.error("Found long running test.",
test_file=test,
current_mean_time=current_mean,
maximum_expected_time=target,
historic_max_time=historic_max)
failed = True
LOGGER.info("Done comparing resmoke tests runtime with historic stats.")
if failed:
percent = int((HISTORIC_MAX_MULTIPLIER - 1) * 100)
LOGGER.error(
f"The test failed due to its runtime taking {percent}% more than the recent max"
" and can negatively contribute to the future patch build experience."
" Consider checking if there is an unexpected regression.")
LOGGER.error(
"If the test is being intentionally expanded, please split it up into separate"
" JS files that run as separate tests.")
LOGGER.error(
"If you believe the test has inherently large variability, please consider writing"
" a new test instead of modifying this one.")
LOGGER.error(
"For any other questions or concerns, please reach out to #server-testing."
)
sys.exit(1)
def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None:
"""
Activate the associated generated executions based in the running build.
The `--expansion-file` should contain all the configuration needed to generate the tasks.
\f
:param expansion_file: Configuration file.
:param evergreen_config: Evergreen configuration file.
:param verbose: Use verbose logging.
"""
enable_logging(verbose)
expansions = EvgExpansions.from_yaml_file(expansion_file)
evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config)
activate_task(expansions.build_id, expansions.task, evg_api)
def main(expansion_file: str, evg_api_config: str, evg_project_config: str,
output_file: str, verbose: bool,
resmoke_run_args: Tuple[str]) -> None:
"""
Generate task configuration for a build variant.
\f
:param expansion_file: Location of evergreen expansions for task.
:param evg_api_config: Location of file containing evergreen API authentication information.
:param evg_project_config: Location of file containing evergreen project configuration.
:param output_file: Location to write generated configuration to.
:param verbose: Should verbose logging be used.
:param resmoke_run_args: Args to forward to `resmoke.py run`.
"""
enable_logging(verbose)
end_date = datetime.utcnow().replace(microsecond=0)
start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS)
evg_expansions = EvgExpansions.from_yaml_file(expansion_file)
# pylint: disable=no-value-for-parameter
def dependencies(binder: inject.Binder) -> None:
binder.bind(EvgExpansions, evg_expansions)
binder.bind(
SuiteSplitConfig,
evg_expansions.build_suite_split_config(start_date, end_date))
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(GenTaskOptions,
evg_expansions.build_evg_config_gen_options())
binder.bind(EvergreenApi,
RetryingEvergreenApi.get_api(config_file=evg_api_config))
binder.bind(EvergreenProjectConfig,
parse_evergreen_file(evg_project_config))
binder.bind(GenerationConfiguration,
GenerationConfiguration.from_yaml_file())
binder.bind(ResmokeProxyService,
ResmokeProxyService(" ".join(resmoke_run_args)))
inject.configure(dependencies)
orchestrator = GenerateBuildVariantOrchestrator() # pylint: disable=no-value-for-parameter
start_time = perf_counter()
orchestrator.generate(evg_expansions.task_id, evg_expansions.build_variant,
output_file)
end_time = perf_counter()
LOGGER.info("Total runtime", duration=end_time - start_time)
def main(
verbose: bool,
expansion_file: str,
evg_api_config: str,
selected_tests_config: str,
):
"""
Select tasks to be run based on changed files in a patch build.
:param verbose: Log extra debug information.
:param expansion_file: Configuration file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param selected_tests_config: Location of config file to connect to elected-tests service.
"""
enable_logging(verbose)
end_date = datetime.utcnow().replace(microsecond=0)
start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS)
evg_expansions = EvgExpansions.from_yaml_file(expansion_file)
def dependencies(binder: inject.Binder) -> None:
binder.bind(EvgExpansions, evg_expansions)
binder.bind(EvergreenApi,
RetryingEvergreenApi.get_api(config_file=evg_api_config))
binder.bind(EvergreenProjectConfig,
parse_evergreen_file(EVERGREEN_FILE))
binder.bind(SelectedTestsClient,
SelectedTestsClient.from_file(selected_tests_config))
binder.bind(
SuiteSplitConfig,
evg_expansions.build_suite_split_config(start_date, end_date))
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options())
binder.bind(
GenerationConfiguration,
GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE))
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
inject.configure(dependencies)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
selected_tests = SelectedTestsOrchestrator() # pylint: disable=no-value-for-parameter
selected_tests.generate(repos, evg_expansions.task_id)
def run_generate_tasks(expansion_file: str,
evergreen_config: Optional[str] = None):
"""
Create a configuration for generate tasks to create sub suites for the specified resmoke suite.
Tests using ReplicaSetFixture will be generated to use 3 nodes and linear_chain=True.
Tests using ShardedClusterFixture will be generated to use 2 shards with 2 nodes each.
The different binary version configurations tested are stored in REPL_MIXED_VERSION_CONFIGS
and SHARDED_MIXED_VERSION_CONFIGS.
The `--expansion-file` should contain all the configuration needed to generate the tasks.
\f
:param expansion_file: Configuration file.
:param evergreen_config: Evergreen configuration file.
"""
enable_logging(False)
end_date = datetime.utcnow().replace(microsecond=0)
start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS)
evg_expansions = EvgExpansions.from_yaml_file(expansion_file)
def dependencies(binder: inject.Binder) -> None:
binder.bind(SuiteSplitConfig,
evg_expansions.get_split_config(start_date, end_date))
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(GenTaskOptions, evg_expansions.get_generation_options())
binder.bind(EvergreenApi,
RetryingEvergreenApi.get_api(config_file=evergreen_config))
binder.bind(
GenerationConfiguration,
GenerationConfiguration.from_yaml_file(
gen_fuzzer.GENERATE_CONFIG_FILE))
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
inject.configure(dependencies)
generate_orchestrator = MultiVersionGenerateOrchestrator() # pylint: disable=no-value-for-parameter
generate_orchestrator.generate(evg_expansions)
def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None:
"""
Create a configuration for generate tasks to create sub suites for the specified resmoke suite.
The `--expansion-file` should contain all the configuration needed to generate the tasks.
\f
:param expansion_file: Configuration file.
:param evergreen_config: Evergreen configuration file.
:param verbose: Use verbose logging.
"""
enable_logging(verbose)
end_date = datetime.utcnow().replace(microsecond=0)
start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS)
evg_expansions = EvgExpansions.from_yaml_file(expansion_file)
def dependencies(binder: inject.Binder) -> None:
binder.bind(
SuiteSplitConfig,
evg_expansions.get_suite_split_config(start_date, end_date))
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(
GenTaskOptions,
evg_expansions.get_evg_config_gen_options(GENERATED_CONFIG_DIR))
binder.bind(EvergreenApi,
RetryingEvergreenApi.get_api(config_file=evergreen_config))
binder.bind(
GenerationConfiguration,
GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE))
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
inject.configure(dependencies)
gen_task_orchestrator = EvgGenResmokeTaskOrchestrator() # pylint: disable=no-value-for-parameter
gen_task_orchestrator.generate_task(
evg_expansions.task_id, evg_expansions.get_suite_split_params(),
evg_expansions.get_gen_params())
def main(evg_api_config: str, task_id: str, suite: str) -> None:
"""Analyze performance results of Google Benchmarks."""
enable_logging(verbose=False)
LOGGER.info("Looking for a baseline task...")
baseline_task_id = get_baseline_task_id(evg_api_config, task_id)
if baseline_task_id:
LOGGER.info("Found baseline task.", task_id=baseline_task_id)
else:
LOGGER.warning("")
LOGGER.warning("Baseline task not found in Evergreen.")
LOGGER.warning("If you think that this is unexpected,"
" please reach out to #server-testing")
LOGGER.warning("")
LOGGER.info("Getting performance data...")
cedar_api = CedarApi(evg_api_config)
current_data = cedar_api.get_perf_data_by_task_id(task_id)
baseline_data = []
try:
baseline_data = cedar_api.get_perf_data_by_task_id(baseline_task_id)
# Swallow HTTPError, since for a new benchmark there might not be historic perf data
except HTTPError as err:
if baseline_task_id:
LOGGER.warning("")
LOGGER.warning(
"Could not get performance data for a baseline task from Cedar",
task_id=baseline_task_id)
LOGGER.warning("", error=err)
LOGGER.warning("If you think that this is unexpected,"
" please reach out to #performance-tooling-users")
LOGGER.warning("")
LOGGER.info("Comparing the current performance data with a baseline data.")
result = compare_data(suite, current_data, baseline_data)
LOGGER.info(f"Performance analysis result:\n{result}")
if not result.passed:
LOGGER.error("Performance data delta has exceeded threshold.")
sys.exit(1)
def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None:
"""Generate fuzzer tests to run in evergreen."""
enable_logging(verbose)
evg_expansions = EvgExpansions.from_yaml_file(expansion_file)
def dependencies(binder: inject.Binder) -> None:
binder.bind(SuiteSplitService, None)
binder.bind(GenTaskOptions, evg_expansions.gen_task_options())
binder.bind(EvergreenApi,
RetryingEvergreenApi.get_api(config_file=evergreen_config))
binder.bind(
GenerationConfiguration,
GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE))
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
inject.configure(dependencies)
gen_fuzzer_orchestrator = EvgGenFuzzerOrchestrator() # pylint: disable=no-value-for-parameter
gen_fuzzer_orchestrator.generate_fuzzer(
evg_expansions.task_id, evg_expansions.fuzzer_gen_task_params())
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, evg_api_config, verbose, task_id, revision,
build_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
enable_logging(verbose)
burn_in_config = BurnInConfig(build_variant=build_variant,
build_id=build_id,
revision=revision)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
generate_config.validate(evg_conf)
gen_task_options = GenTaskOptions(
create_misc_suite=False,
is_patch=True,
generated_config_dir=DEFAULT_CONFIG_DIR,
use_default_timeouts=False,
)
split_task_options = SuiteSplitConfig(
evg_project=project,
target_resmoke_time=60,
max_sub_suites=100,
max_tests_per_suite=1,
start_date=datetime.utcnow(),
end_date=datetime.utcnow(),
default_to_fallback=True,
)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
def dependencies(binder: inject.Binder) -> None:
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
binder.bind(SuiteSplitConfig, split_task_options)
binder.bind(SplitStrategy, greedy_division)
binder.bind(FallbackStrategy, round_robin_fallback)
binder.bind(EvergreenProjectConfig, evg_conf)
binder.bind(GenTaskOptions, gen_task_options)
binder.bind(EvergreenApi, evg_api)
binder.bind(GenerationConfiguration,
GenerationConfiguration.from_yaml_file())
binder.bind(
ResmokeProxyConfig,
ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
binder.bind(EvergreenFileChangeDetector,
EvergreenFileChangeDetector(task_id, evg_api))
binder.bind(BurnInConfig, burn_in_config)
inject.configure(dependencies)
burn_in_orchestrator = MultiversionBurnInOrchestrator() # pylint: disable=no-value-for-parameter
burn_in_orchestrator.generate_tests(repos, generate_config,
generate_tasks_file)
def generate_exclude_yaml(output: str) -> None:
# pylint: disable=too-many-locals
"""
Create a tag file associating multiversion tests to tags for exclusion.
Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
last-lts branch to determine which tests should be denylisted.
"""
enable_logging(False)
location, _ = os.path.split(os.path.abspath(output))
if not os.path.isdir(location):
LOGGER.info(f"Cannot write to {output}. Not generating tag file.")
return
backports_required_latest = read_yaml_file(
os.path.join(ETC_DIR, BACKPORTS_REQUIRED_FILE))
# Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts
# binary we are running tests against. We do this by using the commit hash from the last-lts
# mongo shell executable.
last_lts_commit_hash = get_backports_required_hash_for_shell_version(
mongo_shell_path=LAST_LTS_MONGO_BINARY)
# Get the yaml contents from the last-lts commit.
backports_required_last_lts = get_last_lts_yaml(last_lts_commit_hash)
def diff(list1, list2):
return [elem for elem in (list1 or []) if elem not in (list2 or [])]
suites_latest = backports_required_latest["last-lts"]["suites"] or {}
# Check if the changed syntax for etc/backports_required_for_multiversion_tests.yml has been
# backported.
# This variable and all branches where it's not set can be deleted after backporting the change.
change_backported = "last-lts" in backports_required_last_lts.keys()
if change_backported:
always_exclude = diff(backports_required_latest["last-lts"]["all"],
backports_required_last_lts["last-lts"]["all"])
suites_last_lts: defaultdict = defaultdict(
list, backports_required_last_lts["last-lts"]["suites"])
else:
always_exclude = diff(backports_required_latest["last-lts"]["all"],
backports_required_last_lts["all"])
suites_last_lts: defaultdict = defaultdict(
list, backports_required_last_lts["suites"])
tags = _tags.TagsConfig()
# Tag tests that are excluded from every suite.
for elem in always_exclude:
tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG)
# Tag tests that are excluded on a suite-by-suite basis.
for suite in suites_latest.keys():
test_set = set()
for elem in diff(suites_latest[suite], suites_last_lts[suite]):
test_set.add(elem["test_file"])
for test in test_set:
tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}")
LOGGER.info(f"Writing exclude tags to {output}.")
tags.write_file(
filename=output,
preamble="Tag file that specifies exclusions from multiversion suites."
)
def main():
"""Determine the timeout value a task should use in evergreen."""
parser = argparse.ArgumentParser(description=main.__doc__)
parser.add_argument("--install-dir",
dest="install_dir",
required=True,
help="Path to bin directory of testable installation")
parser.add_argument("--task-name",
dest="task",
required=True,
help="Task being executed.")
parser.add_argument("--suite-name",
dest="suite_name",
required=True,
help="Resmoke suite being run against.")
parser.add_argument("--build-variant",
dest="variant",
required=True,
help="Build variant task is being executed on.")
parser.add_argument("--evg-alias",
dest="evg_alias",
required=True,
help="Evergreen alias used to trigger build.")
parser.add_argument("--timeout",
dest="timeout",
type=int,
help="Timeout to use (in sec).")
parser.add_argument("--exec-timeout",
dest="exec_timeout",
type=int,
help="Exec timeout to use (in sec).")
parser.add_argument("--exec-timeout-factor",
dest="exec_timeout_factor",
type=float,
help="Exec timeout factor to use (in sec).")
parser.add_argument("--out-file",
dest="outfile",
help="File to write configuration to.")
parser.add_argument("--timeout-overrides",
dest="timeout_overrides_file",
default=DEFAULT_TIMEOUT_OVERRIDES,
help="File containing timeout overrides to use.")
parser.add_argument("--evg-api-config",
dest="evg_api_config",
default=DEFAULT_EVERGREEN_AUTH_CONFIG,
help="Evergreen API config file.")
parser.add_argument("--evg-project-config",
dest="evg_project_config",
default=DEFAULT_EVERGREEN_CONFIG,
help="Evergreen project config file.")
options = parser.parse_args()
end_date = datetime.now()
start_date = end_date - HISTORY_LOOKBACK
timeout_override = timedelta(
seconds=options.timeout) if options.timeout else None
exec_timeout_override = timedelta(
seconds=options.exec_timeout) if options.exec_timeout else None
task_name = determine_task_base_name(options.task, options.variant)
timeout_overrides = TimeoutOverrides.from_yaml_file(
os.path.expanduser(options.timeout_overrides_file))
enable_logging(verbose=False)
def dependencies(binder: inject.Binder) -> None:
binder.bind(
EvergreenApi,
RetryingEvergreenApi.get_api(
config_file=os.path.expanduser(options.evg_api_config)))
binder.bind(TimeoutSettings,
TimeoutSettings(start_date=start_date, end_date=end_date))
binder.bind(TimeoutOverrides, timeout_overrides)
binder.bind(
EvergreenProjectConfig,
parse_evergreen_file(os.path.expanduser(
options.evg_project_config)))
binder.bind(
ResmokeProxyService,
ResmokeProxyService(
run_options=f"--installDir={shlex.quote(options.install_dir)}")
)
inject.configure(dependencies)
task_timeout_orchestrator = inject.instance(TaskTimeoutOrchestrator)
task_timeout_orchestrator.determine_timeouts(
timeout_override, exec_timeout_override, options.outfile, task_name,
options.variant, options.evg_alias, options.suite_name,
options.exec_timeout_factor)
