def main():
required_options = ["project",
"reliable_test_min_run",
"unreliable_test_min_run",
"test_fail_rates",
]
parser = optparse.OptionParser(description=__doc__,
usage="Usage: %prog [options] test1 test2 ...")
parser.add_option("--project", dest="project",
default=None,
help="Evergreen project to analyze [REQUIRED].")
parser.add_option("--reliableTestMinimumRun", dest="reliable_test_min_run",
default=None,
type="int",
help="Minimum number of tests runs for test to be considered as reliable"
" [REQUIRED].")
parser.add_option("--unreliableTestMinimumRun", dest="unreliable_test_min_run",
default=None,
type="int",
help="Minimum number of tests runs for test to be considered as unreliable"
" [REQUIRED].")
parser.add_option("--testFailRates", dest="test_fail_rates",
metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE",
default=None,
type="float",
nargs=2,
help="Test fail rates: acceptable fail rate and unacceptable fail rate"
" Specify floating numbers between 0.0 and 1.0 [REQUIRED].")
parser.add_option("--taskFailRates", dest="task_fail_rates",
metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE",
default=None,
type="float",
nargs=2,
help="Task fail rates: acceptable fail rate and unacceptable fail rate."
" Specify floating numbers between 0.0 and 1.0."
" Uses --test-fail-rates if unspecified.")
parser.add_option("--variantFailRates", dest="variant_fail_rates",
metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE",
default=None,
type="float",
nargs=2,
help="Variant fail rates: acceptable fail rate and unacceptable fail rate."
" Specify floating numbers between 0.0 and 1.0."
" Uses --task-fail-rates if unspecified.")
parser.add_option("--distroFailRates", dest="distro_fail_rates",
metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE",
default=None,
type="float",
nargs=2,
help="Distro fail rates: acceptable fail rate and unacceptable fail rate."
" Specify floating numbers between 0.0 and 1.0."
" Uses --variant-fail-rates if unspecified.")
parser.add_option("--tasks", dest="tasks",
default=None,
help="Names of tasks to analyze for tagging unreliable tests."
" If specified and no tests are specified, then only tests"
" associated with the tasks will be analyzed."
" If unspecified and no tests are specified, the list of tasks will be"
" the non-excluded list of tasks from the file specified by"
" '--evergreenYML'.")
parser.add_option("--variants", dest="variants",
default="",
help="Names of variants to analyze for tagging unreliable tests.")
parser.add_option("--distros", dest="distros",
default="",
help="Names of distros to analyze for tagging unreliable tests [UNUSED].")
parser.add_option("--evergreenYML", dest="evergreen_yml",
default="etc/evergreen.yml",
help="Evergreen YML file used to get the list of tasks,"
" defaults to '%default'.")
parser.add_option("--lifecycleFile", dest="lifecycle_file",
default="etc/test_lifecycle.yml",
help="Evergreen lifecycle file to update, defaults to '%default'.")
parser.add_option("--reliableDays", dest="reliable_days",
default=7,
type="int",
help="Number of days to check for reliable tests, defaults to '%default'.")
parser.add_option("--unreliableDays", dest="unreliable_days",
default=28,
type="int",
help="Number of days to check for unreliable tests, defaults to '%default'.")
parser.add_option("--batchGroupSize", dest="batch_size",
default=100,
type="int",
help="Size of test batch group, defaults to '%default'.")
(options, tests) = parser.parse_args()
for option in required_options:
if not getattr(options, option):
parser.print_help()
parser.error("Missing required option")
evg_conf = evergreen.EvergreenProjectConfig(options.evergreen_yml)
use_test_tasks_membership = False
tasks = options.tasks.split(",") if options.tasks else []
if not tasks:
# If no tasks are specified, then the list of tasks is all.
tasks = evg_conf.lifecycle_task_names
use_test_tasks_membership = True
variants = options.variants.split(",") if options.variants else []
distros = options.distros.split(",") if options.distros else []
check_fail_rates("Test", options.test_fail_rates[0], options.test_fail_rates[1])
# The less specific failures rates are optional and default to a lower level value.
if not options.task_fail_rates:
options.task_fail_rates = options.test_fail_rates
else:
check_fail_rates("Task", options.task_fail_rates[0], options.task_fail_rates[1])
if not options.variant_fail_rates:
options.variant_fail_rates = options.task_fail_rates
else:
check_fail_rates("Variant", options.variant_fail_rates[0], options.variant_fail_rates[1])
if not options.distro_fail_rates:
options.distro_fail_rates = options.variant_fail_rates
else:
check_fail_rates("Distro", options.distro_fail_rates[0], options.distro_fail_rates[1])
check_days("Reliable days", options.reliable_days)
check_days("Unreliable days", options.unreliable_days)
orig_lifecycle = read_yaml_file(options.lifecycle_file)
lifecycle = copy.deepcopy(orig_lifecycle)
test_tasks_membership = get_test_tasks_membership(evg_conf)
# If no tests are specified then the list of tests is generated from the list of tasks.
if not tests:
tests = get_tests_from_tasks(tasks, test_tasks_membership)
if not options.tasks:
use_test_tasks_membership = True
commit_first, commit_last = git_commit_range_since("{}.days".format(options.unreliable_days))
commit_prior = git_commit_prior(commit_first)
# For efficiency purposes, group the tests and process in batches of batch_size.
test_groups = create_batch_groups(create_test_groups(tests), options.batch_size)
for tests in test_groups:
# Find all associated tasks for the test_group if tasks or tests were not specified.
if use_test_tasks_membership:
tasks_set = set()
for test in tests:
tasks_set = tasks_set.union(test_tasks_membership[test])
tasks = list(tasks_set)
if not tasks:
print("Warning - No tasks found for tests {}, skipping this group.".format(tests))
continue
report = tf.HistoryReport(period_type="revision",
start=commit_prior,
end=commit_last,
group_period=options.reliable_days,
project=options.project,
tests=tests,
tasks=tasks,
variants=variants,
distros=distros)
view_report = report.generate_report()
# We build up report_combo to check for more specific test failures rates.
report_combo = []
# TODO EVG-1665: Uncomment this line once this has been supported.
# for combo in ["test", "task", "variant", "distro"]:
for combo in ["test", "task", "variant"]:
report_combo.append(combo)
if combo == "distro":
acceptable_fail_rate = options.distro_fail_rates[0]
unacceptable_fail_rate = options.distro_fail_rates[1]
elif combo == "variant":
acceptable_fail_rate = options.variant_fail_rates[0]
unacceptable_fail_rate = options.variant_fail_rates[1]
elif combo == "task":
acceptable_fail_rate = options.task_fail_rates[0]
unacceptable_fail_rate = options.task_fail_rates[1]
else:
acceptable_fail_rate = options.test_fail_rates[0]
unacceptable_fail_rate = options.test_fail_rates[1]
# Unreliable tests are analyzed from the entire period.
update_lifecycle(lifecycle,
view_report.view_summary(group_on=report_combo),
unreliable_test,
True,
unacceptable_fail_rate,
options.unreliable_test_min_run)
# Reliable tests are analyzed from the last period, i.e., last 14 days.
(reliable_start_date, reliable_end_date) = view_report.last_period()
update_lifecycle(lifecycle,
view_report.view_summary(group_on=report_combo,
start_date=reliable_start_date,
end_date=reliable_end_date),
reliable_test,
False,
acceptable_fail_rate,
options.reliable_test_min_run)
# Update the lifecycle_file only if there have been changes.
if orig_lifecycle != lifecycle:
write_yaml_file(options.lifecycle_file, lifecycle)
def main():
values, args = parse_command_line()
# If a resmoke.py command wasn't passed in, use a simple version.
if not args:
args = ["python", "buildscripts/resmoke.py", "--repeat=2"]
# Load the dict of tests to run.
if values.test_list_file:
tests_by_task = _load_tests_file(values.test_list_file)
# If there are no tests to run, carry on.
if tests_by_task is None:
test_results = {"failures": 0, "results": []}
_write_report_file(test_results, values.report_file)
sys.exit(0)
# Run the executor finder.
else:
# Parse the Evergreen project configuration file.
evergreen_conf = evergreen.EvergreenProjectConfig(values.evergreen_file)
if values.buildvariant is None:
print "Option buildVariant must be specified to find changed tests.\n", \
"Select from the following: \n" \
"\t", "\n\t".join(sorted(evergreen_conf.variant_names))
sys.exit(1)
changed_tests = find_changed_tests(values.branch,
values.base_commit,
values.max_revisions,
values.buildvariant,
values.check_evergreen)
exclude_suites, exclude_tasks, exclude_tests = find_exclude_tests(values.selector_file)
changed_tests = filter_tests(changed_tests, exclude_tests)
# If there are no changed tests, exit cleanly.
if not changed_tests:
print "No new or modified tests found."
_write_report_file({}, values.test_list_outfile)
sys.exit(0)
suites = resmokelib.parser.get_suites(values, changed_tests)
tests_by_executor = create_executor_list(suites, exclude_suites)
tests_by_task = create_task_list(evergreen_conf,
values.buildvariant,
tests_by_executor,
exclude_tasks)
if values.test_list_outfile is not None:
_write_report_file(tests_by_task, values.test_list_outfile)
# If we're not in noExec mode, run the tests.
if not values.no_exec:
test_results = {"failures": 0, "results": []}
for task in sorted(tests_by_task):
resmoke_cmd = copy.deepcopy(args)
resmoke_cmd.extend(shlex.split(tests_by_task[task]["resmoke_args"]))
resmoke_cmd.extend(tests_by_task[task]["tests"])
try:
subprocess.check_call(resmoke_cmd, shell=False)
except subprocess.CalledProcessError as err:
print "Resmoke returned an error with task:", task
_save_report_data(test_results, values.report_file, task)
_write_report_file(test_results, values.report_file)
sys.exit(err.returncode)
_save_report_data(test_results, values.report_file, task)
_write_report_file(test_results, values.report_file)
sys.exit(0)
def test_invalid_path(self):
invalid_path = "non_existing_file"
with self.assertRaises(IOError):
_evergreen.EvergreenProjectConfig(invalid_path)
def setUpClass(cls):
cls.conf = _evergreen.EvergreenProjectConfig(TEST_FILE_PATH)
def main():
"""
Utility for updating a resmoke.py tag file based on computing test failure rates from the
Evergreen API.
"""
parser = optparse.OptionParser(
description=textwrap.dedent(main.__doc__),
usage="Usage: %prog [options] [test1 test2 ...]")
data_options = optparse.OptionGroup(
parser,
title="Data options",
description=
("Options used to configure what historical test failure data to retrieve from"
" Evergreen."))
parser.add_option_group(data_options)
data_options.add_option(
"--project",
dest="project",
metavar="<project-name>",
default=tf.TestHistory.DEFAULT_PROJECT,
help="The Evergreen project to analyze. Defaults to '%default'.")
data_options.add_option(
"--tasks",
dest="tasks",
metavar="<task1,task2,...>",
help=
("The Evergreen tasks to analyze for tagging unreliable tests. If specified in"
" additional to having test positional arguments, then only tests that run under the"
" specified Evergreen tasks will be analyzed. If omitted, then the list of tasks"
" defaults to the non-excluded list of tasks from the specified"
" --evergreenProjectConfig file."))
data_options.add_option(
"--variants",
dest="variants",
metavar="<variant1,variant2,...>",
default="",
help=
"The Evergreen build variants to analyze for tagging unreliable tests."
)
data_options.add_option(
"--distros",
dest="distros",
metavar="<distro1,distro2,...>",
default="",
help="The Evergreen distros to analyze for tagging unreliable tests.")
data_options.add_option(
"--evergreenProjectConfig",
dest="evergreen_project_config",
metavar="<project-config-file>",
default="etc/evergreen.yml",
help=
("The Evergreen project configuration file used to get the list of tasks if --tasks is"
" omitted. Defaults to '%default'."))
model_options = optparse.OptionGroup(
parser,
title="Model options",
description=
("Options used to configure whether (test,), (test, task),"
" (test, task, variant), and (test, task, variant, distro) combinations are"
" considered unreliable."))
parser.add_option_group(model_options)
model_options.add_option(
"--reliableTestMinRuns",
type="int",
dest="reliable_test_min_runs",
metavar="<reliable-min-runs>",
default=DEFAULT_CONFIG.reliable_min_runs,
help=
("The minimum number of test executions required for a test's failure rate to"
" determine whether the test is considered reliable. If a test has fewer than"
" <reliable-min-runs> executions, then it cannot be considered unreliable."
))
model_options.add_option(
"--unreliableTestMinRuns",
type="int",
dest="unreliable_test_min_runs",
metavar="<unreliable-min-runs>",
default=DEFAULT_CONFIG.unreliable_min_runs,
help=
("The minimum number of test executions required for a test's failure rate to"
" determine whether the test is considered unreliable. If a test has fewer than"
" <unreliable-min-runs> executions, then it cannot be considered unreliable."
))
model_options.add_option(
"--testFailRates",
type="float",
nargs=2,
dest="test_fail_rates",
metavar="<test-acceptable-fail-rate> <test-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.test_fail_rates,
help=
("Controls how readily a test is considered unreliable. Each failure rate must be a"
" number between 0 and 1 (inclusive) with"
" <test-unacceptable-fail-rate> >= <test-acceptable-fail-rate>. If a test fails no"
" more than <test-acceptable-fail-rate> in <reliable-days> time, then it is"
" considered reliable. Otherwise, if a test fails at least as much as"
" <test-unacceptable-fail-rate> in <test-unreliable-days> time, then it is considered"
" unreliable. Defaults to %default."))
model_options.add_option(
"--taskFailRates",
type="float",
nargs=2,
dest="task_fail_rates",
metavar="<task-acceptable-fail-rate> <task-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.task_fail_rates,
help=
("Controls how readily a (test, task) combination is considered unreliable. Each"
" failure rate must be a number between 0 and 1 (inclusive) with"
" <task-unacceptable-fail-rate> >= <task-acceptable-fail-rate>. If a (test, task)"
" combination fails no more than <task-acceptable-fail-rate> in <reliable-days> time,"
" then it is considered reliable. Otherwise, if a test fails at least as much as"
" <task-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
" unreliable. Defaults to %default."))
model_options.add_option(
"--variantFailRates",
type="float",
nargs=2,
dest="variant_fail_rates",
metavar=
"<variant-acceptable-fail-rate> <variant-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.variant_fail_rates,
help=
("Controls how readily a (test, task, variant) combination is considered unreliable."
" Each failure rate must be a number between 0 and 1 (inclusive) with"
" <variant-unacceptable-fail-rate> >= <variant-acceptable-fail-rate>. If a"
" (test, task, variant) combination fails no more than <variant-acceptable-fail-rate>"
" in <reliable-days> time, then it is considered reliable. Otherwise, if a test fails"
" at least as much as <variant-unacceptable-fail-rate> in <unreliable-days> time,"
" then it is considered unreliable. Defaults to %default."))
model_options.add_option(
"--distroFailRates",
type="float",
nargs=2,
dest="distro_fail_rates",
metavar="<distro-acceptable-fail-rate> <distro-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.distro_fail_rates,
help=
("Controls how readily a (test, task, variant, distro) combination is considered"
" unreliable. Each failure rate must be a number between 0 and 1 (inclusive) with"
" <distro-unacceptable-fail-rate> >= <distro-acceptable-fail-rate>. If a"
" (test, task, variant, distro) combination fails no more than"
" <distro-acceptable-fail-rate> in <reliable-days> time, then it is considered"
" reliable. Otherwise, if a test fails at least as much as"
" <distro-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
" unreliable. Defaults to %default."))
model_options.add_option(
"--reliableDays",
type="int",
dest="reliable_days",
metavar="<ndays>",
default=DEFAULT_CONFIG.reliable_time_period.days,
help=
("The time period to analyze when determining if a test has become reliable. Defaults"
" to %default day(s)."))
model_options.add_option(
"--unreliableDays",
type="int",
dest="unreliable_days",
metavar="<ndays>",
default=DEFAULT_CONFIG.unreliable_time_period.days,
help=
("The time period to analyze when determining if a test has become unreliable."
" Defaults to %default day(s)."))
parser.add_option(
"--resmokeTagFile",
dest="tag_file",
metavar="<tagfile>",
default="etc/test_lifecycle.yml",
help=
("The resmoke.py tag file to update. If --metadataRepo is specified, it"
" is the relative path in the metadata repository, otherwise it can be"
" an absolute path or a relative path from the current directory."
" Defaults to '%default'."))
parser.add_option(
"--metadataRepo",
dest="metadata_repo_url",
metavar="<metadata-repo-url>",
default="[email protected]:mongodb/mongo-test-metadata.git",
help=("The repository that contains the lifecycle file. "
"It will be cloned in the current working directory. "
"Defaults to '%default'."))
parser.add_option(
"--referencesFile",
dest="references_file",
metavar="<references-file>",
default="references.yml",
help=
("The YAML file in the metadata repository that contains the revision "
"mappings. Defaults to '%default'."))
parser.add_option(
"--requestBatchSize",
type="int",
dest="batch_size",
metavar="<batch-size>",
default=100,
help=
("The maximum number of tests to query the Evergreen API for in a single"
" request. A higher value for this option will reduce the number of"
" roundtrips between this client and Evergreen. Defaults to %default."
))
commit_options = optparse.OptionGroup(
parser,
title="Commit options",
description=
("Options used to configure whether and how to commit the updated test"
" lifecycle tags."))
parser.add_option_group(commit_options)
commit_options.add_option(
"--commit",
action="store_true",
dest="commit",
default=False,
help="Indicates that the updated tag file should be committed.")
commit_options.add_option(
"--jiraConfig",
dest="jira_config",
metavar="<jira-config>",
default=None,
help=
("The YAML file containing the JIRA access configuration ('user', 'password',"
"'server')."))
commit_options.add_option(
"--gitUserName",
dest="git_user_name",
metavar="<git-user-name>",
default="Test Lifecycle",
help=
("The git user name that will be set before committing to the metadata repository."
" Defaults to '%default'."))
commit_options.add_option(
"--gitUserEmail",
dest="git_user_email",
metavar="<git-user-email>",
default="*****@*****.**",
help=
("The git user email address that will be set before committing to the metadata"
" repository. Defaults to '%default'."))
logging_options = optparse.OptionGroup(
parser,
title="Logging options",
description=
"Options used to configure the logging output of the script.")
parser.add_option_group(logging_options)
logging_options.add_option(
"--logLevel",
dest="log_level",
metavar="<log-level>",
choices=["DEBUG", "INFO", "WARNING", "ERROR"],
default="INFO",
help=
("The log level. Accepted values are: DEBUG, INFO, WARNING and ERROR."
" Defaults to '%default'."))
logging_options.add_option(
"--logFile",
dest="log_file",
metavar="<log-file>",
default=None,
help=
"The destination file for the logs output. Defaults to the standard output."
)
(options, tests) = parser.parse_args()
if options.distros:
warnings.warn((
"Until https://jira.mongodb.org/browse/EVG-1665 is implemented, distro information"
" isn't returned by the Evergreen API. This option will therefore be ignored."
), RuntimeWarning)
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s",
level=options.log_level,
filename=options.log_file)
evg_conf = ci_evergreen.EvergreenProjectConfig(
options.evergreen_project_config)
use_test_tasks_membership = False
tasks = options.tasks.split(",") if options.tasks else []
if not tasks:
# If no tasks are specified, then the list of tasks is all.
tasks = evg_conf.lifecycle_task_names
use_test_tasks_membership = True
variants = options.variants.split(",") if options.variants else []
distros = options.distros.split(",") if options.distros else []
config = Config(
test_fail_rates=Rates(*options.test_fail_rates),
task_fail_rates=Rates(*options.task_fail_rates),
variant_fail_rates=Rates(*options.variant_fail_rates),
distro_fail_rates=Rates(*options.distro_fail_rates),
reliable_min_runs=options.reliable_test_min_runs,
reliable_time_period=datetime.timedelta(days=options.reliable_days),
unreliable_min_runs=options.unreliable_test_min_runs,
unreliable_time_period=datetime.timedelta(
days=options.unreliable_days))
validate_config(config)
lifecycle_tags_file = make_lifecycle_tags_file(options, config)
if not lifecycle_tags_file:
sys.exit(1)
test_tasks_membership = get_test_tasks_membership(evg_conf)
# If no tests are specified then the list of tests is generated from the list of tasks.
if not tests:
tests = get_tests_from_tasks(tasks, test_tasks_membership)
if not options.tasks:
use_test_tasks_membership = True
commit_first, commit_last = git_commit_range_since("{}.days".format(
options.unreliable_days))
commit_prior = git_commit_prior(commit_first)
# For efficiency purposes, group the tests and process in batches of batch_size.
test_groups = create_batch_groups(create_test_groups(tests),
options.batch_size)
LOGGER.info("Updating the tags")
for tests in test_groups:
# Find all associated tasks for the test_group if tasks or tests were not specified.
if use_test_tasks_membership:
tasks_set = set()
for test in tests:
tasks_set = tasks_set.union(test_tasks_membership[test])
tasks = list(tasks_set)
if not tasks:
LOGGER.warning("No tasks found for tests %s, skipping this group.",
tests)
continue
test_history = tf.TestHistory(project=options.project,
tests=tests,
tasks=tasks,
variants=variants,
distros=distros)
history_data = test_history.get_history_by_revision(
start_revision=commit_prior, end_revision=commit_last)
report = tf.Report(history_data)
update_tags(lifecycle_tags_file.changelog_lifecycle, config, report)
# Remove tags that are no longer relevant
clean_up_tags(lifecycle_tags_file.changelog_lifecycle, evg_conf)
# We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there
# have been changes to the tags. In particular, we avoid modifying the file when only the header
# comment for the YAML file would change.
if lifecycle_tags_file.is_modified():
lifecycle_tags_file.write()
if options.commit:
commit_ok = lifecycle_tags_file.commit()
if not commit_ok:
sys.exit(1)
else:
LOGGER.info("The tags have not been modified.")
def main():
"""
Utility for updating a resmoke.py tag file based on computing test failure rates from the
Evergreen API.
"""
parser = optparse.OptionParser(
description=textwrap.dedent(main.__doc__),
usage="Usage: %prog [options] [test1 test2 ...]")
data_options = optparse.OptionGroup(
parser,
title="Data options",
description=
("Options used to configure what historical test failure data to retrieve from"
" Evergreen."))
parser.add_option_group(data_options)
data_options.add_option(
"--project",
dest="project",
metavar="<project-name>",
default=tf.TestHistory.DEFAULT_PROJECT,
help="The Evergreen project to analyze. Defaults to '%default'.")
data_options.add_option(
"--tasks",
dest="tasks",
metavar="<task1,task2,...>",
help=
("The Evergreen tasks to analyze for tagging unreliable tests. If specified in"
" additional to having test positional arguments, then only tests that run under the"
" specified Evergreen tasks will be analyzed. If omitted, then the list of tasks"
" defaults to the non-excluded list of tasks from the specified"
" --evergreenProjectConfig file."))
data_options.add_option(
"--variants",
dest="variants",
metavar="<variant1,variant2,...>",
default="",
help=
"The Evergreen build variants to analyze for tagging unreliable tests."
)
data_options.add_option(
"--distros",
dest="distros",
metavar="<distro1,distro2,...>",
default="",
help="The Evergreen distros to analyze for tagging unreliable tests.")
data_options.add_option(
"--evergreenProjectConfig",
dest="evergreen_project_config",
metavar="<project-config-file>",
default="etc/evergreen.yml",
help=
("The Evergreen project configuration file used to get the list of tasks if --tasks is"
" omitted. Defaults to '%default'."))
model_options = optparse.OptionGroup(
parser,
title="Model options",
description=
("Options used to configure whether (test,), (test, task),"
" (test, task, variant), and (test, task, variant, distro) combinations are"
" considered unreliable."))
parser.add_option_group(model_options)
model_options.add_option(
"--reliableTestMinRuns",
type="int",
dest="reliable_test_min_runs",
metavar="<reliable-min-runs>",
default=DEFAULT_CONFIG.reliable_min_runs,
help=
("The minimum number of test executions required for a test's failure rate to"
" determine whether the test is considered reliable. If a test has fewer than"
" <reliable-min-runs> executions, then it cannot be considered unreliable."
))
model_options.add_option(
"--unreliableTestMinRuns",
type="int",
dest="unreliable_test_min_runs",
metavar="<unreliable-min-runs>",
default=DEFAULT_CONFIG.unreliable_min_runs,
help=
("The minimum number of test executions required for a test's failure rate to"
" determine whether the test is considered unreliable. If a test has fewer than"
" <unreliable-min-runs> executions, then it cannot be considered unreliable."
))
model_options.add_option(
"--testFailRates",
type="float",
nargs=2,
dest="test_fail_rates",
metavar="<test-acceptable-fail-rate> <test-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.test_fail_rates,
help=
("Controls how readily a test is considered unreliable. Each failure rate must be a"
" number between 0 and 1 (inclusive) with"
" <test-unacceptable-fail-rate> >= <test-acceptable-fail-rate>. If a test fails no"
" more than <test-acceptable-fail-rate> in <reliable-days> time, then it is"
" considered reliable. Otherwise, if a test fails at least as much as"
" <test-unacceptable-fail-rate> in <test-unreliable-days> time, then it is considered"
" unreliable. Defaults to %default."))
model_options.add_option(
"--taskFailRates",
type="float",
nargs=2,
dest="task_fail_rates",
metavar="<task-acceptable-fail-rate> <task-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.task_fail_rates,
help=
("Controls how readily a (test, task) combination is considered unreliable. Each"
" failure rate must be a number between 0 and 1 (inclusive) with"
" <task-unacceptable-fail-rate> >= <task-acceptable-fail-rate>. If a (test, task)"
" combination fails no more than <task-acceptable-fail-rate> in <reliable-days> time,"
" then it is considered reliable. Otherwise, if a test fails at least as much as"
" <task-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
" unreliable. Defaults to %default."))
model_options.add_option(
"--variantFailRates",
type="float",
nargs=2,
dest="variant_fail_rates",
metavar=
"<variant-acceptable-fail-rate> <variant-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.variant_fail_rates,
help=
("Controls how readily a (test, task, variant) combination is considered unreliable."
" Each failure rate must be a number between 0 and 1 (inclusive) with"
" <variant-unacceptable-fail-rate> >= <variant-acceptable-fail-rate>. If a"
" (test, task, variant) combination fails no more than <variant-acceptable-fail-rate>"
" in <reliable-days> time, then it is considered reliable. Otherwise, if a test fails"
" at least as much as <variant-unacceptable-fail-rate> in <unreliable-days> time,"
" then it is considered unreliable. Defaults to %default."))
model_options.add_option(
"--distroFailRates",
type="float",
nargs=2,
dest="distro_fail_rates",
metavar="<distro-acceptable-fail-rate> <distro-unacceptable-fail-rate>",
default=DEFAULT_CONFIG.distro_fail_rates,
help=
("Controls how readily a (test, task, variant, distro) combination is considered"
" unreliable. Each failure rate must be a number between 0 and 1 (inclusive) with"
" <distro-unacceptable-fail-rate> >= <distro-acceptable-fail-rate>. If a"
" (test, task, variant, distro) combination fails no more than"
" <distro-acceptable-fail-rate> in <reliable-days> time, then it is considered"
" reliable. Otherwise, if a test fails at least as much as"
" <distro-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
" unreliable. Defaults to %default."))
model_options.add_option(
"--reliableDays",
type="int",
dest="reliable_days",
metavar="<ndays>",
default=DEFAULT_CONFIG.reliable_time_period.days,
help=
("The time period to analyze when determining if a test has become reliable. Defaults"
" to %default day(s)."))
model_options.add_option(
"--unreliableDays",
type="int",
dest="unreliable_days",
metavar="<ndays>",
default=DEFAULT_CONFIG.unreliable_time_period.days,
help=
("The time period to analyze when determining if a test has become unreliable."
" Defaults to %default day(s)."))
parser.add_option(
"--resmokeTagFile",
dest="tag_file",
metavar="<tagfile>",
default="etc/test_lifecycle.yml",
help="The resmoke.py tag file to update. Defaults to '%default'.")
parser.add_option(
"--requestBatchSize",
type="int",
dest="batch_size",
metavar="<batch-size>",
default=100,
help=
("The maximum number of tests to query the Evergreen API for in a single"
" request. A higher value for this option will reduce the number of"
" roundtrips between this client and Evergreen. Defaults to %default."
))
(options, tests) = parser.parse_args()
if options.distros:
warnings.warn((
"Until https://jira.mongodb.org/browse/EVG-1665 is implemented, distro information"
" isn't returned by the Evergreen API. This option will therefore be ignored."
), RuntimeWarning)
evg_conf = ci_evergreen.EvergreenProjectConfig(
options.evergreen_project_config)
use_test_tasks_membership = False
tasks = options.tasks.split(",") if options.tasks else []
if not tasks:
# If no tasks are specified, then the list of tasks is all.
tasks = evg_conf.lifecycle_task_names
use_test_tasks_membership = True
variants = options.variants.split(",") if options.variants else []
distros = options.distros.split(",") if options.distros else []
config = Config(
test_fail_rates=Rates(*options.test_fail_rates),
task_fail_rates=Rates(*options.task_fail_rates),
variant_fail_rates=Rates(*options.variant_fail_rates),
distro_fail_rates=Rates(*options.distro_fail_rates),
reliable_min_runs=options.reliable_test_min_runs,
reliable_time_period=datetime.timedelta(days=options.reliable_days),
unreliable_min_runs=options.unreliable_test_min_runs,
unreliable_time_period=datetime.timedelta(
days=options.unreliable_days))
validate_config(config)
lifecycle = ci_tags.TagsConfig.from_file(options.tag_file,
cmp_func=compare_tags)
test_tasks_membership = get_test_tasks_membership(evg_conf)
# If no tests are specified then the list of tests is generated from the list of tasks.
if not tests:
tests = get_tests_from_tasks(tasks, test_tasks_membership)
if not options.tasks:
use_test_tasks_membership = True
commit_first, commit_last = git_commit_range_since("{}.days".format(
options.unreliable_days))
commit_prior = git_commit_prior(commit_first)
# For efficiency purposes, group the tests and process in batches of batch_size.
test_groups = create_batch_groups(create_test_groups(tests),
options.batch_size)
for tests in test_groups:
# Find all associated tasks for the test_group if tasks or tests were not specified.
if use_test_tasks_membership:
tasks_set = set()
for test in tests:
tasks_set = tasks_set.union(test_tasks_membership[test])
tasks = list(tasks_set)
if not tasks:
print(
"Warning - No tasks found for tests {}, skipping this group.".
format(tests))
continue
test_history = tf.TestHistory(project=options.project,
tests=tests,
tasks=tasks,
variants=variants,
distros=distros)
history_data = test_history.get_history_by_revision(
start_revision=commit_prior, end_revision=commit_last)
report = tf.Report(history_data)
update_tags(lifecycle, config, report)
# Remove tags that are no longer relevant
cleanup_tags(lifecycle, evg_conf)
# We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there
# have been changes to the tags. In particular, we avoid modifying the file when only the header
# comment for the YAML file would change.
if lifecycle.is_modified():
write_yaml_file(options.tag_file, lifecycle)
