def main(): required_options = ["project", "reliable_test_min_run", "unreliable_test_min_run", "test_fail_rates", ] parser = optparse.OptionParser(description=__doc__, usage="Usage: %prog [options] test1 test2 ...") parser.add_option("--project", dest="project", default=None, help="Evergreen project to analyze [REQUIRED].") parser.add_option("--reliableTestMinimumRun", dest="reliable_test_min_run", default=None, type="int", help="Minimum number of tests runs for test to be considered as reliable" " [REQUIRED].") parser.add_option("--unreliableTestMinimumRun", dest="unreliable_test_min_run", default=None, type="int", help="Minimum number of tests runs for test to be considered as unreliable" " [REQUIRED].") parser.add_option("--testFailRates", dest="test_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help="Test fail rates: acceptable fail rate and unacceptable fail rate" " Specify floating numbers between 0.0 and 1.0 [REQUIRED].") parser.add_option("--taskFailRates", dest="task_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help="Task fail rates: acceptable fail rate and unacceptable fail rate." " Specify floating numbers between 0.0 and 1.0." " Uses --test-fail-rates if unspecified.") parser.add_option("--variantFailRates", dest="variant_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help="Variant fail rates: acceptable fail rate and unacceptable fail rate." " Specify floating numbers between 0.0 and 1.0." " Uses --task-fail-rates if unspecified.") parser.add_option("--distroFailRates", dest="distro_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help="Distro fail rates: acceptable fail rate and unacceptable fail rate." " Specify floating numbers between 0.0 and 1.0." " Uses --variant-fail-rates if unspecified.") parser.add_option("--tasks", dest="tasks", default=None, help="Names of tasks to analyze for tagging unreliable tests." " If specified and no tests are specified, then only tests" " associated with the tasks will be analyzed." " If unspecified and no tests are specified, the list of tasks will be" " the non-excluded list of tasks from the file specified by" " '--evergreenYML'.") parser.add_option("--variants", dest="variants", default="", help="Names of variants to analyze for tagging unreliable tests.") parser.add_option("--distros", dest="distros", default="", help="Names of distros to analyze for tagging unreliable tests [UNUSED].") parser.add_option("--evergreenYML", dest="evergreen_yml", default="etc/evergreen.yml", help="Evergreen YML file used to get the list of tasks," " defaults to '%default'.") parser.add_option("--lifecycleFile", dest="lifecycle_file", default="etc/test_lifecycle.yml", help="Evergreen lifecycle file to update, defaults to '%default'.") parser.add_option("--reliableDays", dest="reliable_days", default=7, type="int", help="Number of days to check for reliable tests, defaults to '%default'.") parser.add_option("--unreliableDays", dest="unreliable_days", default=28, type="int", help="Number of days to check for unreliable tests, defaults to '%default'.") parser.add_option("--batchGroupSize", dest="batch_size", default=100, type="int", help="Size of test batch group, defaults to '%default'.") (options, tests) = parser.parse_args() for option in required_options: if not getattr(options, option): parser.print_help() parser.error("Missing required option") evg_conf = evergreen.EvergreenProjectConfig(options.evergreen_yml) use_test_tasks_membership = False tasks = options.tasks.split(",") if options.tasks else [] if not tasks: # If no tasks are specified, then the list of tasks is all. tasks = evg_conf.lifecycle_task_names use_test_tasks_membership = True variants = options.variants.split(",") if options.variants else [] distros = options.distros.split(",") if options.distros else [] check_fail_rates("Test", options.test_fail_rates[0], options.test_fail_rates[1]) # The less specific failures rates are optional and default to a lower level value. if not options.task_fail_rates: options.task_fail_rates = options.test_fail_rates else: check_fail_rates("Task", options.task_fail_rates[0], options.task_fail_rates[1]) if not options.variant_fail_rates: options.variant_fail_rates = options.task_fail_rates else: check_fail_rates("Variant", options.variant_fail_rates[0], options.variant_fail_rates[1]) if not options.distro_fail_rates: options.distro_fail_rates = options.variant_fail_rates else: check_fail_rates("Distro", options.distro_fail_rates[0], options.distro_fail_rates[1]) check_days("Reliable days", options.reliable_days) check_days("Unreliable days", options.unreliable_days) orig_lifecycle = read_yaml_file(options.lifecycle_file) lifecycle = copy.deepcopy(orig_lifecycle) test_tasks_membership = get_test_tasks_membership(evg_conf) # If no tests are specified then the list of tests is generated from the list of tasks. if not tests: tests = get_tests_from_tasks(tasks, test_tasks_membership) if not options.tasks: use_test_tasks_membership = True commit_first, commit_last = git_commit_range_since("{}.days".format(options.unreliable_days)) commit_prior = git_commit_prior(commit_first) # For efficiency purposes, group the tests and process in batches of batch_size. test_groups = create_batch_groups(create_test_groups(tests), options.batch_size) for tests in test_groups: # Find all associated tasks for the test_group if tasks or tests were not specified. if use_test_tasks_membership: tasks_set = set() for test in tests: tasks_set = tasks_set.union(test_tasks_membership[test]) tasks = list(tasks_set) if not tasks: print("Warning - No tasks found for tests {}, skipping this group.".format(tests)) continue report = tf.HistoryReport(period_type="revision", start=commit_prior, end=commit_last, group_period=options.reliable_days, project=options.project, tests=tests, tasks=tasks, variants=variants, distros=distros) view_report = report.generate_report() # We build up report_combo to check for more specific test failures rates. report_combo = [] # TODO EVG-1665: Uncomment this line once this has been supported. # for combo in ["test", "task", "variant", "distro"]: for combo in ["test", "task", "variant"]: report_combo.append(combo) if combo == "distro": acceptable_fail_rate = options.distro_fail_rates[0] unacceptable_fail_rate = options.distro_fail_rates[1] elif combo == "variant": acceptable_fail_rate = options.variant_fail_rates[0] unacceptable_fail_rate = options.variant_fail_rates[1] elif combo == "task": acceptable_fail_rate = options.task_fail_rates[0] unacceptable_fail_rate = options.task_fail_rates[1] else: acceptable_fail_rate = options.test_fail_rates[0] unacceptable_fail_rate = options.test_fail_rates[1] # Unreliable tests are analyzed from the entire period. update_lifecycle(lifecycle, view_report.view_summary(group_on=report_combo), unreliable_test, True, unacceptable_fail_rate, options.unreliable_test_min_run) # Reliable tests are analyzed from the last period, i.e., last 14 days. (reliable_start_date, reliable_end_date) = view_report.last_period() update_lifecycle(lifecycle, view_report.view_summary(group_on=report_combo, start_date=reliable_start_date, end_date=reliable_end_date), reliable_test, False, acceptable_fail_rate, options.reliable_test_min_run) # Update the lifecycle_file only if there have been changes. if orig_lifecycle != lifecycle: write_yaml_file(options.lifecycle_file, lifecycle)
def main(): values, args = parse_command_line() # If a resmoke.py command wasn't passed in, use a simple version. if not args: args = ["python", "buildscripts/resmoke.py", "--repeat=2"] # Load the dict of tests to run. if values.test_list_file: tests_by_task = _load_tests_file(values.test_list_file) # If there are no tests to run, carry on. if tests_by_task is None: test_results = {"failures": 0, "results": []} _write_report_file(test_results, values.report_file) sys.exit(0) # Run the executor finder. else: # Parse the Evergreen project configuration file. evergreen_conf = evergreen.EvergreenProjectConfig(values.evergreen_file) if values.buildvariant is None: print "Option buildVariant must be specified to find changed tests.\n", \ "Select from the following: \n" \ "\t", "\n\t".join(sorted(evergreen_conf.variant_names)) sys.exit(1) changed_tests = find_changed_tests(values.branch, values.base_commit, values.max_revisions, values.buildvariant, values.check_evergreen) exclude_suites, exclude_tasks, exclude_tests = find_exclude_tests(values.selector_file) changed_tests = filter_tests(changed_tests, exclude_tests) # If there are no changed tests, exit cleanly. if not changed_tests: print "No new or modified tests found." _write_report_file({}, values.test_list_outfile) sys.exit(0) suites = resmokelib.parser.get_suites(values, changed_tests) tests_by_executor = create_executor_list(suites, exclude_suites) tests_by_task = create_task_list(evergreen_conf, values.buildvariant, tests_by_executor, exclude_tasks) if values.test_list_outfile is not None: _write_report_file(tests_by_task, values.test_list_outfile) # If we're not in noExec mode, run the tests. if not values.no_exec: test_results = {"failures": 0, "results": []} for task in sorted(tests_by_task): resmoke_cmd = copy.deepcopy(args) resmoke_cmd.extend(shlex.split(tests_by_task[task]["resmoke_args"])) resmoke_cmd.extend(tests_by_task[task]["tests"]) try: subprocess.check_call(resmoke_cmd, shell=False) except subprocess.CalledProcessError as err: print "Resmoke returned an error with task:", task _save_report_data(test_results, values.report_file, task) _write_report_file(test_results, values.report_file) sys.exit(err.returncode) _save_report_data(test_results, values.report_file, task) _write_report_file(test_results, values.report_file) sys.exit(0)
def test_invalid_path(self): invalid_path = "non_existing_file" with self.assertRaises(IOError): _evergreen.EvergreenProjectConfig(invalid_path)
def setUpClass(cls): cls.conf = _evergreen.EvergreenProjectConfig(TEST_FILE_PATH)
def main(): """ Utility for updating a resmoke.py tag file based on computing test failure rates from the Evergreen API. """ parser = optparse.OptionParser( description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] [test1 test2 ...]") data_options = optparse.OptionGroup( parser, title="Data options", description= ("Options used to configure what historical test failure data to retrieve from" " Evergreen.")) parser.add_option_group(data_options) data_options.add_option( "--project", dest="project", metavar="<project-name>", default=tf.TestHistory.DEFAULT_PROJECT, help="The Evergreen project to analyze. Defaults to '%default'.") data_options.add_option( "--tasks", dest="tasks", metavar="<task1,task2,...>", help= ("The Evergreen tasks to analyze for tagging unreliable tests. If specified in" " additional to having test positional arguments, then only tests that run under the" " specified Evergreen tasks will be analyzed. If omitted, then the list of tasks" " defaults to the non-excluded list of tasks from the specified" " --evergreenProjectConfig file.")) data_options.add_option( "--variants", dest="variants", metavar="<variant1,variant2,...>", default="", help= "The Evergreen build variants to analyze for tagging unreliable tests." ) data_options.add_option( "--distros", dest="distros", metavar="<distro1,distro2,...>", default="", help="The Evergreen distros to analyze for tagging unreliable tests.") data_options.add_option( "--evergreenProjectConfig", dest="evergreen_project_config", metavar="<project-config-file>", default="etc/evergreen.yml", help= ("The Evergreen project configuration file used to get the list of tasks if --tasks is" " omitted. Defaults to '%default'.")) model_options = optparse.OptionGroup( parser, title="Model options", description= ("Options used to configure whether (test,), (test, task)," " (test, task, variant), and (test, task, variant, distro) combinations are" " considered unreliable.")) parser.add_option_group(model_options) model_options.add_option( "--reliableTestMinRuns", type="int", dest="reliable_test_min_runs", metavar="<reliable-min-runs>", default=DEFAULT_CONFIG.reliable_min_runs, help= ("The minimum number of test executions required for a test's failure rate to" " determine whether the test is considered reliable. If a test has fewer than" " <reliable-min-runs> executions, then it cannot be considered unreliable." )) model_options.add_option( "--unreliableTestMinRuns", type="int", dest="unreliable_test_min_runs", metavar="<unreliable-min-runs>", default=DEFAULT_CONFIG.unreliable_min_runs, help= ("The minimum number of test executions required for a test's failure rate to" " determine whether the test is considered unreliable. If a test has fewer than" " <unreliable-min-runs> executions, then it cannot be considered unreliable." )) model_options.add_option( "--testFailRates", type="float", nargs=2, dest="test_fail_rates", metavar="<test-acceptable-fail-rate> <test-unacceptable-fail-rate>", default=DEFAULT_CONFIG.test_fail_rates, help= ("Controls how readily a test is considered unreliable. Each failure rate must be a" " number between 0 and 1 (inclusive) with" " <test-unacceptable-fail-rate> >= <test-acceptable-fail-rate>. If a test fails no" " more than <test-acceptable-fail-rate> in <reliable-days> time, then it is" " considered reliable. Otherwise, if a test fails at least as much as" " <test-unacceptable-fail-rate> in <test-unreliable-days> time, then it is considered" " unreliable. Defaults to %default.")) model_options.add_option( "--taskFailRates", type="float", nargs=2, dest="task_fail_rates", metavar="<task-acceptable-fail-rate> <task-unacceptable-fail-rate>", default=DEFAULT_CONFIG.task_fail_rates, help= ("Controls how readily a (test, task) combination is considered unreliable. Each" " failure rate must be a number between 0 and 1 (inclusive) with" " <task-unacceptable-fail-rate> >= <task-acceptable-fail-rate>. If a (test, task)" " combination fails no more than <task-acceptable-fail-rate> in <reliable-days> time," " then it is considered reliable. Otherwise, if a test fails at least as much as" " <task-unacceptable-fail-rate> in <unreliable-days> time, then it is considered" " unreliable. Defaults to %default.")) model_options.add_option( "--variantFailRates", type="float", nargs=2, dest="variant_fail_rates", metavar= "<variant-acceptable-fail-rate> <variant-unacceptable-fail-rate>", default=DEFAULT_CONFIG.variant_fail_rates, help= ("Controls how readily a (test, task, variant) combination is considered unreliable." " Each failure rate must be a number between 0 and 1 (inclusive) with" " <variant-unacceptable-fail-rate> >= <variant-acceptable-fail-rate>. If a" " (test, task, variant) combination fails no more than <variant-acceptable-fail-rate>" " in <reliable-days> time, then it is considered reliable. Otherwise, if a test fails" " at least as much as <variant-unacceptable-fail-rate> in <unreliable-days> time," " then it is considered unreliable. Defaults to %default.")) model_options.add_option( "--distroFailRates", type="float", nargs=2, dest="distro_fail_rates", metavar="<distro-acceptable-fail-rate> <distro-unacceptable-fail-rate>", default=DEFAULT_CONFIG.distro_fail_rates, help= ("Controls how readily a (test, task, variant, distro) combination is considered" " unreliable. Each failure rate must be a number between 0 and 1 (inclusive) with" " <distro-unacceptable-fail-rate> >= <distro-acceptable-fail-rate>. If a" " (test, task, variant, distro) combination fails no more than" " <distro-acceptable-fail-rate> in <reliable-days> time, then it is considered" " reliable. Otherwise, if a test fails at least as much as" " <distro-unacceptable-fail-rate> in <unreliable-days> time, then it is considered" " unreliable. Defaults to %default.")) model_options.add_option( "--reliableDays", type="int", dest="reliable_days", metavar="<ndays>", default=DEFAULT_CONFIG.reliable_time_period.days, help= ("The time period to analyze when determining if a test has become reliable. Defaults" " to %default day(s).")) model_options.add_option( "--unreliableDays", type="int", dest="unreliable_days", metavar="<ndays>", default=DEFAULT_CONFIG.unreliable_time_period.days, help= ("The time period to analyze when determining if a test has become unreliable." " Defaults to %default day(s).")) parser.add_option( "--resmokeTagFile", dest="tag_file", metavar="<tagfile>", default="etc/test_lifecycle.yml", help= ("The resmoke.py tag file to update. If --metadataRepo is specified, it" " is the relative path in the metadata repository, otherwise it can be" " an absolute path or a relative path from the current directory." " Defaults to '%default'.")) parser.add_option( "--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>", default="[email protected]:mongodb/mongo-test-metadata.git", help=("The repository that contains the lifecycle file. " "It will be cloned in the current working directory. " "Defaults to '%default'.")) parser.add_option( "--referencesFile", dest="references_file", metavar="<references-file>", default="references.yml", help= ("The YAML file in the metadata repository that contains the revision " "mappings. Defaults to '%default'.")) parser.add_option( "--requestBatchSize", type="int", dest="batch_size", metavar="<batch-size>", default=100, help= ("The maximum number of tests to query the Evergreen API for in a single" " request. A higher value for this option will reduce the number of" " roundtrips between this client and Evergreen. Defaults to %default." )) commit_options = optparse.OptionGroup( parser, title="Commit options", description= ("Options used to configure whether and how to commit the updated test" " lifecycle tags.")) parser.add_option_group(commit_options) commit_options.add_option( "--commit", action="store_true", dest="commit", default=False, help="Indicates that the updated tag file should be committed.") commit_options.add_option( "--jiraConfig", dest="jira_config", metavar="<jira-config>", default=None, help= ("The YAML file containing the JIRA access configuration ('user', 'password'," "'server').")) commit_options.add_option( "--gitUserName", dest="git_user_name", metavar="<git-user-name>", default="Test Lifecycle", help= ("The git user name that will be set before committing to the metadata repository." " Defaults to '%default'.")) commit_options.add_option( "--gitUserEmail", dest="git_user_email", metavar="<git-user-email>", default="*****@*****.**", help= ("The git user email address that will be set before committing to the metadata" " repository. Defaults to '%default'.")) logging_options = optparse.OptionGroup( parser, title="Logging options", description= "Options used to configure the logging output of the script.") parser.add_option_group(logging_options) logging_options.add_option( "--logLevel", dest="log_level", metavar="<log-level>", choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO", help= ("The log level. Accepted values are: DEBUG, INFO, WARNING and ERROR." " Defaults to '%default'.")) logging_options.add_option( "--logFile", dest="log_file", metavar="<log-file>", default=None, help= "The destination file for the logs output. Defaults to the standard output." ) (options, tests) = parser.parse_args() if options.distros: warnings.warn(( "Until https://jira.mongodb.org/browse/EVG-1665 is implemented, distro information" " isn't returned by the Evergreen API. This option will therefore be ignored." ), RuntimeWarning) logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level, filename=options.log_file) evg_conf = ci_evergreen.EvergreenProjectConfig( options.evergreen_project_config) use_test_tasks_membership = False tasks = options.tasks.split(",") if options.tasks else [] if not tasks: # If no tasks are specified, then the list of tasks is all. tasks = evg_conf.lifecycle_task_names use_test_tasks_membership = True variants = options.variants.split(",") if options.variants else [] distros = options.distros.split(",") if options.distros else [] config = Config( test_fail_rates=Rates(*options.test_fail_rates), task_fail_rates=Rates(*options.task_fail_rates), variant_fail_rates=Rates(*options.variant_fail_rates), distro_fail_rates=Rates(*options.distro_fail_rates), reliable_min_runs=options.reliable_test_min_runs, reliable_time_period=datetime.timedelta(days=options.reliable_days), unreliable_min_runs=options.unreliable_test_min_runs, unreliable_time_period=datetime.timedelta( days=options.unreliable_days)) validate_config(config) lifecycle_tags_file = make_lifecycle_tags_file(options, config) if not lifecycle_tags_file: sys.exit(1) test_tasks_membership = get_test_tasks_membership(evg_conf) # If no tests are specified then the list of tests is generated from the list of tasks. if not tests: tests = get_tests_from_tasks(tasks, test_tasks_membership) if not options.tasks: use_test_tasks_membership = True commit_first, commit_last = git_commit_range_since("{}.days".format( options.unreliable_days)) commit_prior = git_commit_prior(commit_first) # For efficiency purposes, group the tests and process in batches of batch_size. test_groups = create_batch_groups(create_test_groups(tests), options.batch_size) LOGGER.info("Updating the tags") for tests in test_groups: # Find all associated tasks for the test_group if tasks or tests were not specified. if use_test_tasks_membership: tasks_set = set() for test in tests: tasks_set = tasks_set.union(test_tasks_membership[test]) tasks = list(tasks_set) if not tasks: LOGGER.warning("No tasks found for tests %s, skipping this group.", tests) continue test_history = tf.TestHistory(project=options.project, tests=tests, tasks=tasks, variants=variants, distros=distros) history_data = test_history.get_history_by_revision( start_revision=commit_prior, end_revision=commit_last) report = tf.Report(history_data) update_tags(lifecycle_tags_file.changelog_lifecycle, config, report) # Remove tags that are no longer relevant clean_up_tags(lifecycle_tags_file.changelog_lifecycle, evg_conf) # We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there # have been changes to the tags. In particular, we avoid modifying the file when only the header # comment for the YAML file would change. if lifecycle_tags_file.is_modified(): lifecycle_tags_file.write() if options.commit: commit_ok = lifecycle_tags_file.commit() if not commit_ok: sys.exit(1) else: LOGGER.info("The tags have not been modified.")
def main(): """ Utility for updating a resmoke.py tag file based on computing test failure rates from the Evergreen API. """ parser = optparse.OptionParser( description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] [test1 test2 ...]") data_options = optparse.OptionGroup( parser, title="Data options", description= ("Options used to configure what historical test failure data to retrieve from" " Evergreen.")) parser.add_option_group(data_options) data_options.add_option( "--project", dest="project", metavar="<project-name>", default=tf.TestHistory.DEFAULT_PROJECT, help="The Evergreen project to analyze. Defaults to '%default'.") data_options.add_option( "--tasks", dest="tasks", metavar="<task1,task2,...>", help= ("The Evergreen tasks to analyze for tagging unreliable tests. If specified in" " additional to having test positional arguments, then only tests that run under the" " specified Evergreen tasks will be analyzed. If omitted, then the list of tasks" " defaults to the non-excluded list of tasks from the specified" " --evergreenProjectConfig file.")) data_options.add_option( "--variants", dest="variants", metavar="<variant1,variant2,...>", default="", help= "The Evergreen build variants to analyze for tagging unreliable tests." ) data_options.add_option( "--distros", dest="distros", metavar="<distro1,distro2,...>", default="", help="The Evergreen distros to analyze for tagging unreliable tests.") data_options.add_option( "--evergreenProjectConfig", dest="evergreen_project_config", metavar="<project-config-file>", default="etc/evergreen.yml", help= ("The Evergreen project configuration file used to get the list of tasks if --tasks is" " omitted. Defaults to '%default'.")) model_options = optparse.OptionGroup( parser, title="Model options", description= ("Options used to configure whether (test,), (test, task)," " (test, task, variant), and (test, task, variant, distro) combinations are" " considered unreliable.")) parser.add_option_group(model_options) model_options.add_option( "--reliableTestMinRuns", type="int", dest="reliable_test_min_runs", metavar="<reliable-min-runs>", default=DEFAULT_CONFIG.reliable_min_runs, help= ("The minimum number of test executions required for a test's failure rate to" " determine whether the test is considered reliable. If a test has fewer than" " <reliable-min-runs> executions, then it cannot be considered unreliable." )) model_options.add_option( "--unreliableTestMinRuns", type="int", dest="unreliable_test_min_runs", metavar="<unreliable-min-runs>", default=DEFAULT_CONFIG.unreliable_min_runs, help= ("The minimum number of test executions required for a test's failure rate to" " determine whether the test is considered unreliable. If a test has fewer than" " <unreliable-min-runs> executions, then it cannot be considered unreliable." )) model_options.add_option( "--testFailRates", type="float", nargs=2, dest="test_fail_rates", metavar="<test-acceptable-fail-rate> <test-unacceptable-fail-rate>", default=DEFAULT_CONFIG.test_fail_rates, help= ("Controls how readily a test is considered unreliable. Each failure rate must be a" " number between 0 and 1 (inclusive) with" " <test-unacceptable-fail-rate> >= <test-acceptable-fail-rate>. If a test fails no" " more than <test-acceptable-fail-rate> in <reliable-days> time, then it is" " considered reliable. Otherwise, if a test fails at least as much as" " <test-unacceptable-fail-rate> in <test-unreliable-days> time, then it is considered" " unreliable. Defaults to %default.")) model_options.add_option( "--taskFailRates", type="float", nargs=2, dest="task_fail_rates", metavar="<task-acceptable-fail-rate> <task-unacceptable-fail-rate>", default=DEFAULT_CONFIG.task_fail_rates, help= ("Controls how readily a (test, task) combination is considered unreliable. Each" " failure rate must be a number between 0 and 1 (inclusive) with" " <task-unacceptable-fail-rate> >= <task-acceptable-fail-rate>. If a (test, task)" " combination fails no more than <task-acceptable-fail-rate> in <reliable-days> time," " then it is considered reliable. Otherwise, if a test fails at least as much as" " <task-unacceptable-fail-rate> in <unreliable-days> time, then it is considered" " unreliable. Defaults to %default.")) model_options.add_option( "--variantFailRates", type="float", nargs=2, dest="variant_fail_rates", metavar= "<variant-acceptable-fail-rate> <variant-unacceptable-fail-rate>", default=DEFAULT_CONFIG.variant_fail_rates, help= ("Controls how readily a (test, task, variant) combination is considered unreliable." " Each failure rate must be a number between 0 and 1 (inclusive) with" " <variant-unacceptable-fail-rate> >= <variant-acceptable-fail-rate>. If a" " (test, task, variant) combination fails no more than <variant-acceptable-fail-rate>" " in <reliable-days> time, then it is considered reliable. Otherwise, if a test fails" " at least as much as <variant-unacceptable-fail-rate> in <unreliable-days> time," " then it is considered unreliable. Defaults to %default.")) model_options.add_option( "--distroFailRates", type="float", nargs=2, dest="distro_fail_rates", metavar="<distro-acceptable-fail-rate> <distro-unacceptable-fail-rate>", default=DEFAULT_CONFIG.distro_fail_rates, help= ("Controls how readily a (test, task, variant, distro) combination is considered" " unreliable. Each failure rate must be a number between 0 and 1 (inclusive) with" " <distro-unacceptable-fail-rate> >= <distro-acceptable-fail-rate>. If a" " (test, task, variant, distro) combination fails no more than" " <distro-acceptable-fail-rate> in <reliable-days> time, then it is considered" " reliable. Otherwise, if a test fails at least as much as" " <distro-unacceptable-fail-rate> in <unreliable-days> time, then it is considered" " unreliable. Defaults to %default.")) model_options.add_option( "--reliableDays", type="int", dest="reliable_days", metavar="<ndays>", default=DEFAULT_CONFIG.reliable_time_period.days, help= ("The time period to analyze when determining if a test has become reliable. Defaults" " to %default day(s).")) model_options.add_option( "--unreliableDays", type="int", dest="unreliable_days", metavar="<ndays>", default=DEFAULT_CONFIG.unreliable_time_period.days, help= ("The time period to analyze when determining if a test has become unreliable." " Defaults to %default day(s).")) parser.add_option( "--resmokeTagFile", dest="tag_file", metavar="<tagfile>", default="etc/test_lifecycle.yml", help="The resmoke.py tag file to update. Defaults to '%default'.") parser.add_option( "--requestBatchSize", type="int", dest="batch_size", metavar="<batch-size>", default=100, help= ("The maximum number of tests to query the Evergreen API for in a single" " request. A higher value for this option will reduce the number of" " roundtrips between this client and Evergreen. Defaults to %default." )) (options, tests) = parser.parse_args() if options.distros: warnings.warn(( "Until https://jira.mongodb.org/browse/EVG-1665 is implemented, distro information" " isn't returned by the Evergreen API. This option will therefore be ignored." ), RuntimeWarning) evg_conf = ci_evergreen.EvergreenProjectConfig( options.evergreen_project_config) use_test_tasks_membership = False tasks = options.tasks.split(",") if options.tasks else [] if not tasks: # If no tasks are specified, then the list of tasks is all. tasks = evg_conf.lifecycle_task_names use_test_tasks_membership = True variants = options.variants.split(",") if options.variants else [] distros = options.distros.split(",") if options.distros else [] config = Config( test_fail_rates=Rates(*options.test_fail_rates), task_fail_rates=Rates(*options.task_fail_rates), variant_fail_rates=Rates(*options.variant_fail_rates), distro_fail_rates=Rates(*options.distro_fail_rates), reliable_min_runs=options.reliable_test_min_runs, reliable_time_period=datetime.timedelta(days=options.reliable_days), unreliable_min_runs=options.unreliable_test_min_runs, unreliable_time_period=datetime.timedelta( days=options.unreliable_days)) validate_config(config) lifecycle = ci_tags.TagsConfig.from_file(options.tag_file, cmp_func=compare_tags) test_tasks_membership = get_test_tasks_membership(evg_conf) # If no tests are specified then the list of tests is generated from the list of tasks. if not tests: tests = get_tests_from_tasks(tasks, test_tasks_membership) if not options.tasks: use_test_tasks_membership = True commit_first, commit_last = git_commit_range_since("{}.days".format( options.unreliable_days)) commit_prior = git_commit_prior(commit_first) # For efficiency purposes, group the tests and process in batches of batch_size. test_groups = create_batch_groups(create_test_groups(tests), options.batch_size) for tests in test_groups: # Find all associated tasks for the test_group if tasks or tests were not specified. if use_test_tasks_membership: tasks_set = set() for test in tests: tasks_set = tasks_set.union(test_tasks_membership[test]) tasks = list(tasks_set) if not tasks: print( "Warning - No tasks found for tests {}, skipping this group.". format(tests)) continue test_history = tf.TestHistory(project=options.project, tests=tests, tasks=tasks, variants=variants, distros=distros) history_data = test_history.get_history_by_revision( start_revision=commit_prior, end_revision=commit_last) report = tf.Report(history_data) update_tags(lifecycle, config, report) # Remove tags that are no longer relevant cleanup_tags(lifecycle, evg_conf) # We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there # have been changes to the tags. In particular, we avoid modifying the file when only the header # comment for the YAML file would change. if lifecycle.is_modified(): write_yaml_file(options.tag_file, lifecycle)