def generate_exclude_yaml(task_path_suffix: str, output: str) -> None: # pylint: disable=too-many-locals """ Create a tag file associating multiversion tests to tags for exclusion. Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the last-lts branch to determine which tests should be blacklisted. """ enable_logging() backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE) # Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts # binary we are running tests against. We do this by using the commit hash from the last-lts # mongo shell executable. last_lts_commit_hash = get_backports_required_last_lts_hash(task_path_suffix) # Get the yaml contents from the last-lts commit. backports_required_last_lts = get_last_lts_yaml(last_lts_commit_hash) def diff(list1, list2): return [elem for elem in (list1 or []) if elem not in (list2 or [])] suites_latest = backports_required_latest["suites"] or {} # Check if the changed syntax for etc/backports_required_multiversion.yml has been backported. # This variable and all branches where it's not set can be deleted after backporting the change. change_backported = "all" in backports_required_last_lts.keys() if change_backported: always_exclude = diff(backports_required_latest["all"], backports_required_last_lts["all"]) suites_last_lts: defaultdict = defaultdict(list, backports_required_last_lts["suites"]) else: always_exclude = backports_required_latest["all"] or [] suites_last_lts = defaultdict(list, backports_required_last_lts) tags = _tags.TagsConfig() # Tag tests that are excluded from every suite. for elem in always_exclude: tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG) # Tag tests that are excluded on a suite-by-suite basis. for suite in suites_latest.keys(): test_set = set() for elem in diff(suites_latest[suite], suites_last_lts[suite]): test_set.add(elem["test_file"]) for test in test_set: tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}") LOGGER.info(f"Writing exclude tags to {output}.") tags.write_file(filename=output, preamble="Tag file that specifies exclusions from multiversion suites.")
def test_tag_order_custom_cmp(self): test_kind = "js_test" test_pattern = "jstests/core/example.js" def custom_cmp(tag_a, tag_b): return cmp(tag_a.split("|"), tag_b.split("|")) conf = _tags.TagsConfig(TEST_FILE_PATH, cmp_func=custom_cmp) tags = conf.get_tags(test_kind, test_pattern) self.assertEqual(["tag1", "tag2", "tag3"], tags) # Add a tag that should be at the start. conf.add_tag(test_kind, test_pattern, "ta|g2") tags = conf.get_tags(test_kind, test_pattern) self.assertEqual(["ta|g2", "tag1", "tag2", "tag3"], tags) # Add a tag that should be in the middle. conf.add_tag(test_kind, test_pattern, "tag1|aaa") tags = conf.get_tags(test_kind, test_pattern) self.assertEqual(["ta|g2", "tag1", "tag1|aaa", "tag2", "tag3"], tags)
def setUp(self): lifecycle = ci_tags.TagsConfig({"selector": {}}) self.summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog( lifecycle)
def main(): required_options = [ "project", "reliable_test_min_run", "unreliable_test_min_run", "test_fail_rates", ] parser = optparse.OptionParser( description=__doc__, usage="Usage: %prog [options] test1 test2 ...") parser.add_option("--project", dest="project", default=None, help="Evergreen project to analyze [REQUIRED].") parser.add_option( "--reliableTestMinimumRun", dest="reliable_test_min_run", default=None, type="int", help="Minimum number of tests runs for test to be considered as reliable" " [REQUIRED].") parser.add_option( "--unreliableTestMinimumRun", dest="unreliable_test_min_run", default=None, type="int", help= "Minimum number of tests runs for test to be considered as unreliable" " [REQUIRED].") parser.add_option( "--testFailRates", dest="test_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help="Test fail rates: acceptable fail rate and unacceptable fail rate" " Specify floating numbers between 0.0 and 1.0 [REQUIRED].") parser.add_option( "--taskFailRates", dest="task_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help="Task fail rates: acceptable fail rate and unacceptable fail rate." " Specify floating numbers between 0.0 and 1.0." " Uses --test-fail-rates if unspecified.") parser.add_option( "--variantFailRates", dest="variant_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help= "Variant fail rates: acceptable fail rate and unacceptable fail rate." " Specify floating numbers between 0.0 and 1.0." " Uses --task-fail-rates if unspecified.") parser.add_option( "--distroFailRates", dest="distro_fail_rates", metavar="ACCEPTABLE-FAILRATE UNACCEPTABLE-FAILRATE", default=None, type="float", nargs=2, help= "Distro fail rates: acceptable fail rate and unacceptable fail rate." " Specify floating numbers between 0.0 and 1.0." " Uses --variant-fail-rates if unspecified.") parser.add_option( "--tasks", dest="tasks", default=None, help="Names of tasks to analyze for tagging unreliable tests." " If specified and no tests are specified, then only tests" " associated with the tasks will be analyzed." " If unspecified and no tests are specified, the list of tasks will be" " the non-excluded list of tasks from the file specified by" " '--evergreenYML'.") parser.add_option( "--variants", dest="variants", default="", help="Names of variants to analyze for tagging unreliable tests.") parser.add_option( "--distros", dest="distros", default="", help= "Names of distros to analyze for tagging unreliable tests [UNUSED].") parser.add_option("--evergreenYML", dest="evergreen_yml", default="etc/evergreen.yml", help="Evergreen YML file used to get the list of tasks," " defaults to '%default'.") parser.add_option( "--lifecycleFile", dest="lifecycle_file", default="etc/test_lifecycle.yml", help="Evergreen lifecycle file to update, defaults to '%default'.") parser.add_option( "--reliableDays", dest="reliable_days", default=7, type="int", help= "Number of days to check for reliable tests, defaults to '%default'.") parser.add_option( "--unreliableDays", dest="unreliable_days", default=28, type="int", help= "Number of days to check for unreliable tests, defaults to '%default'." ) parser.add_option("--batchGroupSize", dest="batch_size", default=100, type="int", help="Size of test batch group, defaults to '%default'.") (options, tests) = parser.parse_args() for option in required_options: if not getattr(options, option): parser.print_help() parser.error("Missing required option") evg_conf = evergreen.EvergreenProjectConfig(options.evergreen_yml) use_test_tasks_membership = False tasks = options.tasks.split(",") if options.tasks else [] if not tasks: # If no tasks are specified, then the list of tasks is all. tasks = evg_conf.lifecycle_task_names use_test_tasks_membership = True variants = options.variants.split(",") if options.variants else [] distros = options.distros.split(",") if options.distros else [] check_fail_rates("Test", options.test_fail_rates[0], options.test_fail_rates[1]) # The less specific failures rates are optional and default to a lower level value. if not options.task_fail_rates: options.task_fail_rates = options.test_fail_rates else: check_fail_rates("Task", options.task_fail_rates[0], options.task_fail_rates[1]) if not options.variant_fail_rates: options.variant_fail_rates = options.task_fail_rates else: check_fail_rates("Variant", options.variant_fail_rates[0], options.variant_fail_rates[1]) if not options.distro_fail_rates: options.distro_fail_rates = options.variant_fail_rates else: check_fail_rates("Distro", options.distro_fail_rates[0], options.distro_fail_rates[1]) check_days("Reliable days", options.reliable_days) check_days("Unreliable days", options.unreliable_days) lifecycle = tags.TagsConfig(options.lifecycle_file, cmp_func=compare_tags) test_tasks_membership = get_test_tasks_membership(evg_conf) # If no tests are specified then the list of tests is generated from the list of tasks. if not tests: tests = get_tests_from_tasks(tasks, test_tasks_membership) if not options.tasks: use_test_tasks_membership = True commit_first, commit_last = git_commit_range_since("{}.days".format( options.unreliable_days)) commit_prior = git_commit_prior(commit_first) # For efficiency purposes, group the tests and process in batches of batch_size. test_groups = create_batch_groups(create_test_groups(tests), options.batch_size) for tests in test_groups: # Find all associated tasks for the test_group if tasks or tests were not specified. if use_test_tasks_membership: tasks_set = set() for test in tests: tasks_set = tasks_set.union(test_tasks_membership[test]) tasks = list(tasks_set) if not tasks: print( "Warning - No tasks found for tests {}, skipping this group.". format(tests)) continue report = tf.HistoryReport(period_type="revision", start=commit_prior, end=commit_last, group_period=options.reliable_days, project=options.project, tests=tests, tasks=tasks, variants=variants, distros=distros) view_report = report.generate_report() # We build up report_combo to check for more specific test failures rates. report_combo = [] # TODO EVG-1665: Uncomment this line once this has been supported. # for combo in ["test", "task", "variant", "distro"]: for combo in ["test", "task", "variant"]: report_combo.append(combo) if combo == "distro": acceptable_fail_rate = options.distro_fail_rates[0] unacceptable_fail_rate = options.distro_fail_rates[1] elif combo == "variant": acceptable_fail_rate = options.variant_fail_rates[0] unacceptable_fail_rate = options.variant_fail_rates[1] elif combo == "task": acceptable_fail_rate = options.task_fail_rates[0] unacceptable_fail_rate = options.task_fail_rates[1] else: acceptable_fail_rate = options.test_fail_rates[0] unacceptable_fail_rate = options.test_fail_rates[1] # Unreliable tests are analyzed from the entire period. update_lifecycle(lifecycle, view_report.view_summary(group_on=report_combo), unreliable_test, True, unacceptable_fail_rate, options.unreliable_test_min_run) # Reliable tests are analyzed from the last period, i.e., last 14 days. (reliable_start_date, reliable_end_date) = view_report.last_period() update_lifecycle( lifecycle, view_report.view_summary(group_on=report_combo, start_date=reliable_start_date, end_date=reliable_end_date), reliable_test, False, acceptable_fail_rate, options.reliable_test_min_run) # Update the lifecycle_file only if there have been changes. if lifecycle.is_modified(): write_yaml_file(options.lifecycle_file, lifecycle)
def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Logger) -> None: """ Create a tag file associating multiversion tests to tags for exclusion. Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the last-lts and/or last-continuous branch to determine which tests should be denylisted. """ output = os.path.abspath(output) location, _ = os.path.split(output) if not os.path.isdir(location): logger.info(f"Cannot write to {output}. Not generating tag file.") return backports_required_latest = read_yaml_file( os.path.join(ETC_DIR, BACKPORTS_REQUIRED_FILE)) # Get the state of the backports_required_for_multiversion_tests.yml file for the old # binary we are running tests against. We do this by using the commit hash from the old # mongo shell executable. shell_version = { MultiversionOptions.LAST_LTS: multiversionconstants.LAST_LTS_MONGO_BINARY, MultiversionOptions.LAST_CONTINUOUS: multiversionconstants.LAST_CONTINUOUS_MONGO_BINARY, }[old_bin_version] old_version_commit_hash = get_backports_required_hash_for_shell_version( mongo_shell_path=shell_version) # Get the yaml contents from the old commit. logger.info( f"Downloading file from commit hash of old branch {old_version_commit_hash}" ) backports_required_old = get_old_yaml(old_version_commit_hash) def diff(list1, list2): return [elem for elem in (list1 or []) if elem not in (list2 or [])] def get_suite_exclusions(version_key): _suites_latest = backports_required_latest[version_key]["suites"] or {} # Check if the changed syntax for etc/backports_required_for_multiversion_tests.yml has been # backported. # This variable and all branches where it's not set can be deleted after backporting the change. change_backported = version_key in backports_required_old.keys() if change_backported: _always_exclude = diff( backports_required_latest[version_key]["all"], backports_required_old[version_key]["all"]) _suites_old: defaultdict = defaultdict( list, backports_required_old[version_key]["suites"]) else: _always_exclude = diff( backports_required_latest[version_key]["all"], backports_required_old["all"]) _suites_old: defaultdict = defaultdict( list, backports_required_old["suites"]) return _suites_latest, _suites_old, _always_exclude suites_latest, suites_old, always_exclude = get_suite_exclusions( old_bin_version.replace("_", "-")) tags = _tags.TagsConfig() # Tag tests that are excluded from every suite. for elem in always_exclude: tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG) # Tag tests that are excluded on a suite-by-suite basis. for suite in suites_latest.keys(): test_set = set() for elem in diff(suites_latest[suite], suites_old[suite]): test_set.add(elem["test_file"]) for test in test_set: tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}") logger.info(f"Writing exclude tags to {output}.") tags.write_file( filename=output, preamble="Tag file that specifies exclusions from multiversion suites." )
def test_invalid_path(self): invalid_path = "non_existing_file" with self.assertRaises(IOError): _tags.TagsConfig(invalid_path)
def setUp(self): self.conf = _tags.TagsConfig(TEST_FILE_PATH)