def make_config(expansions_file: Any) -> Config:
"""Group expansions into config."""
expansions = read_config_file(expansions_file)
current_task_name = expansions.get("task_name", "powercycle")
task_names = expansions.get("task_names", "powercycle_smoke_skip_compile")
# Avoid duplicated task names
task_names = {task_name for task_name in task_names.split(" ")}
num_tasks = int(expansions.get("num_tasks", 10))
timeout_params = {
"exec_timeout_secs": int(expansions.get("exec_timeout_secs", 7200)),
"timeout_secs": int(expansions.get("timeout_secs", 1800)),
}
remote_credentials_vars = {
"private_key_file": "src/powercycle.pem",
"private_key_remote": "${__project_aws_ssh_key_value}",
}
set_up_ec2_instance_vars = {
"set_up_retry_count": int(expansions.get("set_up_retry_count", 2)),
}
run_powercycle_vars = {
"run_powercycle_args": expansions.get("run_powercycle_args"),
}
build_variant = expansions.get("build_variant")
distro = expansions.get("distro_id")
return Config(current_task_name, task_names, num_tasks, timeout_params,
remote_credentials_vars, set_up_ec2_instance_vars,
run_powercycle_vars, build_variant, distro)
def main(
verbose: bool,
expansion_file: str,
evg_api_config: str,
selected_tests_config: str,
):
"""
Select tasks to be run based on changed files in a patch build.
:param verbose: Log extra debug information.
:param expansion_file: Configuration file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param selected_tests_config: Location of config file to connect to elected-tests service.
"""
_configure_logging(verbose)
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
selected_tests_service = SelectedTestsService.from_file(
selected_tests_config)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
buildscripts.resmokelib.parser.set_run_options()
task_expansions = read_config.read_config_file(expansion_file)
origin_build_variants = task_expansions[
"selected_tests_buildvariants"].split(" ")
config_dict_of_suites_and_tasks = run(evg_api, evg_conf,
selected_tests_service,
task_expansions, repos,
origin_build_variants)
write_file_dict(SELECTED_TESTS_CONFIG_DIR, config_dict_of_suites_and_tasks)
def get_config_options(cmd_line_options, config_file):
"""
Get the configuration to use for generated tests.
Command line options override config file options.
:param cmd_line_options: Command line options specified.
:param config_file: config file to use.
:return: ConfigOptions to use.
"""
# pylint: disable=too-many-locals
config_file_data = read_config.read_config_file(config_file)
fallback_num_sub_suites = int(
read_config.get_config_value("fallback_num_sub_suites", cmd_line_options, config_file_data,
required=True))
max_sub_suites = read_config.get_config_value("max_sub_suites", cmd_line_options,
config_file_data)
project = read_config.get_config_value("project", cmd_line_options, config_file_data,
required=True)
resmoke_args = read_config.get_config_value("resmoke_args", cmd_line_options, config_file_data,
default="")
resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
config_file_data)
target_resmoke_time = int(
read_config.get_config_value("target_resmoke_time", cmd_line_options, config_file_data,
default=60))
run_multiple_jobs = read_config.get_config_value("run_multiple_jobs", cmd_line_options,
config_file_data, default="true")
task = remove_gen_suffix(
read_config.get_config_value("task_name", cmd_line_options, config_file_data,
required=True))
task_id = read_config.get_config_value("task_id", cmd_line_options, config_file_data,
required=True)
suite = read_config.get_config_value("suite", cmd_line_options, config_file_data, default=task)
variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
required=True)
use_default_timeouts = read_config.get_config_value("use_default_timeouts", cmd_line_options,
config_file_data, default=False)
use_large_distro = read_config.get_config_value("use_large_distro", cmd_line_options,
config_file_data, default=False)
large_distro_name = read_config.get_config_value("large_distro_name", cmd_line_options,
config_file_data)
use_multiversion = read_config.get_config_value("use_multiversion", cmd_line_options,
config_file_data)
is_patch = read_config.get_config_value("is_patch", cmd_line_options, config_file_data)
if is_patch:
is_patch = strtobool(is_patch)
depends_on = split_if_exists(
read_config.get_config_value("depends_on", cmd_line_options, config_file_data))
build_id = read_config.get_config_value("build_id", cmd_line_options, config_file_data)
repeat_suites = int(
read_config.get_config_value("resmoke_repeat_suites", cmd_line_options, config_file_data,
default=1))
return ConfigOptions(build_id, depends_on, fallback_num_sub_suites, is_patch, large_distro_name,
max_sub_suites, project, repeat_suites, resmoke_args, resmoke_jobs_max,
run_multiple_jobs, suite, target_resmoke_time, task, task_id,
use_default_timeouts, use_large_distro, use_multiversion, variant)
def get_config_options(cmd_line_options, config_file):
"""
Get the configuration to use for generated tests.
Command line options override config file options.
:param cmd_line_options: Command line options specified.
:param config_file: config file to use.
:return: ConfigOptions to use.
"""
# pylint: disable=too-many-locals
config_file_data = read_config.read_config_file(config_file)
fallback_num_sub_suites = int(
read_config.get_config_value("fallback_num_sub_suites", cmd_line_options, config_file_data,
required=True))
max_sub_suites = read_config.get_config_value("max_sub_suites", cmd_line_options,
config_file_data)
project = read_config.get_config_value("project", cmd_line_options, config_file_data,
required=True)
resmoke_args = read_config.get_config_value("resmoke_args", cmd_line_options, config_file_data,
default="")
resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
config_file_data)
target_resmoke_time = int(
read_config.get_config_value("target_resmoke_time", cmd_line_options, config_file_data,
default=60))
run_multiple_jobs = read_config.get_config_value("run_multiple_jobs", cmd_line_options,
config_file_data, default="true")
task = remove_gen_suffix(
read_config.get_config_value("task_name", cmd_line_options, config_file_data,
required=True))
task_id = read_config.get_config_value("task_id", cmd_line_options, config_file_data,
required=True)
suite = read_config.get_config_value("suite", cmd_line_options, config_file_data, default=task)
variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
required=True)
use_default_timeouts = read_config.get_config_value("use_default_timeouts", cmd_line_options,
config_file_data, default=False)
use_large_distro = read_config.get_config_value("use_large_distro", cmd_line_options,
config_file_data, default=False)
large_distro_name = read_config.get_config_value("large_distro_name", cmd_line_options,
config_file_data)
use_multiversion = read_config.get_config_value("use_multiversion", cmd_line_options,
config_file_data)
is_patch = read_config.get_config_value("is_patch", cmd_line_options, config_file_data)
if is_patch:
is_patch = strtobool(is_patch)
depends_on = split_if_exists(
read_config.get_config_value("depends_on", cmd_line_options, config_file_data))
build_id = read_config.get_config_value("build_id", cmd_line_options, config_file_data)
repeat_suites = int(
read_config.get_config_value("resmoke_repeat_suites", cmd_line_options, config_file_data,
default=1))
return ConfigOptions(build_id, depends_on, fallback_num_sub_suites, is_patch, large_distro_name,
max_sub_suites, project, repeat_suites, resmoke_args, resmoke_jobs_max,
run_multiple_jobs, suite, target_resmoke_time, task, task_id,
use_default_timeouts, use_large_distro, use_multiversion, variant)
def from_file(cls, filepath, required_keys, defaults, formats):
"""
Create an instance of ConfigOptions based on the given config file.
:param filepath: Path to file containing configuration.
:param required_keys: Set of keys required by this config.
:param defaults: Dict of default values for keys.
:param formats: Dict with functions to format values before returning.
:return: Instance of ConfigOptions.
"""
return cls(read_config.read_config_file(filepath), required_keys, defaults, formats)
def _get_selected_tests_task_configuration(expansion_file):
"""
Look up task config of the selected tests task.
:param expansion_file: Configuration file.
return: Task configuration values.
"""
expansions = read_config.read_config_file(expansion_file)
return {
"name_of_generating_task": expansions["task_name"],
"name_of_generating_build_variant": expansions["build_variant"],
"name_of_generating_build_id": expansions["build_id"]
}
def main(evergreen_api):
"""Execute Main program."""
parser = argparse.ArgumentParser(description=main.__doc__)
parser.add_argument("--expansion-file", dest="expansion_file", type=str,
help="Location of expansions file generated by evergreen.")
cmd_line_options = parser.parse_args()
expansions_file_data = read_config.read_config_file(cmd_line_options.expansion_file)
shrub_config = Configuration()
buildvariant_map = _create_evg_buildvariant_map(expansions_file_data)
_generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, buildvariant_map)
_write_to_file(shrub_config)
def get_config_options(cmd_line_options, config_file):
"""
Get the configuration to use for generated tests.
Command line options override config file options.
:param cmd_line_options: Command line options specified.
:param config_file: config file to use.
:return: ConfigOptions to use.
"""
config_file_data = read_config.read_config_file(config_file)
fallback_num_sub_suites = read_config.get_config_value(
"fallback_num_sub_suites",
cmd_line_options,
config_file_data,
required=True)
max_sub_suites = read_config.get_config_value("max_sub_suites",
cmd_line_options,
config_file_data)
project = read_config.get_config_value("project",
cmd_line_options,
config_file_data,
required=True)
resmoke_args = read_config.get_config_value("resmoke_args",
cmd_line_options,
config_file_data,
default="")
resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max",
cmd_line_options,
config_file_data)
run_multiple_jobs = read_config.get_config_value("run_multiple_jobs",
cmd_line_options,
config_file_data,
default="true")
task = read_config.get_config_value("task",
cmd_line_options,
config_file_data,
required=True)
suite = read_config.get_config_value("suite",
cmd_line_options,
config_file_data,
default=task)
variant = read_config.get_config_value("build_variant",
cmd_line_options,
config_file_data,
required=True)
return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project,
resmoke_args, resmoke_jobs_max, run_multiple_jobs,
suite, task, variant)
def main(expansion_file: str):
"""
Run new or changed tests in repeated mode to validate their stability.
burn_in_tags detects jstests that are new or changed since the last git command and then
runs those tests in a loop to validate their reliability.
\f
:param expansion_file: The expansion file containing the configuration params.
"""
evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
expansions_file_data = read_config.read_config_file(expansion_file)
evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
burn_in(expansions_file_data, evg_conf, evg_api, repos)
def from_file(cls, filepath: str, overwrites: Dict[str, Any],
required_keys: Set[str], defaults: Dict[str, Any],
formats: Dict[str, type]):
"""
Create an instance of SelectedTestsConfigOptions based on the given config file.
:param filepath: Path to file containing configuration.
:param overwrites: Dict of configuration values to overwrite those listed in filepath.
:param required_keys: Set of keys required by this config.
:param defaults: Dict of default values for keys.
:param formats: Dict with functions to format values before returning.
:return: Instance of SelectedTestsConfigOptions.
"""
config_from_file = read_config.read_config_file(filepath)
return cls({
**config_from_file,
**overwrites
}, required_keys, defaults, formats)
def main(expansions_file: str = "expansions.yml") -> None:
"""Implementation."""
logging.basicConfig(
format="[%(levelname)s] %(message)s",
level=logging.INFO,
stream=sys.stdout,
)
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
expansions = read_config_file(expansions_file)
build_id = expansions["build_id"]
current_task_id = expansions["task_id"]
gen_task_name = expansions["gen_task"]
evg_api = get_evergreen_api()
build_tasks = evg_api.tasks_by_build(build_id)
gen_task_id = [task.task_id for task in build_tasks if gen_task_name in task.task_id][0]
gen_task_url = f"{EVERGREEN_HOST}/task/{gen_task_id}"
while evg_api.task_by_id(gen_task_id).is_active():
LOGGER.info(
f"Waiting for '{gen_task_name}' task to generate powercycle tasks:\n{gen_task_url}")
time.sleep(WATCH_INTERVAL_SECS)
build_tasks = evg_api.tasks_by_build(build_id)
powercycle_task_ids = [
task.task_id for task in build_tasks
if not task.display_only and task.task_id != current_task_id and task.task_id != gen_task_id
and "powercycle" in task.task_id
]
LOGGER.info(f"Watching powercycle tasks:\n{get_links(powercycle_task_ids)}")
long_running_task_ids = watch_tasks(powercycle_task_ids, evg_api, WATCH_INTERVAL_SECS)
if long_running_task_ids:
LOGGER.error(
f"Found powercycle tasks that are running for more than {POWERCYCLE_TASK_EXEC_TIMEOUT_SECS} "
f"seconds and most likely something is going wrong in those tasks:\n{get_links(long_running_task_ids)}"
)
LOGGER.error(
"Hopefully hosts from the tasks are still in run at the time you are seeing this "
"and the Build team is able to check them to diagnose the issue.")
sys.exit(1)
def _get_config_options(cmd_line_options, config_file): # pylint: disable=too-many-locals
"""
Get the configuration to use.
Command line options override config files options.
:param cmd_line_options: Command line options specified.
:param config_file: config file to use.
:return: ConfigOptions to use.
"""
config_file_data = read_config.read_config_file(config_file)
num_files = int(
read_config.get_config_value("num_files", cmd_line_options, config_file_data,
required=True))
num_tasks = int(
read_config.get_config_value("num_tasks", cmd_line_options, config_file_data,
required=True))
resmoke_args = read_config.get_config_value("resmoke_args", cmd_line_options, config_file_data,
default="")
npm_command = read_config.get_config_value("npm_command", cmd_line_options, config_file_data,
default="jstestfuzz")
jstestfuzz_vars = read_config.get_config_value("jstestfuzz_vars", cmd_line_options,
config_file_data, default="")
name = read_config.get_config_value("name", cmd_line_options, config_file_data, required=True)
variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
required=True)
continue_on_failure = read_config.get_config_value("continue_on_failure", cmd_line_options,
config_file_data, default="false")
resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
config_file_data, default="0")
should_shuffle = read_config.get_config_value("should_shuffle", cmd_line_options,
config_file_data, default="false")
timeout_secs = read_config.get_config_value("timeout_secs", cmd_line_options, config_file_data,
default="1800")
use_multiversion = read_config.get_config_value("task_path_suffix", cmd_line_options,
config_file_data, default=False)
return ConfigOptions(num_files, num_tasks, resmoke_args, npm_command, jstestfuzz_vars, name,
variant, continue_on_failure, resmoke_jobs_max, should_shuffle,
timeout_secs, use_multiversion)
def _get_config_options(cmd_line_options, config_file): # pylint: disable=too-many-locals
"""
Get the configuration to use.
Command line options override config files options.
:param cmd_line_options: Command line options specified.
:param config_file: config file to use.
:return: ConfigOptions to use.
"""
config_file_data = read_config.read_config_file(config_file)
num_files = int(
read_config.get_config_value("num_files", cmd_line_options, config_file_data,
required=True))
num_tasks = int(
read_config.get_config_value("num_tasks", cmd_line_options, config_file_data,
required=True))
resmoke_args = read_config.get_config_value("resmoke_args", cmd_line_options, config_file_data,
default="")
npm_command = read_config.get_config_value("npm_command", cmd_line_options, config_file_data,
default="jstestfuzz")
jstestfuzz_vars = read_config.get_config_value("jstestfuzz_vars", cmd_line_options,
config_file_data, default="")
name = read_config.get_config_value("name", cmd_line_options, config_file_data, required=True)
variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
required=True)
continue_on_failure = read_config.get_config_value("continue_on_failure", cmd_line_options,
config_file_data, default="false")
resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
config_file_data, default="0")
should_shuffle = read_config.get_config_value("should_shuffle", cmd_line_options,
config_file_data, default="false")
timeout_secs = read_config.get_config_value("timeout_secs", cmd_line_options, config_file_data,
default="1800")
use_multiversion = read_config.get_config_value("task_path_suffix", cmd_line_options,
config_file_data, default=False)
return ConfigOptions(num_files, num_tasks, resmoke_args, npm_command, jstestfuzz_vars, name,
variant, continue_on_failure, resmoke_jobs_max, should_shuffle,
timeout_secs, use_multiversion)
def get_config_options(cmd_line_options, config_file):
"""
Get the configuration to use for generated tests.
Command line options override config file options.
:param cmd_line_options: Command line options specified.
:param config_file: config file to use.
:return: ConfigOptions to use.
"""
config_file_data = read_config.read_config_file(config_file)
fallback_num_sub_suites = read_config.get_config_value(
"fallback_num_sub_suites", cmd_line_options, config_file_data, required=True)
max_sub_suites = read_config.get_config_value("max_sub_suites", cmd_line_options,
config_file_data)
project = read_config.get_config_value("project", cmd_line_options, config_file_data,
required=True)
resmoke_args = read_config.get_config_value("resmoke_args", cmd_line_options, config_file_data,
default="")
resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
config_file_data)
target_resmoke_time = int(
read_config.get_config_value("target_resmoke_time", cmd_line_options, config_file_data,
default=60))
run_multiple_jobs = read_config.get_config_value("run_multiple_jobs", cmd_line_options,
config_file_data, default="true")
task = read_config.get_config_value("task", cmd_line_options, config_file_data, required=True)
suite = read_config.get_config_value("suite", cmd_line_options, config_file_data, default=task)
variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
required=True)
use_large_distro = read_config.get_config_value("use_large_distro", cmd_line_options,
config_file_data, default=False)
large_distro_name = read_config.get_config_value("large_distro_name", cmd_line_options,
config_file_data)
return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project, resmoke_args,
resmoke_jobs_max, target_resmoke_time, run_multiple_jobs, suite, task,
variant, use_large_distro, large_distro_name)
def main(
verbose: bool,
expansion_file: str,
evg_api_config: str,
selected_tests_config: str,
):
"""
Select tasks to be run based on changed files in a patch build.
:param verbose: Log extra debug information.
:param expansion_file: Configuration file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param selected_tests_config: Location of config file to connect to elected-tests service.
"""
_configure_logging(verbose)
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
selected_tests_service = SelectedTestsService.from_file(
selected_tests_config)
repo = Repo(".")
changed_files = find_changed_files(repo)
buildscripts.resmokelib.parser.set_options()
LOGGER.debug("Found changed files", files=changed_files)
selected_tests_variant_expansions = read_config.read_config_file(
expansion_file)
origin_build_variants = selected_tests_variant_expansions[
"selected_tests_buildvariants"].split(" ")
config_dict_of_suites_and_tasks = run(evg_api, evg_conf,
selected_tests_service,
selected_tests_variant_expansions,
changed_files, origin_build_variants)
write_file_dict(SELECTED_TESTS_CONFIG_DIR, config_dict_of_suites_and_tasks)
