def mocked_api(): """Return an Evergreen API with a mocked session.""" api = EvergreenApi() api.session = MagicMock() response_mock = MagicMock() response_mock.status_code = 200 response_mock.json.return_value = [{'create_time': "2019-03-10T02:43:49.330"}] api.session.get.return_value = response_mock return api
def _activate_archive_debug_symbols(evg_api: EvergreenApi, task_list): debug_iter = filter( lambda tsk: tsk.display_name == ACTIVATE_ARCHIVE_DIST_TEST_DEBUG_TASK, task_list) activate_symbol_tasks = list(debug_iter) if len(activate_symbol_tasks) == 1: activated_symbol_task = activate_symbol_tasks[0] if not activated_symbol_task.activated: LOGGER.info("Activating debug symbols archival", task_id=activated_symbol_task.task_id) evg_api.configure_task(activated_symbol_task.task_id, activated=True)
def activate_task(build_id: str, task_name: str, evg_api: EvergreenApi) -> None: """ Activate the given task in the specified build. :param build_id: Build to activate task in. :param task_name: Name of task to activate. :param evg_api: Evergreen API client. """ build = evg_api.build_by_id(build_id) task_list = build.get_tasks() for task in task_list: if task.display_name == task_name: LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) evg_api.configure_task(task.task_id, activated=True)
def find(project: str, look_back: datetime, evg_api: EvergreenApi, n_threads: int = DEFAULT_THREADS) -> Dict: """ Find test flips in the evergreen project. :param project: Evergreen project to analyze. :param look_back: Look at commits until the given project. :param evg_api: Evergreen API. :param n_threads: Number of threads to use. :return: Dictionary of commits that introduced task flips. """ LOGGER.debug("Starting find_flips iteration") version_iterator = evg_api.versions_by_project(project) with Executor(max_workers=n_threads) as exe: jobs = [] for next_version, version, prev_version in windowed_iter( version_iterator, 3): log = LOGGER.bind(version=version.version_id) log.debug("Starting to look") if version.create_time < look_back: log.debug("done", create_time=version.create_time) break work_item = WorkItem(version, next_version, prev_version) jobs.append(exe.submit(_flips_for_version, work_item)) results = [job.result() for job in jobs] return {r.revision: r.flipped_tasks for r in results if r.flipped_tasks}
def remove_task_configs_already_in_build(task_configs: Dict[str, Dict], evg_api: EvergreenApi, build_variant_config: Variant, version_id: str) -> None: """ Remove the task_configs that exist for tasks that have been pulled into the build manually. :param task_configs: The task configurations for the tasks to be generated. :param evg_api: Evergreen API object. :param build_variant_config: Config of build variant to collect task info from. :param version_id: The version_id of the version running selected tests. """ version = evg_api.version_by_id(version_id) try: build = version.build_by_variant(build_variant_config.name) except KeyError: LOGGER.debug("No build exists on this build variant for this version yet", variant=build_variant_config.name) build = None if build: tasks_already_in_build = build.get_tasks() for task in tasks_already_in_build: if task.display_name in task_configs: LOGGER.info( "Will not generate task that has already been pulled into the build manually", variant=build_variant_config.name, task_already_in_build=task.display_name) del task_configs[task.display_name]
def _get_most_recent_version(evg_api: EvergreenApi, project: str) -> Version: """ Fetch the most recent version in an Evergreen project. :param evg_api: An instance of the evg_api client :param project: The name of the evergreen project to analyze :return: evg_api client instance of the Version """ version_iterator = evg_api.versions_by_project(project) return next(version_iterator)
def activate_task(build_id: str, task_name: str, evg_api: EvergreenApi) -> None: """ Activate the given task in the specified build. :param build_id: Build to activate task in. :param task_name: Name of task to activate. :param evg_api: Evergreen API client. """ build = evg_api.build_by_id(build_id) task_list = build.get_tasks() for task in task_list: if task.display_name == task_name: LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) evg_api.configure_task(task.task_id, activated=True) if any(ARCHIVE_DIST_TEST_TASK in dependency["id"] for dependency in task.depends_on): _activate_archive_debug_symbols(evg_api, task_list)
def get_evg_project(evg_api: EvergreenApi, project: str) -> Optional[Project]: """ Fetch an Evergreen project's info from the Evergreen API. :param evg_api: An instance of the evg_api client :param project: The name of the evergreen project to analyze. :return: evg_api client instance of the project """ for evergreen_project in evg_api.all_projects(): if evergreen_project.identifier == project: return evergreen_project return None
def find_build_for_previous_compile_task(evg_api: EvergreenApi, target: TargetBuild) -> Build: """ Find build_id of the base revision. :param evg_api: Evergreen.py object. :param target: Build being targeted. :return: build_id of the base revision. """ project_prefix = target.project.replace("-", "_") version_of_base_revision = "{}_{}".format(project_prefix, target.revision) version = evg_api.version_by_id(version_of_base_revision) build = version.build_by_variant(target.build_variant) return build
def activate_task(expansions: EvgExpansions, evg_api: EvergreenApi) -> None: """ Activate the given task in the specified build. :param expansions: Evergreen expansions file contents. :param evg_api: Evergreen API client. """ if expansions.task == BURN_IN_TAGS: version = evg_api.version_by_id(expansions.version_id) for base_build_variant in expansions.burn_in_tag_buildvariants_list: build_variant = f"{base_build_variant}-required" try: build = version.build_by_variant(build_variant) except KeyError: LOGGER.warning( "It is likely nothing to burn_in, so burn_in_tags build variant" " was not generated. Skipping...", build_variant=build_variant) continue task_list = build.get_tasks() for task in task_list: if task.display_name == BURN_IN_TESTS: LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) evg_api.configure_task(task.task_id, activated=True) else: build = evg_api.build_by_id(expansions.build_id) task_list = build.get_tasks() for task in task_list: if task.display_name == expansions.task: LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) evg_api.configure_task(task.task_id, activated=True)
def cli(ctx, display_format): ctx.ensure_object(dict) ctx.obj['api'] = EvergreenApi.get_api(use_config_file=True) ctx.obj['format'] = display_format
def mocked_api(mocked_api_response): """Return an Evergreen API with a mocked session.""" api = EvergreenApi() api._session = MagicMock() api._session.request.return_value = mocked_api_response return api
options = self.parse_commandline() self.options = options if options.verbose: enable_logging() if not should_tasks_be_generated(self.evergreen_api, self.config_options.task_id): LOGGER.info("Not generating configuration due to previous successful generation.") return LOGGER.debug("Starting execution for options %s", options) LOGGER.debug("config options %s", self.config_options) end_date = datetime.datetime.utcnow().replace(microsecond=0) start_date = end_date - datetime.timedelta(days=options.duration_days) prepare_directory_for_suite(CONFIG_DIR) suites = self.calculate_suites(start_date, end_date) LOGGER.debug("Creating %d suites for %s", len(suites), self.config_options.task) render_suite(suites, self.config_options.suite) render_misc_suite(self.test_list, self.config_options.suite) self.write_evergreen_configuration(suites, self.config_options.task) if __name__ == "__main__": Main(EvergreenApi.get_api(config_file=CONFIG_FILE)).main()
def create_task_mappings( cls, evg_api: EvergreenApi, evergreen_project: str, version_limit: VersionLimit, file_regex: Pattern, module_name: Optional[str] = None, module_file_regex: Optional[Pattern] = None, build_regex: Optional[Pattern] = None, ) -> Tuple[TaskMappings, Optional[str]]: """ Create the task mappings for an evergreen project. Optionally looks at an associated module. :param evg_api: An instance of the evg_api client :param evergreen_project: The name of the evergreen project to analyze. :param version_limit: The point at which to start analyzing versions of the project. :param file_regex: Regex pattern to match changed files against. :param module_name: Name of the module associated with the evergreen project to also analyze :param module_file_regex: Regex pattern to match changed files of the module against. :param build_regex: Regex pattern to match build variant names against. :return: An instance of TaskMappings and version_id of the most recent version analyzed. """ LOGGER.info("Starting to generate task mappings", version_limit=version_limit) project_versions = evg_api.versions_by_project(evergreen_project) task_mappings: Dict = {} module_repo = None branch = None repo_name = None most_recent_version_analyzed = None with TemporaryDirectory() as temp_dir: try: base_repo = _get_evg_project_and_init_repo( evg_api, evergreen_project, temp_dir) except ValueError: LOGGER.warning("Unexpected exception", exc_info=True) raise jobs = [] with Executor(max_workers=MAX_WORKERS) as exe: for next_version, version, prev_version in windowed_iter( project_versions, 3): if not most_recent_version_analyzed: most_recent_version_analyzed = version.version_id LOGGER.info( "Calculated most_recent_version_analyzed", most_recent_version_analyzed= most_recent_version_analyzed, ) if version_limit.check_version_before_limit(version): break if branch is None or repo_name is None: branch = version.branch repo_name = version.repo LOGGER.info( "Processing mappings for version", version=version.version_id, create_time=version.create_time, ) try: diff = _get_diff(base_repo, version.revision, prev_version.revision) except ValueError: LOGGER.warning("Unexpected exception", exc_info=True) continue changed_files = _get_filtered_files( diff, file_regex, repo_name) if module_name: try: cur_module = _get_associated_module( version, module_name) prev_module = _get_associated_module( prev_version, module_name) # even though we don't need the module info for next_version, we run # this check to raise an error if the next version has a config error _get_associated_module(next_version, module_name) except RetryError: LOGGER.warning( "Manifest not found for version, version may have config error", version=version.version_id, prev_version=prev_version.version_id, next_version=next_version.version_id, exc_info=True, ) continue if cur_module is not None and module_repo is None: module_repo = init_repo(temp_dir, cur_module.repo, cur_module.branch, cur_module.owner) module_changed_files = _get_module_changed_files( module_repo, cur_module, prev_module, module_file_regex # type: ignore ) changed_files = changed_files.union( module_changed_files) jobs.append( exe.submit( _process_evg_version, prev_version, version, next_version, build_regex, changed_files, )) for job in jobs: changed_files, flipped_tasks = job.result() _map_tasks_to_files(changed_files, flipped_tasks, task_mappings) return ( TaskMappings(task_mappings, evergreen_project, branch), most_recent_version_analyzed, )
if options.verbose: enable_logging() if not should_tasks_be_generated(self.evergreen_api, self.config_options.task_id): LOGGER.info( "Not generating configuration due to previous successful generation." ) return LOGGER.debug("Starting execution for options %s", options) LOGGER.debug("config options %s", self.config_options) end_date = datetime.datetime.utcnow().replace(microsecond=0) start_date = end_date - datetime.timedelta(days=options.duration_days) prepare_directory_for_suite(CONFIG_DIR) suites = self.calculate_suites(start_date, end_date) LOGGER.debug("Creating %d suites for %s", len(suites), self.config_options.task) render_suite(suites, self.config_options.suite) render_misc_suite(self.test_list, self.config_options.suite) self.write_evergreen_configuration(suites, self.config_options.task) if __name__ == "__main__": Main(EvergreenApi.get_api(config_file=CONFIG_FILE)).main()