def find(project: str, look_back: datetime, evg_api: EvergreenApi, n_threads: int = DEFAULT_THREADS) -> Dict: """ Find test flips in the evergreen project. :param project: Evergreen project to analyze. :param look_back: Look at commits until the given project. :param evg_api: Evergreen API. :param n_threads: Number of threads to use. :return: Dictionary of commits that introduced task flips. """ LOGGER.debug("Starting find_flips iteration") version_iterator = evg_api.versions_by_project(project) with Executor(max_workers=n_threads) as exe: jobs = [] for next_version, version, prev_version in windowed_iter( version_iterator, 3): log = LOGGER.bind(version=version.version_id) log.debug("Starting to look") if version.create_time < look_back: log.debug("done", create_time=version.create_time) break work_item = WorkItem(version, next_version, prev_version) jobs.append(exe.submit(_flips_for_version, work_item)) results = [job.result() for job in jobs] return {r.revision: r.flipped_tasks for r in results if r.flipped_tasks}
def _get_most_recent_version(evg_api: EvergreenApi, project: str) -> Version: """ Fetch the most recent version in an Evergreen project. :param evg_api: An instance of the evg_api client :param project: The name of the evergreen project to analyze :return: evg_api client instance of the Version """ version_iterator = evg_api.versions_by_project(project) return next(version_iterator)
def create_task_mappings( cls, evg_api: EvergreenApi, evergreen_project: str, version_limit: VersionLimit, file_regex: Pattern, module_name: Optional[str] = None, module_file_regex: Optional[Pattern] = None, build_regex: Optional[Pattern] = None, ) -> Tuple[TaskMappings, Optional[str]]: """ Create the task mappings for an evergreen project. Optionally looks at an associated module. :param evg_api: An instance of the evg_api client :param evergreen_project: The name of the evergreen project to analyze. :param version_limit: The point at which to start analyzing versions of the project. :param file_regex: Regex pattern to match changed files against. :param module_name: Name of the module associated with the evergreen project to also analyze :param module_file_regex: Regex pattern to match changed files of the module against. :param build_regex: Regex pattern to match build variant names against. :return: An instance of TaskMappings and version_id of the most recent version analyzed. """ LOGGER.info("Starting to generate task mappings", version_limit=version_limit) project_versions = evg_api.versions_by_project(evergreen_project) task_mappings: Dict = {} module_repo = None branch = None repo_name = None most_recent_version_analyzed = None with TemporaryDirectory() as temp_dir: try: base_repo = _get_evg_project_and_init_repo( evg_api, evergreen_project, temp_dir) except ValueError: LOGGER.warning("Unexpected exception", exc_info=True) raise jobs = [] with Executor(max_workers=MAX_WORKERS) as exe: for next_version, version, prev_version in windowed_iter( project_versions, 3): if not most_recent_version_analyzed: most_recent_version_analyzed = version.version_id LOGGER.info( "Calculated most_recent_version_analyzed", most_recent_version_analyzed= most_recent_version_analyzed, ) if version_limit.check_version_before_limit(version): break if branch is None or repo_name is None: branch = version.branch repo_name = version.repo LOGGER.info( "Processing mappings for version", version=version.version_id, create_time=version.create_time, ) try: diff = _get_diff(base_repo, version.revision, prev_version.revision) except ValueError: LOGGER.warning("Unexpected exception", exc_info=True) continue changed_files = _get_filtered_files( diff, file_regex, repo_name) if module_name: try: cur_module = _get_associated_module( version, module_name) prev_module = _get_associated_module( prev_version, module_name) # even though we don't need the module info for next_version, we run # this check to raise an error if the next version has a config error _get_associated_module(next_version, module_name) except RetryError: LOGGER.warning( "Manifest not found for version, version may have config error", version=version.version_id, prev_version=prev_version.version_id, next_version=next_version.version_id, exc_info=True, ) continue if cur_module is not None and module_repo is None: module_repo = init_repo(temp_dir, cur_module.repo, cur_module.branch, cur_module.owner) module_changed_files = _get_module_changed_files( module_repo, cur_module, prev_module, module_file_regex # type: ignore ) changed_files = changed_files.union( module_changed_files) jobs.append( exe.submit( _process_evg_version, prev_version, version, next_version, build_regex, changed_files, )) for job in jobs: changed_files, flipped_tasks = job.result() _map_tasks_to_files(changed_files, flipped_tasks, task_mappings) return ( TaskMappings(task_mappings, evergreen_project, branch), most_recent_version_analyzed, )