def find(project: str,
look_back: datetime,
evg_api: EvergreenApi,
n_threads: int = DEFAULT_THREADS) -> Dict:
"""
Find test flips in the evergreen project.
:param project: Evergreen project to analyze.
:param look_back: Look at commits until the given project.
:param evg_api: Evergreen API.
:param n_threads: Number of threads to use.
:return: Dictionary of commits that introduced task flips.
"""
LOGGER.debug("Starting find_flips iteration")
version_iterator = evg_api.versions_by_project(project)
with Executor(max_workers=n_threads) as exe:
jobs = []
for next_version, version, prev_version in windowed_iter(
version_iterator, 3):
log = LOGGER.bind(version=version.version_id)
log.debug("Starting to look")
if version.create_time < look_back:
log.debug("done", create_time=version.create_time)
break
work_item = WorkItem(version, next_version, prev_version)
jobs.append(exe.submit(_flips_for_version, work_item))
results = [job.result() for job in jobs]
return {r.revision: r.flipped_tasks for r in results if r.flipped_tasks}
def _get_most_recent_version(evg_api: EvergreenApi, project: str) -> Version:
"""
Fetch the most recent version in an Evergreen project.
:param evg_api: An instance of the evg_api client
:param project: The name of the evergreen project to analyze
:return: evg_api client instance of the Version
"""
version_iterator = evg_api.versions_by_project(project)
return next(version_iterator)
def create_task_mappings(
cls,
evg_api: EvergreenApi,
evergreen_project: str,
version_limit: VersionLimit,
file_regex: Pattern,
module_name: Optional[str] = None,
module_file_regex: Optional[Pattern] = None,
build_regex: Optional[Pattern] = None,
) -> Tuple[TaskMappings, Optional[str]]:
"""
Create the task mappings for an evergreen project. Optionally looks at an associated module.
:param evg_api: An instance of the evg_api client
:param evergreen_project: The name of the evergreen project to analyze.
:param version_limit: The point at which to start analyzing versions of the project.
:param file_regex: Regex pattern to match changed files against.
:param module_name: Name of the module associated with the evergreen project to also analyze
:param module_file_regex: Regex pattern to match changed files of the module against.
:param build_regex: Regex pattern to match build variant names against.
:return: An instance of TaskMappings and version_id of the most recent version analyzed.
"""
LOGGER.info("Starting to generate task mappings",
version_limit=version_limit)
project_versions = evg_api.versions_by_project(evergreen_project)
task_mappings: Dict = {}
module_repo = None
branch = None
repo_name = None
most_recent_version_analyzed = None
with TemporaryDirectory() as temp_dir:
try:
base_repo = _get_evg_project_and_init_repo(
evg_api, evergreen_project, temp_dir)
except ValueError:
LOGGER.warning("Unexpected exception", exc_info=True)
raise
jobs = []
with Executor(max_workers=MAX_WORKERS) as exe:
for next_version, version, prev_version in windowed_iter(
project_versions, 3):
if not most_recent_version_analyzed:
most_recent_version_analyzed = version.version_id
LOGGER.info(
"Calculated most_recent_version_analyzed",
most_recent_version_analyzed=
most_recent_version_analyzed,
)
if version_limit.check_version_before_limit(version):
break
if branch is None or repo_name is None:
branch = version.branch
repo_name = version.repo
LOGGER.info(
"Processing mappings for version",
version=version.version_id,
create_time=version.create_time,
)
try:
diff = _get_diff(base_repo, version.revision,
prev_version.revision)
except ValueError:
LOGGER.warning("Unexpected exception", exc_info=True)
continue
changed_files = _get_filtered_files(
diff, file_regex, repo_name)
if module_name:
try:
cur_module = _get_associated_module(
version, module_name)
prev_module = _get_associated_module(
prev_version, module_name)
# even though we don't need the module info for next_version, we run
# this check to raise an error if the next version has a config error
_get_associated_module(next_version, module_name)
except RetryError:
LOGGER.warning(
"Manifest not found for version, version may have config error",
version=version.version_id,
prev_version=prev_version.version_id,
next_version=next_version.version_id,
exc_info=True,
)
continue
if cur_module is not None and module_repo is None:
module_repo = init_repo(temp_dir, cur_module.repo,
cur_module.branch,
cur_module.owner)
module_changed_files = _get_module_changed_files(
module_repo,
cur_module,
prev_module,
module_file_regex # type: ignore
)
changed_files = changed_files.union(
module_changed_files)
jobs.append(
exe.submit(
_process_evg_version,
prev_version,
version,
next_version,
build_regex,
changed_files,
))
for job in jobs:
changed_files, flipped_tasks = job.result()
_map_tasks_to_files(changed_files, flipped_tasks,
task_mappings)
return (
TaskMappings(task_mappings, evergreen_project, branch),
most_recent_version_analyzed,
)