def GetPossibleRevertInfoFromRevision(revision):
"""Parse message to get information of reverting and reverted cls."""
git_repo = CachedGitilesRepository(
HttpClientAppengine(),
'https://chromium.googlesource.com/chromium/src.git')
change_log = git_repo.GetChangeLog(revision)
if not change_log: # pragma: no cover
return {}
reverted_revision = change_log.reverted_revision
if not reverted_revision:
return {}
reverted_cl_change_log = git_repo.GetChangeLog(reverted_revision)
data = {
'action': 'Reverted',
'fixed_revision': reverted_revision,
'fixed_cl_review_url': (reverted_cl_change_log.code_review_url
if reverted_cl_change_log else None),
'fixed_cl_commit_position': (reverted_cl_change_log.commit_position
if reverted_cl_change_log else None),
'fixing_revision': revision,
'fixing_cl_review_url': change_log.code_review_url,
'fixing_cl_commit_position': change_log.commit_position
}
return data
def PullChangeLogs(start_revision,
end_revision,
repo_url=CHROMIUM_GIT_REPOSITORY_URL,
ref=None,
**kwargs):
"""Pulls change logs for CLs between start_revision and end_revision.
Args:
start_revision (str): Start revision of the range, excluded.
end_revision (str): End revision of the range, included. If end_revision is
None, pulls all changes after start_revision.
repo_url (str): Url of the git repo. Default to chromium repo url.
kwargs(dict): Keyword arguments passed as additional params for the query.
Returns:
A dict with the following form:
{
'git_hash_revision1': common.change_log.ChangeLog.ToDict(),
...
}
"""
if not start_revision:
return {}
git_repo = CachedGitilesRepository(FinditHttpClient(), repo_url, ref)
change_logs = {}
change_log_list = git_repo.GetChangeLogs(start_revision, end_revision,
**kwargs)
for change_log in change_log_list:
change_logs[change_log.revision] = change_log
return change_logs
def MapCommitPositionsToGitHashes(end_revision,
end_commit_position,
start_commit_position,
repo_url=CHROMIUM_GIT_REPOSITORY_URL,
ref=None):
"""Gets git_hashes of commit_positions between start_commit_position and
end_commit_position, both ends are included.
Args:
end_revision (str): Revision of the end commit.
end_commit_position (int): Commit position of the end commit.
start_commit_position (int): Commit position of the start commit.
It cannot be greater than end_commit_position.
repo_url (str): Url of the git repo. Default to chromium repo url.
ref (str): Name of the ref.
Returns:
dict: A map of commit_positions in range to the corresponding git_hashes.
For example, return
{
1: 'rev1',
2: 'rev2',
3: 'rev3'
}
if end_commit_position is 3 and start_commit_position is 1.
"""
assert start_commit_position <= end_commit_position, (
'start_commit_position {} is greater than end_commit_position {}'.
format(start_commit_position, end_commit_position))
git_repo = CachedGitilesRepository(FinditHttpClient(), repo_url, ref)
commit_position_range = end_commit_position - start_commit_position + 1
logs, _ = git_repo.GetNChangeLogs(end_revision, commit_position_range)
return dict((log.commit_position, log.revision) for log in logs)
def GetGitBlame(repo_url, revision, touched_file_path, ref=None):
"""Gets git blames of touched_file.
Args:
repo_url (str): Url to the repo.
revision (str): Revision for the change.
touched_file_path (str): Full path of a file in change_log.
"""
git_repo = CachedGitilesRepository(FinditHttpClient(), repo_url, ref)
return git_repo.GetBlame(touched_file_path, revision)
def _GetGitBlame(repo_info, touched_file_path):
"""Gets git blames of touched_file.
Args:
repo_info (dict): The repo_url and revision for the build cycle.
touched_file_path (str): Full path of a file in change_log.
"""
if repo_info:
repo_url = repo_info['repo_url']
git_repo = CachedGitilesRepository(HttpClientAppengine(), repo_url)
revision = repo_info['revision']
return git_repo.GetBlame(touched_file_path, revision)
def RunImpl(self, parameters):
analysis = ndb.Key(urlsafe=parameters.analysis_urlsafe_key).get()
assert analysis, 'Analysis unexpectedly missing!'
git_repo = CachedGitilesRepository(
FinditHttpClient(), constants.CHROMIUM_GIT_REPOSITORY_URL)
change_log = git_repo.GetChangeLog(parameters.flakiness.revision)
flakiness = parameters.flakiness
data_point = data_point_util.ConvertFlakinessToDataPoint(flakiness)
data_point.commit_timestamp = change_log.committer.time
analysis.flakiness_verification_data_points.append(data_point)
analysis.put()
def GetCommitsInfo(revisions, repo_url=CHROMIUM_GIT_REPOSITORY_URL, ref=None):
"""Gets commit_positions and review urls for revisions."""
git_repo = CachedGitilesRepository(FinditHttpClient(), repo_url, ref)
cls = {}
for revision in revisions:
cls[revision] = {'revision': revision, 'repo_name': 'chromium'}
change_log = git_repo.GetChangeLog(revision)
if change_log:
cls[revision]['commit_position'] = (change_log.commit_position)
cls[revision]['url'] = (change_log.code_review_url
or change_log.commit_url)
cls[revision]['author'] = change_log.author.email
return cls
def _GetCommitsBetweenRevisions(start_revision, end_revision):
"""Gets the revisions between start_revision and end_revision.
Args:
start_revision (str): The revision for which to get changes after. This
revision is not included in the returned list.
end_revision (str): The last revision in the range to return.
Returns:
A list of revisions sorted in order by oldest to newest.
"""
repo = CachedGitilesRepository(HttpClientAppengine(), _CHROMIUM_REPO_URL)
commits = repo.GetCommitsBetweenRevisions(start_revision, end_revision)
commits.reverse()
return commits
def run(self, client_id, crash_keys, publish_to_client=False):
"""Reruns analysis for a batch of crashes.
Args:
client_id (CrashClient): The client whose crash we should iterate.
crash_keys (list): A list of urlsafe encodings of crash keys.
"""
client = PredatorForClientID(
client_id, CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
updated = []
for key in crash_keys:
key = ndb.Key(urlsafe=key)
crash = key.get()
crash.ReInitialize(client)
updated.append(crash)
ndb.put_multi(updated)
for crash in updated:
logging.info('Initialize analysis for crash %s', crash.identifiers)
if publish_to_client:
run_analysis = yield CrashAnalysisPipeline(
client_id, crash.identifiers)
with pipeline.After(run_analysis):
yield PublishResultPipeline(client_id, crash.identifiers)
else:
yield CrashAnalysisPipeline(client_id, crash.identifiers)
def run(self, raw_crash_data):
"""Fire off pipelines to run the analysis and publish its results.
N.B., due to the structure of AppEngine pipelines, this method must
accept the same arguments as are passed to ``__init__``; however,
because they were already passed to ``__init__`` there's no use in
recieving them here. Thus, we discard all the arguments to this method
(except for ``self``, naturally).
"""
predator_client = PredatorForClientID(
self._client_id,
CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
crash_data = predator_client.GetCrashData(raw_crash_data)
need_analysis = predator_client.NeedsNewAnalysis(crash_data)
if need_analysis:
logging.info('New %s analysis is scheduled for %s',
self._client_id, crash_data.identifiers)
UpdateCrashAnalysisData(crash_data, predator_client)
run_analysis = yield CrashAnalysisPipeline(self._client_id,
crash_data.identifiers)
with pipeline.After(run_analysis):
yield PublishResultPipeline(self._client_id,
crash_data.identifiers)
else:
yield PublishResultPipeline(self._client_id,
crash_data.identifiers)
def __init__(self, client_id, crash_identifiers):
super(CrashBasePipeline, self).__init__(client_id, crash_identifiers)
self._crash_identifiers = crash_identifiers
self._predator = PredatorForClientID(
client_id, CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
self._predator.SetLog(self.log)
def __init__(self, client_id, crash_identifiers):
super(CrashBasePipeline, self).__init__(client_id, crash_identifiers)
self._crash_identifiers = crash_identifiers
self._findit = FinditForClientID(
client_id,
CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
def GetCulpritInfo(repo_name, revision):
"""Returns culprit info of the given revision.
Returns commit position, code-review url, host and change_id.
"""
# TODO(stgao): get repo url at runtime based on the given repo name.
# unused arg - pylint: disable=W0612,W0613
repo = CachedGitilesRepository(
HttpClientAppengine(),
'https://chromium.googlesource.com/chromium/src.git')
change_log = repo.GetChangeLog(revision)
return {
'commit_position': change_log.commit_position,
'code_review_url': change_log.code_review_url,
'review_server_host': change_log.review_server_host,
'review_change_id': change_log.review_change_id
}
def ChangeCommittedWithinTime(revision,
repo_url=CHROMIUM_GIT_REPOSITORY_URL,
hours=24,
ref=None):
"""Returns True if the change was committed within the time given."""
delta = timedelta(hours=hours)
git_repo = CachedGitilesRepository(FinditHttpClient(), repo_url, ref)
change_log = git_repo.GetChangeLog(revision)
culprit_commit_time = change_log.committer.time
in_time = time_util.GetUTCNow() - culprit_commit_time < delta
if not in_time:
logging.info(
'Culprit %s was committed over %d hours ago, stop auto '
'commit.', revision, hours)
return in_time
def _RetrieveManifest(repo_url, revision, os_platform): # pragma: no cover.
"""Returns the manifest of all the dependencies for the given revision.
Args:
repo_url (str): The url to the Gitiles project of the root repository.
revision (str): The revision of the root repository.
os_platform (str): The platform of the code checkout.
Returns:
A list of DependencyRepository instances ordered reversely by the relative
path of each dependency checkout in the checkout of the root repository.
The longer the relative path, the smaller index in the returned list.
The reverse order is to make it easy to reliably determine which dependency
a file is from, when given a file path relative to the root repository.
"""
manifest = []
root_dir = 'src/'
def AddDependencyToManifest(path, url, revision): # pragma: no cover.
if path.startswith(root_dir):
path = path[len(root_dir):]
assert not path.startswith('//')
path = '//' + path
if not path.endswith('/'):
path = path + '/'
# Parse the url to extract the hostname and project name.
# For "https://chromium.google.com/chromium/src.git", we get
# ParseResult(netloc='chromium.google.com', path='/chromium/src.git', ...)
result = urlparse.urlparse(url)
assert result.path, 'No project extracted from %s' % url
manifest.append(
DependencyRepository(
path=path,
server_host=result.netloc,
project=result.path[1:], # Strip the leading '/'.
revision=revision))
# Add the root repository.
AddDependencyToManifest('src/', repo_url, revision)
# Add all the dependent repositories.
# DEPS fetcher now assumes chromium/src and master branch.
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(FinditHttpClient()))
deps = dep_fetcher.GetDependency(revision, os_platform)
for path, dep in deps.iteritems():
# Remove clause when crbug.com/929315 gets fixed.
if path in _BLACKLISTED_DEPS.get(repo_url, []):
continue
AddDependencyToManifest(path, dep.repo_url, dep.revision)
manifest.sort(key=lambda x: len(x.path), reverse=True)
return manifest
def _RecordCacheStats(build, report):
"""Save the bot's state at the end of a successful.
This function aims to save the following data in the data store:
- The last revision that the bot synced to under the specific work
directory (named cache) it used for its local checkout.
- The latest revision fetched into the bot's local git cache, which is shared
across all work directories.
These are saved as commit positions rather than revision hashes for faster
comparisons when selecting a bot for new tryjobs.
"""
bot = swarmbot_util.GetBot(build)
cache_name = swarmbot_util.GetBuilderCacheName(build)
if bot and cache_name:
git_repo = CachedGitilesRepository(
FinditHttpClient(), services_constants.CHROMIUM_GIT_REPOSITORY_URL)
last_checked_out_revision = report.get('last_checked_out_revision')
last_checked_out_cp = (
git_repo.GetChangeLog(last_checked_out_revision).commit_position
if last_checked_out_revision else None)
cached_revision = report.get('previously_cached_revision')
cached_cp = git_repo.GetChangeLog(
cached_revision).commit_position if cached_revision else None
bad_revision = json.loads(build.response.get(
'parameters_json', '{}')).get('properties', {}).get('bad_revision')
bad_cp = git_repo.GetChangeLog(
bad_revision).commit_position if bad_revision else None
# If the bad_revision is later than the previously cached revision, that
# means that the bot had to sync with the remote repository, and the local
# git cache was updated to that revision at least.
latest_synced_cp = max(bad_cp, cached_cp)
cache_stats = WfTryBotCache.Get(cache_name)
cache_stats.AddBot(bot, last_checked_out_cp, latest_synced_cp)
# TODO(robertocn): Record the time it took to complete the task
# with a cold or warm cache.
cache_stats.put()
def GetCodeReviewInfoForACommit(revision,
repo_url=CHROMIUM_GIT_REPOSITORY_URL,
ref=None):
"""Returns change info of the given revision.
Returns commit position, code-review url, host and change_id.
"""
repo = CachedGitilesRepository(FinditHttpClient(), repo_url, ref)
change_log = repo.GetChangeLog(revision)
if not change_log:
return {}
return {
'commit_position': change_log.commit_position,
'code_review_url': change_log.code_review_url,
'review_server_host': change_log.review_server_host,
'review_change_id': change_log.review_change_id,
'author': change_log.author.ToDict(),
'committer': change_log.committer.ToDict(),
}
def _GetChromiumWATCHLISTS():
repo_url = 'https://chromium.googlesource.com/chromium/src'
source = CachedGitilesRepository(FinditHttpClient(), repo_url).GetSource(
'WATCHLISTS', 'master')
if not source:
return None
# https://cs.chromium.org/chromium/src/WATCHLISTS is in python.
definitions = ast.literal_eval(source).get('WATCHLIST_DEFINITIONS')
return dict((k, v['filepath']) for k, v in definitions.iteritems())
def _GetDependencies(chromium_revision, os_platform):
"""Returns the dependencies used by the specified chromium revision."""
deps = {}
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(HttpClientAppengine()))
for path, dependency in dep_fetcher.GetDependency(chromium_revision,
os_platform).iteritems():
deps[path] = {
'repo_url': dependency.repo_url,
'revision': dependency.revision,
}
return deps
def RunImpl(self, parameters):
"""Appends a DataPoint to a MasterFlakeAnalysis."""
analysis_urlsafe_key = parameters.analysis_urlsafe_key
analysis = ndb.Key(urlsafe=analysis_urlsafe_key).get()
assert analysis, 'Analysis unexpectedly missing'
flakiness = parameters.flakiness
if flakiness_util.MaximumSwarmingTaskRetriesReached(flakiness):
run_swarming_util.ReportSwarmingTaskError(analysis, flakiness.error)
analysis.LogError(
'Swarming task ended in error after {} failed attempts. Giving '
'up'.format(flakiness.failed_swarming_task_attempts))
raise pipeline.Abort()
git_repo = CachedGitilesRepository(FinditHttpClient(),
constants.CHROMIUM_GIT_REPOSITORY_URL)
change_log = git_repo.GetChangeLog(flakiness.revision)
data_point = data_point_util.ConvertFlakinessToDataPoint(flakiness)
data_point.commit_timestamp = change_log.committer.time
analysis.data_points.append(data_point)
analysis.put()
def _GetFileContentFromGitiles(report, file_path,
revision): # pragma: no cover.
"""Fetches the content of a specific revision of a file from gitiles.
Args:
report (PostsubmitReport): The report that the file is associated with.
file_path (str): Source absolute path to the file.
revision (str): The gitile revision of the file.
Returns:
The content of the source file."""
assert file_path.startswith('//'), 'All file path should start with "//".'
assert revision, 'A valid revision is required'
dependency = _GetMatchedDependencyRepository(report, file_path)
assert dependency, (
'%s file does not belong to any dependency repository' % file_path)
# Calculate the relative path to the root of the dependency repository itself.
relative_file_path = file_path[len(dependency.path):]
repo = CachedGitilesRepository(FinditHttpClient(), dependency.project_url)
return repo.GetSource(relative_file_path, revision)
def HandleGet(self):
"""Update the repo_to_dep_path in config from the lastest DEPS."""
# Update repo_to_dep_path to the latest information.
dep_fetcher = ChromeDependencyFetcher(
CachedGitilesRepository.Factory(HttpClientAppengine()))
repo_to_dep_path = GetRepoToDepPath(dep_fetcher)
if not repo_to_dep_path: # pragma: no cover.
return self.CreateError('Fail to update repo_to_dep_path config.', 400)
crash_config = CrashConfig.Get()
crash_config.Update(users.User(app_identity.get_service_account_name()),
True, repo_to_dep_path=repo_to_dep_path)
def GetCommitsBetweenRevisionsInOrder(start_revision,
end_revision,
repo_url=CHROMIUM_GIT_REPOSITORY_URL,
ascending=True,
ref=None):
"""Gets the revisions between start_revision and end_revision.
Args:
start_revision (str): The revision for which to get changes after. This
revision is not included in the returned list.
end_revision (str): The last revision in the range to return.
repo_url (str): Url of the git repo. Default to chromium repo url.
ascending (bool): Whether the commits should be in chronological order.
Returns:
A list of revisions sorted in order chronologically.
"""
repo = CachedGitilesRepository(FinditHttpClient(), repo_url, ref)
commits = repo.GetCommitsBetweenRevisions(start_revision, end_revision)
if ascending:
commits.reverse()
return commits
def _GetChangedLinesForDependencyRepo(roll, file_path_in_log, line_numbers):
"""Gets changed line numbers for file in failure log.
Tests if the same lines mentioned in failure log are changed within
the DEPS roll, if so, return those line numbers.
"""
roll_repo = CachedGitilesRepository(HttpClientAppengine(),
roll['repo_url'])
old_revision = roll['old_revision']
new_revision = roll['new_revision']
old_change_log = roll_repo.GetChangeLog(old_revision)
old_rev_author_time = old_change_log.author.time
new_change_log = roll_repo.GetChangeLog(new_revision)
new_rev_author_time = new_change_log.author.time
file_change_type = None
changed_line_numbers = []
if old_rev_author_time >= new_rev_author_time:
# If the DEPS roll is downgrade, bail out.
return file_change_type, changed_line_numbers
commits_in_roll = roll_repo.GetCommitsBetweenRevisions(
old_revision, new_revision)
file_change_type, culprit_commit = _GetChangeTypeAndCulpritCommit(
file_path_in_log, roll_repo, commits_in_roll)
if culprit_commit is None:
# Bail out if no commits touched the file in the log.
return file_change_type, changed_line_numbers
if file_change_type == ChangeType.MODIFY:
# If the file was modified, use the blame information to determine which
# lines were changed.
blame = roll_repo.GetBlame(file_path_in_log, culprit_commit)
if not blame:
return file_change_type, changed_line_numbers
for region in blame:
if line_numbers:
for line_number in line_numbers:
if (line_number >= region.start
and line_number <= region.start + region.count - 1
and region.revision in commits_in_roll):
# One line which appears in the failure log is changed within
# the DEPS roll.
changed_line_numbers.append(line_number)
return file_change_type, changed_line_numbers
def ExtractDepsInfo(failure_info, change_logs):
"""
Args:
failure_info (BaseFailureInfo): Information about all build failures.
change_logs (dict): Result of PullChangeLogs().
Returns:
A dict with the following form:
{
'deps': {
'path/to/dependency': {
'revision': 'git_hash',
'repo_url': 'https://url/to/dependency/repo.git',
},
...
},
'deps_rolls': {
'git_revision': [
{
'path': 'src/path/to/dependency',
'repo_url': 'https://url/to/dependency/repo.git',
'new_revision': 'git_hash1',
'old_revision': 'git_hash2',
},
...
],
...
}
}
"""
chromium_revision = failure_info.chromium_revision
os_platform = GetOSPlatformName(failure_info.master_name,
failure_info.builder_name)
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(FinditHttpClient()))
return {
'deps': GetDependencies(chromium_revision, os_platform, dep_fetcher),
'deps_rolls': DetectDependencyRolls(change_logs, os_platform,
dep_fetcher)
}
class PullChangelogPipeline(BasePipeline):
"""A pipeline to pull change log of CLs."""
# TODO: for files in dependencies(blink, v8, skia, etc), use blame first.
GIT_REPO = CachedGitilesRepository(
HttpClientAppengine(),
'https://chromium.googlesource.com/chromium/src.git')
# Arguments number differs from overridden method - pylint: disable=W0221
def run(self, failure_info):
"""
Args:
failure_info (dict): Output of pipeline DetectFirstFailurePipeline.run().
Returns:
A dict with the following form:
{
'git_hash_revision1': common.change_log.ChangeLog.ToDict(),
...
}
"""
change_logs = {}
if not failure_info['failed'] or not failure_info['chromium_revision']:
# Bail out if no failed step or no chromium revision.
return change_logs
# Bail out on infra failure
if failure_info.get('failure_type') == failure_type.INFRA:
return change_logs
for build in failure_info.get('builds', {}).values():
for revision in build['blame_list']:
change_log = self.GIT_REPO.GetChangeLog(revision)
if not change_log: # pragma: no cover
raise pipeline.Retry('Failed to get change log for %s' %
revision)
change_logs[revision] = change_log.ToDict()
return change_logs
def NeedNewAnalysis(json_crash_data):
"""Checks if an analysis is needed for this crash.
Args:
json_crash_data (dict): Crash information from clients.
Returns:
True if a new analysis is needed; False otherwise.
"""
if json_crash_data.get('redo'):
logging.info('Force redo crash %s',
repr(json_crash_data['crash_identifiers']))
return True
# N.B., must call FinditForClientID indirectly, for mock testing.
findit_client = crash_pipeline.FinditForClientID(
json_crash_data['client_id'],
CachedGitilesRepository.Factory(HttpClientAppengine()), CrashConfig.Get())
crash_data = findit_client.GetCrashData(json_crash_data)
# Detect the regression range, and decide if we actually need to
# run a new analysis or not.
return findit_client.NeedsNewAnalysis(crash_data)
def _DetectDependencyRolls(change_logs, os_platform):
"""Detect DEPS rolls in the given CL change logs.
Args:
change_logs (dict): Output of pipeline PullChangelogPipeline.run().
Returns:
A dict in the following form:
{
'git_revision': [
{
'path': 'src/path/to/dependency/',
'repo_url': 'https://url/to/dependency/repo.git',
'new_revision': 'git_hash1',
'old_revision': 'git_hash2',
},
...
],
...
}
"""
deps_rolls = {}
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(HttpClientAppengine()))
for revision, change_log in change_logs.iteritems():
# Check DEPS roll only if the chromium DEPS file is changed by the CL.
for touched_file in change_log['touched_files']:
if touched_file['new_path'] == 'DEPS':
# In git, r^ refers to the previous revision of r.
old_revision = '%s^' % revision
rolls = dep_fetcher.GetDependencyRolls(old_revision, revision,
os_platform)
deps_rolls[revision] = [roll.ToDict() for roll in rolls]
break
return deps_rolls
import logging
import mock
from testing_utils import testing
from common.findit_http_client import FinditHttpClient
from gae_libs.gitiles.cached_gitiles_repository import CachedGitilesRepository
from libs.deps import chrome_dependency_fetcher
from libs.deps.dependency import Dependency
from libs.gitiles.diff import ChangeType
from services import deps
from services.parameters import TestFailureInfo
from services.test.build_failure_analysis_test import ChangeLogFromDict
_DEP_FETCHER = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(FinditHttpClient()))
class DepsTest(testing.AppengineTestCase):
def testGetOSPlatformName(self):
master_name = 'chromium.linux'
builder_name = 'android'
self.assertEqual('android',
deps.GetOSPlatformName(master_name, builder_name))
def testGetOSPlatformNameDefault(self):
master_name = 'chromium.linux'
builder_name = 'linux'
self.assertEqual('unix',
deps.GetOSPlatformName(master_name, builder_name))
def AssignWarmCacheHost(tryjob, cache_name, http_client):
"""Selects the best possible slave for a given tryjob.
We try to get as many of the following conditions as possible:
- The bot is available,
- The bot has the named cached requested by the tryjob,
- The revision to test has already been fetched to the bot's local git cache,
- The currently checked out revision at the named cache is the closest
to the revision to test, and if possible it's earlier to it (so that
bot_update only moves forward, preferably)
If a match is found, it is added to the tryjob parameter as a dimension.
Args:
tryjob (buildbucket_client.TryJob): The ready-to-be-scheduled job.
cache_name (str): Previously computed name of the cache to match the
referred build's builder and master.
http_client: http_client to use for swarming and gitiles requests.
"""
if not tryjob.is_swarmbucket_build:
return
request_dimensions = dict([x.split(':', 1) for x in tryjob.dimensions])
bots_with_cache = OnlyAvailable(
GetAllBotsWithCache(request_dimensions, cache_name, http_client))
if bots_with_cache:
# Flake tryjobs check out older code, so there's little benefit in trying to
# optimize the way we do for non-flake tryjobs, we do however select the bot
# with the fewest named caches in an effort to avoid unnecessary cache
# evictions.
if cache_name and cache_name.endswith(
flake_constants.FLAKE_CACHE_SUFFIX):
selected_bot = _GetBotWithFewestNamedCaches(
bots_with_cache)['bot_id']
tryjob.dimensions.append('id:%s' % selected_bot)
return
git_repo = CachedGitilesRepository(
http_client, constants.CHROMIUM_GIT_REPOSITORY_URL)
# TODO(crbug.com/800107): Pass revision as a parameter.
revision = (tryjob.properties.get('bad_revision')
or tryjob.properties.get('test_revision'))
if not revision:
logging.error('Tryjob %s does not have a specified revision.' %
tryjob)
return
target_commit_position = git_repo.GetChangeLog(
revision).commit_position
bots_with_rev = _HaveCommitPositionInLocalGitCache(
bots_with_cache, target_commit_position)
if not bots_with_rev:
selected_bot = _GetBotWithFewestNamedCaches(
bots_with_cache)['bot_id']
tryjob.dimensions.append('id:' + selected_bot)
return
bots_with_latest_earlier_rev_checked_out = _ClosestEarlier(
bots_with_rev, cache_name, target_commit_position)
if bots_with_latest_earlier_rev_checked_out:
tryjob.dimensions.append(
'id:' + bots_with_latest_earlier_rev_checked_out['bot_id'])
return
bots_with_earliest_later_rev_checked_out = _ClosestLater(
bots_with_rev, cache_name, target_commit_position)
if bots_with_earliest_later_rev_checked_out:
tryjob.dimensions.append(
'id:' + bots_with_earliest_later_rev_checked_out['bot_id'])
return
selected_bot = _GetBotWithFewestNamedCaches(bots_with_rev)['bot_id']
tryjob.dimensions.append('id:' + selected_bot)
return
else:
idle_bots = OnlyAvailable(
GetBotsByDimension(request_dimensions, http_client))
if idle_bots:
selected_bot = _GetBotWithFewestNamedCaches(idle_bots)['bot_id']
tryjob.dimensions.append('id:' + selected_bot)
