def _AddRepository(url):
"""Add a repository URL to the database with the default name mapping.
The default short repository name is the last part of the URL.
Returns:
The short repository name.
Raises:
AssertionError: The default name is already in the database.
"""
name = url.split('/')[-1]
# Add to main repositories dict.
repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY)
if name in repositories:
raise AssertionError(
"Attempted to add a repository that's already in the "
'Datastore: %s: %s' % (name, url))
repositories[name] = {'repository_url': url}
namespaced_stored_object.Set(_REPOSITORIES_KEY, repositories)
# Add to URL -> name mapping dict.
urls_to_names = namespaced_stored_object.Get(_URLS_TO_NAMES_KEY)
urls_to_names[url] = name
namespaced_stored_object.Set(_URLS_TO_NAMES_KEY, urls_to_names)
return name
def FetchCachedTestSuites():
"""Fetches cached test suite data."""
cached = namespaced_stored_object.Get(_LIST_SUITES_CACHE_KEY)
if cached is None:
# If the cache test suite list is not set, update it before fetching.
# This is for convenience when testing sending of data to a local instance.
UpdateTestSuites(datastore_hooks.GetNamespace())
cached = namespaced_stored_object.Get(_LIST_SUITES_CACHE_KEY)
return cached
def _MasterNameIsWhitelisted(master_name):
"""Checks whether a master name is acceptable by checking a whitelist."""
bisect_bot_map = namespaced_stored_object.Get(BISECT_BOT_MAP_KEY)
if not bisect_bot_map:
return True # If there's no list available, all names are OK.
whitelisted_masters = list(bisect_bot_map)
return master_name in whitelisted_masters
def FromDict(cls, data):
"""Create a Dep from a dict.
If the repository is a repository URL, it will be translated to its short
form name.
Raises:
KeyError: The repository name is not in the local datastore,
or the git hash is not valid.
"""
repository = data['repository']
# Translate repository if it's a URL.
repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY)
for repo_label, repo_info in repositories.iteritems():
if repository == repo_info['repository_url']:
repository = repo_label
break
dep = cls(repository, data['git_hash'])
try:
gitiles_service.CommitInfo(dep.repository_url, dep.git_hash)
except gitiles_service.NotFoundError as e:
raise KeyError(e)
return dep
def _DomainIsSupported(domain):
"""Checks whether a master name is acceptable by checking a list."""
bisect_bot_map = namespaced_stored_object.Get(BISECT_BOT_MAP_KEY)
if not bisect_bot_map:
return True # If there's no list available, all names are OK.
allow_listed_domain = list(bisect_bot_map)
return domain in allow_listed_domain
def post(self):
repo = self.request.get('repository')
git_hash_1 = self.request.get('git_hash',
self.request.get('git_hash_1'))
git_hash_2 = self.request.get('git_hash_2')
repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY)
if repo not in repositories:
self.response.out.write(
json.dumps({'error': 'Unknown repository: %s' % repo}))
return
repository_url = repositories[repo]['repository_url']
if not git_hash_1:
self.response.out.write(
json.dumps({'error': "No 'git_hash' parameter specified."}))
return
if git_hash_2:
result = gitiles_service.CommitRange(repo, git_hash_1, git_hash_2)
self.response.out.write(json.dumps(result))
return
result = gitiles_service.CommitInfo(repository_url, git_hash_1)
self.response.out.write(json.dumps(result))
def DomainIsExcludedFromTriageBisects(domain):
"""Checks whether a master name is disallowed for alert triage bisects."""
file_bug_bisect_denylist = namespaced_stored_object.Get(
FILE_BUG_BISECT_DENYLIST_KEY)
if not file_bug_bisect_denylist:
return False # If there's no denylist, all masters are allowed.
return domain in file_bug_bisect_denylist
def AddRowsToCache(row_entities):
"""Adds a list of rows to the cache, in revision order.
Updates multiple cache entries for different tests.
Args:
row_entities: List of Row entities.
"""
test_key_to_rows = {}
for row in row_entities:
test_key = row.parent_test
if test_key in test_key_to_rows:
graph_rows = test_key_to_rows[test_key]
else:
test_path = utils.TestPath(test_key)
graph_rows = namespaced_stored_object.Get(_CACHE_KEY % test_path)
if not graph_rows:
# We only want to update caches for tests that people have looked at.
continue
test_key_to_rows[test_key] = graph_rows
revisions = [r[0] for r in graph_rows]
index = bisect.bisect_left(revisions, row.revision)
if index < len(revisions) - 1:
if revisions[index + 1] == row.revision:
return # Already in cache.
graph_rows.insert(index, _MakeTriplet(row))
for test_key in test_key_to_rows:
graph_rows = test_key_to_rows[test_key]
SetCache(utils.TestPath(test_key), graph_rows)
def _BotDimensionsFromBotName(bot_name):
bots_to_dimensions = namespaced_stored_object.Get(_BOTS_TO_DIMENSIONS)
dimensions = bots_to_dimensions.get(bot_name)
if not dimensions:
raise InvalidParamsError('No dimensions for bot %s defined.' %
bot_name)
return dimensions
def _GetAvailableBisectBots(master_name):
"""Gets all available bisect bots corresponding to a master name."""
bisect_bot_map = namespaced_stored_object.Get(can_bisect.BISECT_BOT_MAP_KEY)
for master, platform_bot_pairs in bisect_bot_map.iteritems():
if master_name.startswith(master):
return sorted({bot for _, bot in platform_bot_pairs})
return []
def _GetChartValues(self):
return {
'revision_info': namespaced_stored_object.Get(_REVISION_INFO_KEY)
or {},
'warning_message': layered_cache.Get('warning_message'),
'warning_bug': layered_cache.Get('warning_bug'),
}
def _GetCommitInfoForAlert(alert):
repository_url = None
repositories = namespaced_stored_object.Get('repositories')
test_path = utils.TestPath(alert.test)
if test_path.startswith('ChromiumPerf'):
repository_url = repositories['chromium']['repository_url']
elif test_path.startswith('ClankInternal'):
repository_url = repositories['clank']['repository_url']
if not repository_url:
# Can't get committer info from this repository.
return None
rev = str(auto_bisect.GetRevisionForBisect(alert.end_revision, alert.test))
# TODO(sullivan, dtu): merge this with similar pinoint code.
if (re.match(r'^[0-9]{5,7}$', rev)
and repository_url == repositories['chromium']['repository_url']):
# This is a commit position, need the git hash.
result = crrev_service.GetNumbering(
number=rev,
numbering_identifier='refs/heads/master',
numbering_type='COMMIT_POSITION',
project='chromium',
repo='chromium/src')
rev = result['git_sha']
if not re.match(r'[a-fA-F0-9]{40}$', rev):
# This still isn't a git hash; can't assign bug.
return None
return gitiles_service.CommitInfo(repository_url, rev)
def MasterNameIsBlacklistedForTriageBisects(master_name):
"""Checks whether a master name is blacklisted for alert triage bisects."""
file_bug_bisect_blacklist = namespaced_stored_object.Get(
FILE_BUG_BISECT_BLACKLIST_KEY)
if not file_bug_bisect_blacklist:
return False # If there's no blacklist, all masters are OK.
return master_name in file_bug_bisect_blacklist
def GuessBisectBot(master_name, bot_name):
"""Returns a bisect bot name based on |bot_name| (perf_id) string."""
platform_bot_pairs = []
bisect_bot_map = namespaced_stored_object.Get(can_bisect.BISECT_BOT_MAP_KEY)
if bisect_bot_map:
for master, pairs in bisect_bot_map.iteritems():
if master_name.startswith(master):
platform_bot_pairs = pairs
break
fallback = 'linux_perf_bisect'
if not platform_bot_pairs:
# No bots available.
logging.error('No bisect bots defined for %s.', master_name)
return fallback
bot_name = bot_name.lower()
for platform, bisect_bot in platform_bot_pairs:
if platform.lower() in bot_name:
return bisect_bot
for _, bisect_bot in platform_bot_pairs:
if bisect_bot == fallback:
return fallback
# Nothing was found; log a warning and return a fall-back name.
logging.warning('No bisect bot for %s/%s.', master_name, bot_name)
return platform_bot_pairs[0][1]
def List():
bot_configurations = namespaced_stored_object.Get(BOT_CONFIGURATIONS_KEY)
canonical_names = [
name for name, value in bot_configurations.iteritems()
if 'alias' not in value
]
return sorted(canonical_names, key=string.lower)
def _Repository(repository_url):
repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY)
for repo_label, repo_info in repositories.iteritems():
if repository_url == repo_info['repository_url']:
return repo_label
return None
def get(self):
# TODO: Merge bot_browser_map_2, bot_dimensions_map, and the builder map
# into one mapping.
bots_to_dimensions = namespaced_stored_object.Get(_BOTS_TO_DIMENSIONS)
self.response.out.write(
json.dumps({
'configurations': sorted(bots_to_dimensions.iterkeys()),
}))
def Get(name):
configurations = namespaced_stored_object.Get(BOT_CONFIGURATIONS_KEY) or {}
configuration = configurations.get(name)
if configuration is None:
raise ValueError('Bot configuration not found: "%s"' % (name, ))
if 'alias' in configuration:
return configurations[configuration['alias']]
return configuration
def _GetTryServerBucket(bisect_job):
"""Returns the bucket name to be used by buildbucket."""
master_bucket_map = namespaced_stored_object.Get(_MASTER_BUILDBUCKET_MAP_KEY)
default = 'master.tryserver.chromium.perf'
if not master_bucket_map:
logging.warning(
'Could not get bucket to be used by buildbucket, using default.')
return default
return master_bucket_map.get(bisect_job.master_name, default)
def _AddRepository(url):
name = url.split('/')[-1]
# Add to main repositories dict.
repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY)
if name in repositories:
raise AssertionError(
"Attempted to add a repository that's already in the "
'Datastore: %s: %s' % (name, url))
repositories[name] = {'repository_url': url}
namespaced_stored_object.Set(_REPOSITORIES_KEY, repositories)
# Add to URL -> name mapping dict.
urls_to_names = namespaced_stored_object.Get(_URLS_TO_NAMES_KEY)
urls_to_names[url] = name
namespaced_stored_object.Set(_URLS_TO_NAMES_KEY, urls_to_names)
return name
def _Repository(repository_url):
if repository_url.endswith('.git'):
repository_url = repository_url[:-4]
repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY)
for repo_label, repo_info in repositories.iteritems():
if repository_url == repo_info['repository_url']:
return repo_label
raise KeyError('Unknown repository URL: ' + repository_url)
def GuessBrowserName(bisect_bot):
"""Returns a browser name string for Telemetry to use."""
default = 'release'
browser_map = namespaced_stored_object.Get(_BOT_BROWSER_MAP_KEY)
if not browser_map:
return default
for bot_name_prefix, browser_name in browser_map:
if bisect_bot.startswith(bot_name_prefix):
return browser_name
return default
def testPost_WithKey_UpdatesNamespacedValues(self):
namespaced_stored_object.Set('foo', 'XXXinternalYYY')
namespaced_stored_object.SetExternal('foo', 'XXXYYY')
self.testapp.post('/edit_site_config', {
'key': 'foo',
'external_value': '{"x": "y"}',
'internal_value': '{"x": "yz"}',
'xsrf_token': xsrf.GenerateToken(users.get_current_user()),
})
self.assertEqual({'x': 'yz'}, namespaced_stored_object.Get('foo'))
self.assertEqual({'x': 'y'}, namespaced_stored_object.GetExternal('foo'))
def Repository(url, add_if_missing=False):
if url.endswith('.git'):
url = url[:-4]
urls_to_names = namespaced_stored_object.Get(_URLS_TO_NAMES_KEY)
try:
return urls_to_names[url]
except KeyError:
if add_if_missing:
return _AddRepository(url)
raise
def _GetBuilder(arguments):
configuration = arguments.get('configuration')
builder = arguments.get('builder')
if builder:
pass
elif configuration:
bots = namespaced_stored_object.Get(_BOT_CONFIGURATIONS)
builder = bots[configuration]['builder']
else:
raise TypeError('Missing a "configuration" or a "builder" argument.')
return builder
def _GetBrowser(arguments, used_arguments):
configuration = arguments.get('configuration')
browser = arguments.get('browser')
if browser:
used_arguments['browser'] = browser
elif configuration:
used_arguments['configuration'] = configuration
bots = namespaced_stored_object.Get(_BOT_CONFIGURATIONS)
browser = bots[configuration]['browser']
else:
raise TypeError('Missing a "configuration" or a "browser" argument.')
return browser
def _AddRepository(repository_url):
repositories = namespaced_stored_object.Get(_REPOSITORIES_KEY)
repository = repository_url.split('/')[-1]
if repository.endswith('.git'):
repository = repository[:-4]
if repository in repositories:
raise AssertionError(
"Attempted to add a repository that's already in the "
'Datastore: %s: %s' % (repository, repository_url))
repositories[repository] = {'repository_url': repository_url}
namespaced_stored_object.Set(_REPOSITORIES_KEY, repositories)
def _GetDimensions(arguments):
configuration = arguments.get('configuration')
dimensions = arguments.get('dimensions')
if dimensions:
dimensions = json.loads(dimensions)
elif configuration:
bots = namespaced_stored_object.Get(_BOT_CONFIGURATIONS)
dimensions = bots[configuration]['dimensions']
else:
raise TypeError(
'Missing a "configuration" or a "dimensions" argument.')
return dimensions
def _BuilderType(master_name):
"""Returns the builder_type string to use in the bisect config.
Args:
master_name: The test master name.
Returns:
A string which indicates where the builds should be obtained from.
"""
builder_types = namespaced_stored_object.Get(_BUILDER_TYPES_KEY)
if not builder_types or master_name not in builder_types:
return 'perf'
return builder_types[master_name]
def _ArgumentsWithConfiguration(original_arguments):
# "configuration" is a special argument that maps to a list of preset
# arguments. Pull any arguments from the specified "configuration", if any.
configuration = original_arguments.get('configuration')
if configuration:
configurations = namespaced_stored_object.Get(_BOT_CONFIGURATIONS)
new_arguments = configurations[configuration]
else:
new_arguments = {}
# Override the configuration arguments with the API-provided arguments.
new_arguments.update(original_arguments)
return new_arguments
