def AddUserAgent(cls, category, user_agent):
"""Adds a user agent's browser strings to version-level groups.
AddUserAgent assumes that it does not receive overlapping calls.
- It should only get called by the update-user-groups task queue.
Adds a browser for every version level.
If a level does not have a string, then use the one from the previous level.
For example, "Safari 4.3" would increment the following:
level browser
0 Safari
1 Safari 4
2 Safari 4.3
3 Safari 4.3
Args:
category: a category string like 'network' or 'reflow'.
user_agent: a UserAgent instance.
"""
key_names = [cls.KeyName(category, v) for v in range(4)]
version_levels = range(4)
if category in [t.category for t in all_test_sets.GetVisibleTestSets()]:
key_names.extend([cls.KeyName('summary', v) for v in range(4)])
version_levels.extend(range(4))
level_browsers = memcache.get_multi(key_names,
namespace=cls.MEMCACHE_NAMESPACE)
browser_key_names = []
ua_browsers = user_agent.get_string_list()
max_ua_browsers_index = len(ua_browsers) - 1
for version_level, key_name in zip(version_levels, key_names):
browser = ua_browsers[min(max_ua_browsers_index, version_level)]
if browser not in level_browsers.get(key_name, []):
browser_key_names.append((browser, key_name))
managers = cls.get_by_key_name([x[1] for x in browser_key_names])
updated_managers = []
memcache_mapping = {}
for (browser, key_name), manager in zip(browser_key_names, managers):
if manager is None:
manager = cls.get_or_insert(key_name)
if browser not in manager.browsers:
cls.InsortBrowser(manager.browsers, browser)
updated_managers.append(manager)
memcache_mapping[key_name] = manager.browsers
if updated_managers:
db.put(updated_managers)
memcache.set_multi(memcache_mapping, namespace=cls.MEMCACHE_NAMESPACE)
def GetStats(cls, browsers, categories=None):
"""Return the summary stats for a set of browsers and categories.
Gets stats out of summary memcache. If needed, re-aggregate them for the
categories. These data may come from memcache or all the way from the
datastore.
Args:
browsers: a list of browsers to use instead of version level.
categories: a list of categories like ['security', 'richtext'].
Returns:
{
browser_x: {
category_y: {
'score': score_xy,
'display': display_xy,
'total_runs': total_runs_xy,
}, ...
}, ...
}
"""
summary_stats = memcache.get_multi(
browsers, namespace=cls.MEMCACHE_NAMESPACE)
if not categories:
categories = [t.category for t in all_test_sets.GetVisibleTestSets()]
# Trim any unwanted stats and find any missing stats.
missing_stats = {}
for browser in browsers:
ua_summary_stats = summary_stats.get(browser, {'results': {}})
existing_categories = ua_summary_stats['results'].keys()
for category in existing_categories:
if category not in categories:
del ua_summary_stats['results'][category]
for category in categories:
if category not in existing_categories:
missing_stats.setdefault(category, []).append(browser)
# Load any missing stats
for category, browsers in missing_stats.items():
updated_stats = cls._FindAndUpdateStats(category, browsers)
summary_stats.update(updated_stats)
cls._AddSummaryOfSummaries(summary_stats)
return summary_stats
def UpdateAllStatsCache(request, batch_size=UPDATE_ALL_BATCH_SIZE,
is_uncached_update=False):
categories_str = request.REQUEST.get('categories')
if categories_str:
categories = categories_str.split(',')
else:
categories = [s.category for s in all_test_sets.GetVisibleTestSets()]
if not categories:
return http.HttpResponseServerError('No categories given.')
elif len(categories) > 1:
for category in categories:
attempt = 0
while attempt < 3:
try:
task = taskqueue.Task(url=request.path,
params={'categories': category})
task.add(queue_name='update-stats-cache')
break
except:
attempt += 1
return http.HttpResponse('Queued stats cache update for categories: %s' %
categories)
category = categories[0]
test_set = all_test_sets.GetTestSet(category)
browsers = result_stats.CategoryBrowserManager.GetAllBrowsers(category)
logging.info('Update all stats cache: %s', category)
for i in range(0, len(browsers), batch_size):
params={
'category': category,
'browsers': ','.join(browsers[i:i+batch_size]),
}
if is_uncached_update:
params['is_uncached_update'] = 1
attempt = 0
while attempt < 3:
try:
taskqueue.Task(params=params).add(queue_name='update-stats-cache')
break
except:
attempt += 1
logging.info('Added task for browsers %s to %s.', i, i+batch_size)
return http.HttpResponse('Done creating update tasks.')
def UpdateRecentTests(request):
max_recent_tests = 10
visible_categories = all_test_sets.GetVisibleTestSets()
#logging.info('visible_categories %s' % visible_categories)
prev_recent_tests = memcache.get(util.RECENT_TESTS_MEMCACHE_KEY)
prev_result_parent_key = None
if prev_recent_tests:
prev_result_parent_key = prev_recent_tests[0]['result_parent_key']
recent_tests = []
recent_query = db.Query(ResultParent).order('-created').filter('category IN',
[vis.category for vis in visible_categories])
for result_parent in recent_query.fetch(max_recent_tests):
if str(result_parent.key()) == prev_result_parent_key:
num_needed = max_recent_tests - len(recent_tests)
if num_needed == max_recent_tests:
return http.HttpResponse('No update needed.')
else:
recent_tests.extend(prev_recent_tests[:num_needed])
break
recent_scores = result_parent.GetResults()
test_set = all_test_sets.GetTestSet(result_parent.category)
visible_test_keys = [t.key for t in test_set.VisibleTests()]
recent_stats = test_set.GetStats(visible_test_keys, recent_scores)
recent_tests.append({
'result_parent_key': str(result_parent.key()),
'category': result_parent.category,
'created': result_parent.created,
'user_agent_pretty': result_parent.user_agent.pretty(),
'score': recent_stats['summary_score'],
'display': recent_stats['summary_display'],
})
#logging.info('Setting recent tests: %s' % recent_tests)
memcache.set(util.RECENT_TESTS_MEMCACHE_KEY, recent_tests,
time=settings.STATS_MEMCACHE_TIMEOUT)
#logging.info('Read recent tests: %s' %
# memcache.get(key=util.RECENT_TESTS_MEMCACHE_KEY))
return http.HttpResponse('Done')
class SummaryTest(test_set_base.TestBase):
def __init__(self, category, category_name, summary_doc):
test_set_base.TestBase.__init__(
self,
key=category,
name=category_name,
url=None,
doc=summary_doc,
min_value=0,
max_value=0)
_TESTS = []
for test_set in all_test_sets.GetVisibleTestSets():
_TESTS.append(SummaryTest(
test_set.category, test_set.category_name, test_set.summary_doc))
class SummaryTestSet(test_set_base.TestSet):
def GetTestScoreAndDisplayValue(self, test_key, raw_scores):
"""Get a normalized score (0 to 100) and a value to output to the display.
Args:
test_key: a key for a test_set test.
raw_scores: a dict of raw_scores indexed by test keys.
Returns:
score, display_value
# score is from 0 to 100.
# display_value is the text for the cell.
def UpdateSummaryBrowsers(request):
"""Update all the browsers for the summary category."""
result_stats.CategoryBrowserManager.UpdateSummaryBrowsers(
[ts.category for ts in all_test_sets.GetVisibleTestSets()])
return http.HttpResponse('Success.')