def AddResult(cls,
test_set,
ip,
user_agent_string,
results_str,
is_import=False,
params_str=None,
js_user_agent_string=None,
js_user_agent_family=None,
js_user_agent_v1=None,
js_user_agent_v2=None,
js_user_agent_v3=None,
skip_dirty_update=False,
**kwds):
"""Create result models and stores them as one transaction.
Args:
test_set: an instance of test_set_base.
ip: a string to store as the user's IP. This should be hashed beforehand.
user_agent_string: the http user agent string.
results_str: a string like 'test1=time1,test2=time2,[...]'.
is_import: if True, skip checking test_keys and do not mark dirty.
params_str: a string representation of test_set_params.Params.
js_user_agent_string: chrome frame ua string from client-side JavaScript.
js_user_agent_family: This is an override for the family name to deal
with the fact that IE platform preview (for instance) cannot be
distinguished by user_agent_string, but only in javascript.
js_user_agent_v1: v1 override - see above.
js_user_agent_v2: v1 override - see above.
js_user_agent_v3: v1 override - see above.
skip_dirty_update: For tests, allow skipping the update-dirty task.
kwds: optional fields including 'loader_id'.
Returns:
A ResultParent instance.
"""
logging.debug('ResultParent.AddResult category=%s' % test_set.category)
if params_str in ('None', ''):
# params_str should either unset, None, or a non-empty string
logging.debug('params_str is wonky')
raise ValueError
user_agent = UserAgent.factory(
user_agent_string,
js_user_agent_string=js_user_agent_string,
js_user_agent_family=js_user_agent_family,
js_user_agent_v1=js_user_agent_v1,
js_user_agent_v2=js_user_agent_v2,
js_user_agent_v3=js_user_agent_v3)
parent = cls(category=test_set.category,
ip=ip,
user_agent=user_agent,
user_agent_string_list=user_agent.get_string_list(),
params_str=params_str,
**kwds)
try:
results = test_set.GetResults(results_str,
ignore_key_errors=is_import)
except test_set_base.ParseResultsKeyError, e:
logging.warn(e)
return None
def UpdateUserAgentStringListInResultParentForBrowse(request):
"""Browse results uses this list and it can be out of sync."""
ua_string = request.REQUEST.get('ua')
category = request.REQUEST.get('category')
family, v1, v2, v3 = UserAgent.parse_pretty(ua_string)
logging.info('family %s, v1 %s, v2 %s, v3 %s' % (family, v1, v2, v3))
user_agent = db.Query(UserAgent)
user_agent.filter('family =', family)
user_agent.filter('v1 =', v1)
user_agent.filter('v2 =', v2)
user_agent.filter('v3 =', v3)
logging.info('UA COUNT: %s' % user_agent.count())
user_agents = user_agent.fetch(1000)
for user_agent in user_agents:
logging.info('-----------------------')
string_list = user_agent.get_string_list()
logging.info('UA: %s, %s', (user_agent.key(), string_list))
parents = db.Query(ResultParent)
parents.filter('user_agent =', user_agent.key())
if category:
parents.filter('category =', category)
logging.info('PARENTS COUNT: %s' % parents.count())
for parent in parents.fetch(1000):
logging.info('PARENT: %s' % parent.key())
parent.user_agent_string_list = string_list
parent.put()
return http.HttpResponse('All done MF')
def UpdateCategory(request):
category = request.REQUEST.get('category')
user_agent_key = request.REQUEST.get('user_agent_key')
if not category:
logging.info('cron.UserAgentGroup: No category')
return http.HttpResponse('No category')
test_set = all_test_sets.GetTestSet(category)
if not test_set:
logging.info('cron.UserAgentGroup: Bad category: %s', category)
return http.HttpResponse('Bad category: %s' % category)
if not user_agent_key:
logging.info('cron.UserAgentGroup: No key')
return http.HttpResponse('No key')
try:
user_agent = UserAgent.get(db.Key(user_agent_key))
except db.BadKeyError:
logging.info('cron.UserAgentGroup: Invalid UserAgent key: %s', user_agent_key)
return http.HttpResponse('Invalid UserAgent key: %s' % user_agent_key)
if user_agent:
logging.info('cron.UserAgentGroup: UpdateCategory(%s, %s)', category, user_agent_key)
try:
result_stats.UpdateCategory(category, user_agent)
return http.HttpResponse('Done with UserAgent key=%s' % user_agent_key)
except:
return http.HttpResponse('Got a BadValueError. eff this one.')
else:
return http.HttpResponse('No user_agent with this key.')
def setUp(self):
self.client = Client()
ua_string = (
"Mozilla/5.0 (X11 U Linux armv6l de-DE rv:1.9a6pre) "
"Gecko/20080606 "
"Firefox/3.0a1 Tablet browser 0.3.7 "
"RX-34+RX-44+RX-48_DIABLO_4.2008.23-14"
)
self.ua = UserAgent.factory(ua_string)
def BrowserKey(cls, browser):
VERSION_DIGITS = 8
MAX_VERSION = 99999999
family, v1, v2, v3 = UserAgent.parse_pretty(browser.lower())
extra = None
if family[-1] == ')':
family, extra = family.split(' ', 1)
return (family.lower(),
cls._BrowserKeyPart(v1),
cls._BrowserKeyPart(v2),
cls._BrowserKeyPart(v3),
extra)
def AddResult(cls, test_set, ip, user_agent_string, results_str,
is_import=False, params_str=None,
js_user_agent_string=None,
js_user_agent_family=None,
js_user_agent_v1=None,
js_user_agent_v2=None,
js_user_agent_v3=None,
skip_dirty_update=False,
**kwds):
"""Create result models and stores them as one transaction.
Args:
test_set: an instance of test_set_base.
ip: a string to store as the user's IP. This should be hashed beforehand.
user_agent_string: the http user agent string.
results_str: a string like 'test1=time1,test2=time2,[...]'.
is_import: if True, skip checking test_keys and do not mark dirty.
params_str: a string representation of test_set_params.Params.
js_user_agent_string: chrome frame ua string from client-side JavaScript.
js_user_agent_family: This is an override for the family name to deal
with the fact that IE platform preview (for instance) cannot be
distinguished by user_agent_string, but only in javascript.
js_user_agent_v1: v1 override - see above.
js_user_agent_v2: v1 override - see above.
js_user_agent_v3: v1 override - see above.
skip_dirty_update: For tests, allow skipping the update-dirty task.
kwds: optional fields including 'loader_id'.
Returns:
A ResultParent instance.
"""
logging.debug('ResultParent.AddResult category=%s' % test_set.category)
if params_str in ('None', ''):
# params_str should either unset, None, or a non-empty string
logging.debug('params_str is wonky')
raise ValueError
user_agent = UserAgent.factory(user_agent_string,
js_user_agent_string=js_user_agent_string,
js_user_agent_family=js_user_agent_family,
js_user_agent_v1=js_user_agent_v1,
js_user_agent_v2=js_user_agent_v2,
js_user_agent_v3=js_user_agent_v3)
parent = cls(category=test_set.category,
ip=ip,
user_agent=user_agent,
user_agent_string_list=user_agent.get_string_list(),
params_str=params_str, **kwds)
try:
results = test_set.GetResults(results_str, ignore_key_errors=is_import)
except test_set_base.ParseResultsKeyError, e:
logging.warn(e)
return None
def ConfirmUa(request):
"""Confirm User-Agents"""
search_browser = request.REQUEST.get('browser', '')
confirmed = request.REQUEST.get('confirmed')
search_confirmed = confirmed == 'on'
search_changed = request.REQUEST.get('changed', False)
cursor = request.REQUEST.get('cursor', '')
limit = int(request.REQUEST.get('limit', 30))
if 'search' in request.REQUEST:
pass
elif 'submit' in request.REQUEST:
return SubmitChanges(request)
# Build up the query.
query = db.Query(UserAgent)
if search_browser != '':
string_list = UserAgent.parse_to_string_list(search_browser)
logging.info('string_list: %s' % string_list)
ua_bits = ['family', 'v1', 'v2', 'v3']
for index, item in enumerate(string_list):
logging.info('adding %s=%s' % (ua_bits[index], item))
query.filter('%s =' % ua_bits[index], item)
query.filter('confirmed =', search_confirmed)
query.order('-created')
if cursor:
query.with_cursor(cursor)
user_agents = query.fetch(limit=limit)
params = {
'page_title': 'Confirm User-Agents',
'user_agents': user_agents,
'cursor': cursor,
'next_cursor': query.cursor(),
'search_browser': search_browser,
'search_confirmed': search_confirmed,
'search_changed': search_changed,
'csrf_token': request.session['csrf_token'],
'use_parse_service': True,
}
return Render(request, 'admin/confirm-ua.html', params)
def GetUserAgent(browser): return UserAgent.factory(GetUserAgentString(browser))
def GetUserAgent(browser):
return UserAgent.factory(GetUserAgentString(browser))
def DataDump(request):
"""This is used by bin/data_dump.py to replicate the datastore."""
model = request.REQUEST.get('model')
key_prefix = request.REQUEST.get('key_prefix', '')
keys_list = request.REQUEST.get('keys')
time_limit = int(request.REQUEST.get('time_limit', 3))
if keys_list:
keys = ['%s%s' % (key_prefix, key) for key in keys_list.split(',')]
else:
return http.HttpResponseBadRequest('"keys" is a required parameter.')
start_time = datetime.datetime.now()
if model == 'ResultParent':
query = pager.PagerQuery(ResultParent, keys_only=True)
elif model == 'UserAgent':
query = pager.PagerQuery(UserAgent)
else:
return http.HttpResponseBadRequest(
'model must be one of "ResultParent", "UserAgent".')
data = []
error = None
if model == 'ResultParent':
result_time_query = ResultTime.gql('WHERE ANCESTOR IS :1')
for result_parent_key in keys:
if (datetime.datetime.now() - start_time).seconds > time_limit:
error = 'Over time limit'
break
try:
p = ResultParent.get(result_parent_key)
except db.Timeout:
error = 'db.Timeout: ResultParent'
break
if not p:
data.append({
'model_class': 'ResultParent',
'lost_key': result_parent_key,
})
continue
result_time_query.bind(p.key())
try:
result_times = result_time_query.fetch(1000)
except db.Timeout:
error = 'db.Timeout: ResultTime'
break
row_data = [{
'model_class': 'ResultParent',
'result_parent_key': result_parent_key,
'category': p.category,
'user_agent_key': str(
ResultParent.user_agent.get_value_for_datastore(p)),
'ip': p.ip,
'user_id': p.user and p.user.user_id() or None,
'created': p.created and p.created.isoformat() or None,
'params_str': p.params_str,
'loader_id': hasattr(p, 'loader_id') and p.loader_id or None,
}]
is_dirty = False
for result_time in result_times:
if result_time.dirty:
is_dirty = True
break
row_data.append({
'model_class': 'ResultTime',
'result_time_key': str(result_time.key()),
'result_parent_key': str(result_parent_key),
'test': result_time.test,
'score': result_time.score,
})
if is_dirty:
data.append({'dirty_key': result_parent_key,})
else:
data.extend(row_data)
elif model == 'UserAgent':
try:
user_agents = UserAgent.get(keys)
except db.Timeout:
error = 'db.Timeout: UserAgent'
else:
for key, ua in zip(keys, user_agents):
if ua:
data.append({
'model_class': 'UserAgent',
'user_agent_key': key,
'string': ua.string,
'family': ua.family,
'v1': ua.v1,
'v2': ua.v2,
'v3': ua.v3,
'confirmed': ua.confirmed,
'created': ua.created and ua.created.isoformat() or None,
'js_user_agent_string': (hasattr(ua, 'js_user_agent_string') and
ua.js_user_agent_string or None),
})
else:
data.append({
'model_class': 'UserAgent',
'lost_key': key,
})
response_params = {
'data': data,
}
if error:
response_params['error'] = error
return http.HttpResponse(content=simplejson.dumps(response_params),
content_type='application/json')
