def AddResult(cls, test_set, ip, user_agent_string, results_str, is_import=False, params_str=None, js_user_agent_string=None, js_user_agent_family=None, js_user_agent_v1=None, js_user_agent_v2=None, js_user_agent_v3=None, skip_dirty_update=False, **kwds): """Create result models and stores them as one transaction. Args: test_set: an instance of test_set_base. ip: a string to store as the user's IP. This should be hashed beforehand. user_agent_string: the http user agent string. results_str: a string like 'test1=time1,test2=time2,[...]'. is_import: if True, skip checking test_keys and do not mark dirty. params_str: a string representation of test_set_params.Params. js_user_agent_string: chrome frame ua string from client-side JavaScript. js_user_agent_family: This is an override for the family name to deal with the fact that IE platform preview (for instance) cannot be distinguished by user_agent_string, but only in javascript. js_user_agent_v1: v1 override - see above. js_user_agent_v2: v1 override - see above. js_user_agent_v3: v1 override - see above. skip_dirty_update: For tests, allow skipping the update-dirty task. kwds: optional fields including 'loader_id'. Returns: A ResultParent instance. """ logging.debug('ResultParent.AddResult category=%s' % test_set.category) if params_str in ('None', ''): # params_str should either unset, None, or a non-empty string logging.debug('params_str is wonky') raise ValueError user_agent = UserAgent.factory( user_agent_string, js_user_agent_string=js_user_agent_string, js_user_agent_family=js_user_agent_family, js_user_agent_v1=js_user_agent_v1, js_user_agent_v2=js_user_agent_v2, js_user_agent_v3=js_user_agent_v3) parent = cls(category=test_set.category, ip=ip, user_agent=user_agent, user_agent_string_list=user_agent.get_string_list(), params_str=params_str, **kwds) try: results = test_set.GetResults(results_str, ignore_key_errors=is_import) except test_set_base.ParseResultsKeyError, e: logging.warn(e) return None
def UpdateUserAgentStringListInResultParentForBrowse(request): """Browse results uses this list and it can be out of sync.""" ua_string = request.REQUEST.get('ua') category = request.REQUEST.get('category') family, v1, v2, v3 = UserAgent.parse_pretty(ua_string) logging.info('family %s, v1 %s, v2 %s, v3 %s' % (family, v1, v2, v3)) user_agent = db.Query(UserAgent) user_agent.filter('family =', family) user_agent.filter('v1 =', v1) user_agent.filter('v2 =', v2) user_agent.filter('v3 =', v3) logging.info('UA COUNT: %s' % user_agent.count()) user_agents = user_agent.fetch(1000) for user_agent in user_agents: logging.info('-----------------------') string_list = user_agent.get_string_list() logging.info('UA: %s, %s', (user_agent.key(), string_list)) parents = db.Query(ResultParent) parents.filter('user_agent =', user_agent.key()) if category: parents.filter('category =', category) logging.info('PARENTS COUNT: %s' % parents.count()) for parent in parents.fetch(1000): logging.info('PARENT: %s' % parent.key()) parent.user_agent_string_list = string_list parent.put() return http.HttpResponse('All done MF')
def UpdateCategory(request): category = request.REQUEST.get('category') user_agent_key = request.REQUEST.get('user_agent_key') if not category: logging.info('cron.UserAgentGroup: No category') return http.HttpResponse('No category') test_set = all_test_sets.GetTestSet(category) if not test_set: logging.info('cron.UserAgentGroup: Bad category: %s', category) return http.HttpResponse('Bad category: %s' % category) if not user_agent_key: logging.info('cron.UserAgentGroup: No key') return http.HttpResponse('No key') try: user_agent = UserAgent.get(db.Key(user_agent_key)) except db.BadKeyError: logging.info('cron.UserAgentGroup: Invalid UserAgent key: %s', user_agent_key) return http.HttpResponse('Invalid UserAgent key: %s' % user_agent_key) if user_agent: logging.info('cron.UserAgentGroup: UpdateCategory(%s, %s)', category, user_agent_key) try: result_stats.UpdateCategory(category, user_agent) return http.HttpResponse('Done with UserAgent key=%s' % user_agent_key) except: return http.HttpResponse('Got a BadValueError. eff this one.') else: return http.HttpResponse('No user_agent with this key.')
def setUp(self): self.client = Client() ua_string = ( "Mozilla/5.0 (X11 U Linux armv6l de-DE rv:1.9a6pre) " "Gecko/20080606 " "Firefox/3.0a1 Tablet browser 0.3.7 " "RX-34+RX-44+RX-48_DIABLO_4.2008.23-14" ) self.ua = UserAgent.factory(ua_string)
def BrowserKey(cls, browser): VERSION_DIGITS = 8 MAX_VERSION = 99999999 family, v1, v2, v3 = UserAgent.parse_pretty(browser.lower()) extra = None if family[-1] == ')': family, extra = family.split(' ', 1) return (family.lower(), cls._BrowserKeyPart(v1), cls._BrowserKeyPart(v2), cls._BrowserKeyPart(v3), extra)
def AddResult(cls, test_set, ip, user_agent_string, results_str, is_import=False, params_str=None, js_user_agent_string=None, js_user_agent_family=None, js_user_agent_v1=None, js_user_agent_v2=None, js_user_agent_v3=None, skip_dirty_update=False, **kwds): """Create result models and stores them as one transaction. Args: test_set: an instance of test_set_base. ip: a string to store as the user's IP. This should be hashed beforehand. user_agent_string: the http user agent string. results_str: a string like 'test1=time1,test2=time2,[...]'. is_import: if True, skip checking test_keys and do not mark dirty. params_str: a string representation of test_set_params.Params. js_user_agent_string: chrome frame ua string from client-side JavaScript. js_user_agent_family: This is an override for the family name to deal with the fact that IE platform preview (for instance) cannot be distinguished by user_agent_string, but only in javascript. js_user_agent_v1: v1 override - see above. js_user_agent_v2: v1 override - see above. js_user_agent_v3: v1 override - see above. skip_dirty_update: For tests, allow skipping the update-dirty task. kwds: optional fields including 'loader_id'. Returns: A ResultParent instance. """ logging.debug('ResultParent.AddResult category=%s' % test_set.category) if params_str in ('None', ''): # params_str should either unset, None, or a non-empty string logging.debug('params_str is wonky') raise ValueError user_agent = UserAgent.factory(user_agent_string, js_user_agent_string=js_user_agent_string, js_user_agent_family=js_user_agent_family, js_user_agent_v1=js_user_agent_v1, js_user_agent_v2=js_user_agent_v2, js_user_agent_v3=js_user_agent_v3) parent = cls(category=test_set.category, ip=ip, user_agent=user_agent, user_agent_string_list=user_agent.get_string_list(), params_str=params_str, **kwds) try: results = test_set.GetResults(results_str, ignore_key_errors=is_import) except test_set_base.ParseResultsKeyError, e: logging.warn(e) return None
def ConfirmUa(request): """Confirm User-Agents""" search_browser = request.REQUEST.get('browser', '') confirmed = request.REQUEST.get('confirmed') search_confirmed = confirmed == 'on' search_changed = request.REQUEST.get('changed', False) cursor = request.REQUEST.get('cursor', '') limit = int(request.REQUEST.get('limit', 30)) if 'search' in request.REQUEST: pass elif 'submit' in request.REQUEST: return SubmitChanges(request) # Build up the query. query = db.Query(UserAgent) if search_browser != '': string_list = UserAgent.parse_to_string_list(search_browser) logging.info('string_list: %s' % string_list) ua_bits = ['family', 'v1', 'v2', 'v3'] for index, item in enumerate(string_list): logging.info('adding %s=%s' % (ua_bits[index], item)) query.filter('%s =' % ua_bits[index], item) query.filter('confirmed =', search_confirmed) query.order('-created') if cursor: query.with_cursor(cursor) user_agents = query.fetch(limit=limit) params = { 'page_title': 'Confirm User-Agents', 'user_agents': user_agents, 'cursor': cursor, 'next_cursor': query.cursor(), 'search_browser': search_browser, 'search_confirmed': search_confirmed, 'search_changed': search_changed, 'csrf_token': request.session['csrf_token'], 'use_parse_service': True, } return Render(request, 'admin/confirm-ua.html', params)
def GetUserAgent(browser): return UserAgent.factory(GetUserAgentString(browser))
def DataDump(request): """This is used by bin/data_dump.py to replicate the datastore.""" model = request.REQUEST.get('model') key_prefix = request.REQUEST.get('key_prefix', '') keys_list = request.REQUEST.get('keys') time_limit = int(request.REQUEST.get('time_limit', 3)) if keys_list: keys = ['%s%s' % (key_prefix, key) for key in keys_list.split(',')] else: return http.HttpResponseBadRequest('"keys" is a required parameter.') start_time = datetime.datetime.now() if model == 'ResultParent': query = pager.PagerQuery(ResultParent, keys_only=True) elif model == 'UserAgent': query = pager.PagerQuery(UserAgent) else: return http.HttpResponseBadRequest( 'model must be one of "ResultParent", "UserAgent".') data = [] error = None if model == 'ResultParent': result_time_query = ResultTime.gql('WHERE ANCESTOR IS :1') for result_parent_key in keys: if (datetime.datetime.now() - start_time).seconds > time_limit: error = 'Over time limit' break try: p = ResultParent.get(result_parent_key) except db.Timeout: error = 'db.Timeout: ResultParent' break if not p: data.append({ 'model_class': 'ResultParent', 'lost_key': result_parent_key, }) continue result_time_query.bind(p.key()) try: result_times = result_time_query.fetch(1000) except db.Timeout: error = 'db.Timeout: ResultTime' break row_data = [{ 'model_class': 'ResultParent', 'result_parent_key': result_parent_key, 'category': p.category, 'user_agent_key': str( ResultParent.user_agent.get_value_for_datastore(p)), 'ip': p.ip, 'user_id': p.user and p.user.user_id() or None, 'created': p.created and p.created.isoformat() or None, 'params_str': p.params_str, 'loader_id': hasattr(p, 'loader_id') and p.loader_id or None, }] is_dirty = False for result_time in result_times: if result_time.dirty: is_dirty = True break row_data.append({ 'model_class': 'ResultTime', 'result_time_key': str(result_time.key()), 'result_parent_key': str(result_parent_key), 'test': result_time.test, 'score': result_time.score, }) if is_dirty: data.append({'dirty_key': result_parent_key,}) else: data.extend(row_data) elif model == 'UserAgent': try: user_agents = UserAgent.get(keys) except db.Timeout: error = 'db.Timeout: UserAgent' else: for key, ua in zip(keys, user_agents): if ua: data.append({ 'model_class': 'UserAgent', 'user_agent_key': key, 'string': ua.string, 'family': ua.family, 'v1': ua.v1, 'v2': ua.v2, 'v3': ua.v3, 'confirmed': ua.confirmed, 'created': ua.created and ua.created.isoformat() or None, 'js_user_agent_string': (hasattr(ua, 'js_user_agent_string') and ua.js_user_agent_string or None), }) else: data.append({ 'model_class': 'UserAgent', 'lost_key': key, }) response_params = { 'data': data, } if error: response_params['error'] = error return http.HttpResponse(content=simplejson.dumps(response_params), content_type='application/json')