def device_func(self, obj): """ Reformat the device format. In this, values have been sent with the hits, but if User Agent has been saved this could be parsed in a similar way. This equates to the 'deviceInformation' record. @param obj: Session object @return: The device information """ user_agent = self.retrieve_value(obj, 'cd' + USER_AGENT_CD, 0) if user_agent: ua = ua_parse(user_agent) if ua.is_pc: device = 'desktop' elif ua.is_tablet: device = 'tablet' else: device = 'mobile' dev = {'screenResolution': self.retrieve_value(obj, 'sr', 0), 'browserSize': self.retrieve_value(obj, 'vp', 0), 'language': self.retrieve_value(obj, 'ul', 0), 'browser': ua.browser.family, 'browserVersion': ua.browser.version_string, 'deviceCategory': device, 'operatingSystem': ua.os.family + " " + ua.os.version_string, 'isMobile': False if ua.is_pc else True, 'mobileDeviceModel': None if ua.is_pc else ua.device.model, 'mobileDeviceBranding': None if ua.is_pc else ua.device.brand, 'mobileDeviceInfo': None if ua.is_pc else ua.device.brand + " " + ua.device.family + " " \ + ua.device.model } else: dev = empty_device return {k: dev.get(k) for k in device_order}
def get_template_namespace(self): """Override `tornado.web.RequestHandler.get_template_namespace` static method add those new features: """ namespace = super(BaseHandler, self).get_template_namespace() namespace['site'] = self.config.read_config() try: user_agent = self.request.headers['User-Agent'] except KeyError: user_agent = '-' namespace['req'] = { # 当前请求路径 'path': urlparse.urlparse(self.request.uri).path, 'hostname': self.request.protocol + '://' + self.request.host, # userAgent 实例 'user_agent': ua_parse(user_agent) } namespace['archives'] = self.posts.get_archives() namespace['tags'] = self.posts.get_tags() namespace['categories'] = [] return namespace
def check_user_agent(self, value: str): """ 根据UA,获取设备信息 """ if not value or self.user_agent == value: return self.user_agent = value self.device_key = idshift.hash_md5(f'{self.usrid}:{value}') ua_info = ua_parse(self.user_agent) self.ua_info = dict( device=dict( family=ua_info.device.family, brand=ua_info.device.brand, model=ua_info.device.model, ), os=dict( family=ua_info.os.family, version=ua_info.os.version_string, ), browser=dict( family=ua_info.browser.family, version=ua_info.browser.version_string, ) ) self.app_type = self.get_app_type() self.device_name = ' '.join([i.strip() for i in str(ua_info).split('/')][:2]) up_fields = ['app_type', 'user_agent', 'device_key', 'device_name', 'ua_info', 'updated_at'] self.save(update_fields=up_fields)
def process_data(self, dfs, sids): """ Create an array of sessions, with each session taking a single row @param dfs: The dataframe of sessions @param sids: The session ids @return: """ array = [] # Fetch the Array for x in range(0, len(sids)): # Clear Session Level Values, Dicts and Lists key = sids.iloc[x] obj = dfs[key] user_agent = self.retrieve_value(obj, 'cd30', 0) if user_agent: ua = ua_parse(user_agent) if ua.is_bot: continue try: session = self.session_func(obj) except Exception as ex: self.logger.critical('There was an Execption: {}'.format(ex)) raise Exception(ex) # Add session to list array.append(session.copy()) return array
def parser_ua(ua_string): ''' 解析user-agent ''' info = {} info['spider'] = False try: msg = ua_parse(ua_string) if msg.is_pc: info['dev'] = 'PC' elif msg.is_tablet: info['dev'] = 'Pad' elif msg.is_mobile: info['dev'] = 'MObile' else: info['dev'] = 'Unknow' if msg.is_bot: info['spider'] = True info["type"] = msg.os.family+' '+str(msg.os.version_string) info["ua"] = msg.browser.family+' '+str(msg.browser.version_string) return info except Exception as e: return info
def get_user_agent(): """ Returns a user-agents object if we can get a user agent. Otherwise, returns None. """ if "HTTP_USER_AGENT" not in os.environ: return None else: return ua_parse(os.environ["HTTP_USER_AGENT"])
def calendars(request): try: is_mobile = ua_parse(str(request.META['HTTP_USER_AGENT'])).is_mobile except BaseException: is_mobile = False return render( request, 'calendars.html', context={'is_mobile': is_mobile} )
def get_browser_from_user_agent(self, request): try: user_agent = ua_parse(request.headers['User-Agent']).browser return ("%s_%s" % (user_agent.family, user_agent.version_string)).lower().replace(" ", "_").replace(".", "_") except Exception as e: if DEBUG: print e, type(e) print "could not get User-Agent" return ""
def do_GET(self): parsed_path = urlparse(self.path) msg = """CLIENT VALUES: client_address=%s (%s) command=%s path=%s real path=%s query=%s request_version=%s """ % (self.client_address, self.address_string(), self.command, self.path, parsed_path.path,parsed_path.query, self.request_version,) msg += """SERVER VALUES: server_version=%s sys_version=%s protocol_version=%s """ % (self.server_version, self.sys_version,self.protocol_version,) msg += '\nHEADERS:\n' for name, value in sorted(self.headers.items()): msg += ('\t\t%s=%s\n' % (name, value.rstrip())) #OS guesing feature if self.headers.has_key('user-agent'): user_agent = ua_parse(self.headers['user-agent']) msg += """\nUser-Agent guessing: Browser: %s %s OS: %s %s Device: %s PC: %s Mobile: %s Tablet: %s Touch: %s Bot: %s\n""" % \ (user_agent.browser.family, user_agent.browser.version_string, user_agent.os.family,user_agent.os.version_string, user_agent.device.family, user_agent.is_pc, user_agent.is_mobile,user_agent.is_tablet, user_agent.is_touch_capable,user_agent.is_bot, ) self.send_response(200) self.end_headers() self.wfile.write(msg) #self.wfile.write('OK') #logging.info(message) return
def get_default_view(request): if 'default_mode' not in request.session: user_agent = ua_parse(request.META['HTTP_USER_AGENT']) if user_agent.is_mobile: mode = 'smartphone' else: mode = 'classic' request.session['default_mode'] = mode return request.session['default_mode']
def get_default_view(request): if 'default_mode' not in request.session: user_agent = ua_parse(request.META.get('HTTP_USER_AGENT', '')) if user_agent.is_mobile: mode = 'smartphone' else: mode = 'classic' request.session['default_mode'] = mode return request.session['default_mode']
def get_browser_from_user_agent(self, request): try: user_agent = ua_parse(request.headers['User-Agent']).browser return ("%s_%s" % (user_agent.family, user_agent.version_string) ).lower().replace(" ", "_").replace(".", "_") except Exception as e: if DEBUG: print e, type(e) print "could not get User-Agent" return ""
async def on_connect(self, sid, environ): logger.info('Client connected, sid=%s' % (sid)) print("=" * 100) print(environ) print(dir(environ['aiohttp.request'])) print(environ['aiohttp.request'].scheme) print("=" * 100) client = { 'user_agent': environ.get('HTTP_USER_AGENT'), 'remote_ip': environ['aiohttp.request'].remote, 'device': {}, } payload = { 'sid': sid, 'client': client, } try: user_agent = ua_parse(environ['HTTP_USER_AGENT']) client['device'].update({ 'is_mobile': user_agent.is_mobile, 'is_tablet': user_agent.is_tablet, 'is_touch_capable': user_agent.is_touch_capable, 'is_pc': user_agent.is_pc, 'is_bot': user_agent.is_bot, 'os': { 'family': user_agent.os.family, 'version': user_agent.os.version_string, }, 'device': { 'family': user_agent.device.family, 'brand': user_agent.device.brand, 'model': user_agent.device.model, }, 'browser': { 'family': user_agent.browser.family, 'version': user_agent.browser.version_string, } }) except: logger.warning('Could not parse client user agent (ua=%s)' % (environ.get('HTTP_USER_AGENT'))) self.context.socket_clients[sid] = client self.context.queue_event('socket', 'client.connected', payload=payload) await self.emit('status', self.context.status, room=sid)
def parse_useragent(self, ua): sys.path.append(self.prj_path + "/script/useragent_parser") from user_agents import parse as ua_parse ua_dict = {} user_agent = ua_parse(ua) if user_agent.browser.family != None: ua_dict['browser'] = user_agent.browser.family if user_agent.browser.version_string != None: ua_dict['browser_version'] = user_agent.browser.version_string if user_agent.os.family != None: ua_dict['os'] = user_agent.os.family if user_agent.device.family != None: ua_dict['device'] = user_agent.device.family if user_agent.device.brand != None: ua_dict['device_type'] = user_agent.device.brand if user_agent.device.model != None: ua_dict['device_version'] = user_agent.device.model return ua_dict
def __process_pageviews(self, f): print("BrowserParser - processing pageviews from: " + f) ua_cache = {} with open(f) as csvfile: r = csv.DictReader(csvfile, delimiter=',') for row in r: # save UA to cache user_agent = row['user_agent'] if user_agent not in ua_cache: ua_cache[user_agent] = ua_parse(user_agent) # save all user devices for (user <-> device mapping) browser_id = row['browser_id'] if browser_id not in self.browsers_with_users: self.browsers_with_users[browser_id] = {'user_ids': set()} if row['user_id']: self.browsers_with_users[browser_id]['user_ids'].add( row['user_id']) self.browsers_with_users[browser_id]['ua'] = ua_cache[ user_agent] # continue with pageviews only for subscribers if row['subscriber'] == 'True': continue if row['derived_referer_medium'] == '': continue if browser_id not in self.data: self.data[browser_id] = empty_data_entry( {'user_ids': set()}) record = self.data[browser_id] update_record_from_pageviews_row(record, row) if row['user_id']: record['user_ids'].add(row['user_id']) record['ua'] = ua_cache[user_agent] self.data[browser_id] = record
def parser_ua(self,ua_string): info = {} info['spider'] = False msg = ua_parse(ua_string) if msg.is_pc: info['dev'] = 'PC' elif msg.is_tablet: info['dev'] = 'Pad' elif msg.is_mobile: info['dev'] = 'MObile' else: info['dev'] = 'Unknow' if msg.is_bot: info['spider'] = True info["type"] = msg.os.family+' '+str(msg.os.version_string) info["ua"] = msg.browser.family+' '+str(msg.browser.version_string) return info
def log_event(self, msg=None, event_type=None, action=None, key=None, value=None, agent=None): """ This is the primary user-facing logging interface, so there' s a bit of a high bar for using it. The basic idea of creating a log entry is that we're doing a bit of the semantic logging (i.e. strongly typed) thing, so, depending on which kwargs you use when calling this method, your final outcome/message is going to vary somewhat. That said, none of the kwargs here are mandatory, because context. """ # # baseline attributes # # for those who still raw-dog it; force to ASCII: if msg is not None: msg = msg.encode("ascii", 'ignore') # 0.) method: determine caller method curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) method = calframe[2][3] # 1.) event: determine event type if it's None if event_type is None: event_type = method # 2.) action: figure out the action; set the special action vars if action is None: action = method.split("_")[0] action_word, action_preposition = utils.action_keyword(action) # 3.) key: default the key if we don't get one if key is None: key = " ".join(method.split("_")[1:]) key = key.encode('ascii', 'ignore') if key == "settlement": key = "" # 4.) value; default the value if we don't get one if value is None: value = "UNKNOWN" if type(value) != int: value = value.encode('ascii', 'ignore') # set 'created_by' created_by = None created_by_email = None if request: if hasattr(request, 'User'): created_by = request.User.user['_id'] created_by_email = request.User.user['login'] if agent is None: agent = "user" # set 'attribute_modified' attribute_modified = { 'key': key, 'value': value, } if attribute_modified['key'] is not None: attribute_modified['key_pretty'] = key.replace("_", " ").replace( "and", "&").title() if attribute_modified['value'] is not None: attribute_modified['value_pretty'] = str(value).replace("_", " ") d = { 'version': 1.3, 'agent': agent, "created_on": datetime.now(), 'created_by': created_by, 'created_by_email': created_by_email, "settlement_id": self.settlement_id, "ly": self.get_current_ly(), 'event_type': event_type, 'event': msg, 'modified': { 'attribute': attribute_modified }, } # survivor, if it's a survivor if self.collection == 'survivors': d['survivor_id'] = self.survivor['_id'] # target is the settlement, unless a survivor object calls this method action_target = "settlement" if 'survivor_id' in d.keys(): d['modified']['asset'] = { 'type': 'survivor', '_id': d['survivor_id'], 'name': self.survivor['name'], 'sex': self.get_sex(), } action_target = "survivor" else: d['modified']['asset'] = { "type": "settlement", "name": self.settlement['name'], '_id': self.settlement_id } # create the 'action' d['action'] = { 'word': action_word, 'preposition': str(action_preposition) } # now write the repr, which is like...the simplified sentence of the event if key is None and value is None: d['action']['repr'] = " ".join(['modified', action_target]) elif key is not None and value is None: d['action']['repr'] = " ".join(['modified', action_target, key]) else: if action_preposition is None: d['action']['repr'] = action_word elif action_word in ['set']: d['action']['repr'] = " ".join([ action_word, action_target, key, action_preposition, str(value) ]) elif action_word in ['unset']: d['action']['repr'] = " ".join( [action_word, action_target, key]) elif action_target == "survivor" and action_preposition is not None: d['action']['repr'] = " ".join([ action_word, "'%s'" % value, action_preposition, str(key) ]) else: d['action']['repr'] = " ".join([ action_word, "'%s'" % value, action_preposition, action_target, str(key) ]) # default a message, if incoming message is none if msg is None: # create messages for survivor updates if d['modified']['asset']['type'] == 'survivor': if d['agent'] == 'user' and action_preposition is None: d['event'] = " ".join([ d['created_by_email'], d['action']['word'], "%s [%s]" % (self.survivor['name'], self.get_sex()), ]) elif d['agent'] == 'user' and action_preposition is not None: d['event'] = " ".join([ d['created_by_email'], d['action']['word'], d['modified']['attribute']['value_pretty'], d['action']['preposition'], "%s [%s]" % (self.survivor['name'], self.get_sex()), d['modified']['attribute']['key_pretty'], ]) else: d['event'] = " ".join([ "%s [%s]" % (self.survivor['name'], self.get_sex()), d['action']['word'], d['modified']['attribute']['value_pretty'], d['action']['preposition'], d['modified']['attribute']['key_pretty'], ]) # create messages for settlements elif d['modified']['asset']['type'] == 'settlement': if d['agent'] == 'user' and action_preposition is None: d['event'] = " ".join([ d['created_by_email'], d['action']['word'], self.settlement['name'], ]) else: d['event'] = " ".join([ str(d['created_by_email']), str(d['action']['repr']), ]) else: d['event'] = 'Updated %s' % self # elif agent == "automation": # d['event'] = d['action']['repr'] # enforce terminal punctuation on the event "sentence" if d['event'][-1] not in ['.', '!']: d['event'] = d['event'].strip() d['event'] += "." # d['event'] = d['event'].decode('ascii','replace').encode('utf-8','replace') # finally, if we had a requester, now that we've settled on a message # text, update the requester's latest action with it if 'created_by' is not None: if request: ua_string = str(ua_parse(request.user_agent.string)) request.User.set_latest_action(d['event'], ua_string) # finally, insert the event (i.e. save) utils.mdb.settlement_events.insert(d) self.logger.info("%s event: %s" % (self, d['event']))
def ua(self): if not hasattr(self, '_ua'): self._ua = ua_parse(self.user_agent) return self._ua
def parser_ua(ua_string): ''' 解析user-agent ''' info = {} info['spider'] = False try: msg = ua_parse(ua_string) if msg.is_pc: info['dev'] = 'PC' elif msg.is_tablet: info['dev'] = 'Pad' elif msg.is_mobile: info['dev'] = 'MObile' else: info['dev'] = 'Unknow' if msg.is_bot: info['spider'] = True info["type"] = msg.os.family + ' ' + str(msg.os.version_string) info["ua"] = msg.browser.family + ' ' + str(msg.browser.version_string) return info except Exception as e: return info info = {} addr = "" g1 = GeoIPUtil() g2 = GeoIP2Util() if is_intranet(ipaddr): info["address"] = "局域网内地址" info['weidu'] = info['jingdu'] = info['country'] = info[ 'subdivision'] = info['city'] = "" else: try: longitude, latitude = g1.get_lat_alt(ipaddr) info['weidu'] = latitude info['jingdu'] = longitude country, subdivision, city = g2.get_ip_location(ipaddr) if country == u"中国": if not subdivision and not city: addr = country elif subdivision.find(u'市') == -1 and subdivision not in [ u'上海' ] and city: addr = subdivision.strip(u'省') + u"省\t" + city elif subdivision and not city: addr = country + ' ' + subdivision else: addr = subdivision + "\t" + city else: if subdivision in [ u'台北市', u'新北市', u'基隆市', u'新竹市', u'嘉义市', u'台中市', u'台南市', u'高雄市', u'屏东市' ]: subdivision = "台湾省" + ' ' + subdivision if country in [u'香港', u'澳门'] and subdivision: subdivision = country + ' ' + subdivision country = '中国' elif country in [u'香港', u'澳门'] and not subdivision: subdivision = country country = '中国' if not subdivision and not city: addr = country elif subdivision and not city: addr = country + ' ' + subdivision else: addr = country.replace( u'台湾', u'中国') + "\t" + subdivision + "\t" + city if addr: info["address"] = addr info["country"] = country.replace(u'台湾', u'中国') info["subdivision"] = subdivision info["city"] = city except Exception as e: print str(e) return info
def ua(self): if not hasattr(self, '_ua'): self._ua = ua_parse(self.requests.first().user_agent) return self._ua
def do_GET(self): parsed_path = urlparse(self.path) msg = """CLIENT VALUES: client_address=%s (%s) command=%s path=%s real path=%s query=%s request_version=%s """ % ( self.client_address, self.address_string(), self.command, self.path, parsed_path.path, parsed_path.query, self.request_version, ) msg += """SERVER VALUES: server_version=%s sys_version=%s protocol_version=%s """ % ( self.server_version, self.sys_version, self.protocol_version, ) msg += '\nHEADERS:\n' for name, value in sorted(self.headers.items()): msg += ('\t\t%s=%s\n' % (name, value.rstrip())) #OS guesing feature if self.headers.has_key('user-agent'): user_agent = ua_parse(self.headers['user-agent']) msg += """\nUser-Agent guessing: Browser: %s %s OS: %s %s Device: %s PC: %s Mobile: %s Tablet: %s Touch: %s Bot: %s\n""" % \ (user_agent.browser.family, user_agent.browser.version_string, user_agent.os.family,user_agent.os.version_string, user_agent.device.family, user_agent.is_pc, user_agent.is_mobile,user_agent.is_tablet, user_agent.is_touch_capable,user_agent.is_bot, ) self.send_response(200) self.end_headers() self.wfile.write(msg) #self.wfile.write('OK') #logging.info(message) return