def device_func(self, obj):
"""
Reformat the device format. In this, values have been sent with the hits, but if User Agent has been saved
this could be parsed in a similar way. This equates to the 'deviceInformation' record.
@param obj: Session object
@return: The device information
"""
user_agent = self.retrieve_value(obj, 'cd' + USER_AGENT_CD, 0)
if user_agent:
ua = ua_parse(user_agent)
if ua.is_pc:
device = 'desktop'
elif ua.is_tablet:
device = 'tablet'
else:
device = 'mobile'
dev = {'screenResolution': self.retrieve_value(obj, 'sr', 0),
'browserSize': self.retrieve_value(obj, 'vp', 0),
'language': self.retrieve_value(obj, 'ul', 0),
'browser': ua.browser.family,
'browserVersion': ua.browser.version_string,
'deviceCategory': device,
'operatingSystem': ua.os.family + " " + ua.os.version_string,
'isMobile': False if ua.is_pc else True,
'mobileDeviceModel': None if ua.is_pc else ua.device.model,
'mobileDeviceBranding': None if ua.is_pc else ua.device.brand,
'mobileDeviceInfo': None if ua.is_pc else ua.device.brand + " " + ua.device.family + " " \
+ ua.device.model
}
else:
dev = empty_device
return {k: dev.get(k) for k in device_order}
def get_template_namespace(self):
"""Override `tornado.web.RequestHandler.get_template_namespace` static method
add those new features:
"""
namespace = super(BaseHandler, self).get_template_namespace()
namespace['site'] = self.config.read_config()
try:
user_agent = self.request.headers['User-Agent']
except KeyError:
user_agent = '-'
namespace['req'] = {
# 当前请求路径
'path': urlparse.urlparse(self.request.uri).path,
'hostname': self.request.protocol + '://' + self.request.host,
# userAgent 实例
'user_agent': ua_parse(user_agent)
}
namespace['archives'] = self.posts.get_archives()
namespace['tags'] = self.posts.get_tags()
namespace['categories'] = []
return namespace
def check_user_agent(self, value: str):
""" 根据UA，获取设备信息 """
if not value or self.user_agent == value:
return
self.user_agent = value
self.device_key = idshift.hash_md5(f'{self.usrid}:{value}')
ua_info = ua_parse(self.user_agent)
self.ua_info = dict(
device=dict(
family=ua_info.device.family,
brand=ua_info.device.brand,
model=ua_info.device.model,
),
os=dict(
family=ua_info.os.family,
version=ua_info.os.version_string,
),
browser=dict(
family=ua_info.browser.family,
version=ua_info.browser.version_string,
)
)
self.app_type = self.get_app_type()
self.device_name = ' '.join([i.strip() for i in str(ua_info).split('/')][:2])
up_fields = ['app_type', 'user_agent', 'device_key', 'device_name', 'ua_info', 'updated_at']
self.save(update_fields=up_fields)
def process_data(self, dfs, sids):
"""
Create an array of sessions, with each session taking a single row
@param dfs: The dataframe of sessions
@param sids: The session ids
@return:
"""
array = []
# Fetch the Array
for x in range(0, len(sids)):
# Clear Session Level Values, Dicts and Lists
key = sids.iloc[x]
obj = dfs[key]
user_agent = self.retrieve_value(obj, 'cd30', 0)
if user_agent:
ua = ua_parse(user_agent)
if ua.is_bot:
continue
try:
session = self.session_func(obj)
except Exception as ex:
self.logger.critical('There was an Execption: {}'.format(ex))
raise Exception(ex)
# Add session to list
array.append(session.copy())
return array
def parser_ua(ua_string):
'''
解析user-agent
'''
info = {}
info['spider'] = False
try:
msg = ua_parse(ua_string)
if msg.is_pc:
info['dev'] = 'PC'
elif msg.is_tablet:
info['dev'] = 'Pad'
elif msg.is_mobile:
info['dev'] = 'MObile'
else:
info['dev'] = 'Unknow'
if msg.is_bot:
info['spider'] = True
info["type"] = msg.os.family+' '+str(msg.os.version_string)
info["ua"] = msg.browser.family+' '+str(msg.browser.version_string)
return info
except Exception as e:
return info
def get_user_agent():
""" Returns a user-agents object if we can get a user agent. Otherwise,
returns None. """
if "HTTP_USER_AGENT" not in os.environ:
return None
else:
return ua_parse(os.environ["HTTP_USER_AGENT"])
def calendars(request):
try:
is_mobile = ua_parse(str(request.META['HTTP_USER_AGENT'])).is_mobile
except BaseException:
is_mobile = False
return render(
request,
'calendars.html',
context={'is_mobile': is_mobile}
)
def get_browser_from_user_agent(self, request):
try:
user_agent = ua_parse(request.headers['User-Agent']).browser
return ("%s_%s" % (user_agent.family, user_agent.version_string)).lower().replace(" ", "_").replace(".", "_")
except Exception as e:
if DEBUG:
print e, type(e)
print "could not get User-Agent"
return ""
def do_GET(self):
parsed_path = urlparse(self.path)
msg = """CLIENT VALUES:
client_address=%s (%s)
command=%s
path=%s
real path=%s
query=%s
request_version=%s
""" % (self.client_address, self.address_string(), self.command, self.path,
parsed_path.path,parsed_path.query, self.request_version,)
msg += """SERVER VALUES:
server_version=%s
sys_version=%s
protocol_version=%s
""" % (self.server_version, self.sys_version,self.protocol_version,)
msg += '\nHEADERS:\n'
for name, value in sorted(self.headers.items()):
msg += ('\t\t%s=%s\n' % (name, value.rstrip()))
#OS guesing feature
if self.headers.has_key('user-agent'):
user_agent = ua_parse(self.headers['user-agent'])
msg += """\nUser-Agent guessing:
Browser: %s %s
OS: %s %s
Device: %s
PC: %s
Mobile: %s
Tablet: %s
Touch: %s
Bot: %s\n""" % \
(user_agent.browser.family, user_agent.browser.version_string,
user_agent.os.family,user_agent.os.version_string,
user_agent.device.family,
user_agent.is_pc, user_agent.is_mobile,user_agent.is_tablet,
user_agent.is_touch_capable,user_agent.is_bot,
)
self.send_response(200)
self.end_headers()
self.wfile.write(msg)
#self.wfile.write('OK')
#logging.info(message)
return
def get_default_view(request):
if 'default_mode' not in request.session:
user_agent = ua_parse(request.META['HTTP_USER_AGENT'])
if user_agent.is_mobile:
mode = 'smartphone'
else:
mode = 'classic'
request.session['default_mode'] = mode
return request.session['default_mode']
def get_default_view(request):
if 'default_mode' not in request.session:
user_agent = ua_parse(request.META.get('HTTP_USER_AGENT', ''))
if user_agent.is_mobile:
mode = 'smartphone'
else:
mode = 'classic'
request.session['default_mode'] = mode
return request.session['default_mode']
def get_browser_from_user_agent(self, request):
try:
user_agent = ua_parse(request.headers['User-Agent']).browser
return ("%s_%s" % (user_agent.family, user_agent.version_string)
).lower().replace(" ", "_").replace(".", "_")
except Exception as e:
if DEBUG:
print e, type(e)
print "could not get User-Agent"
return ""
async def on_connect(self, sid, environ):
logger.info('Client connected, sid=%s' % (sid))
print("=" * 100)
print(environ)
print(dir(environ['aiohttp.request']))
print(environ['aiohttp.request'].scheme)
print("=" * 100)
client = {
'user_agent': environ.get('HTTP_USER_AGENT'),
'remote_ip': environ['aiohttp.request'].remote,
'device': {},
}
payload = {
'sid': sid,
'client': client,
}
try:
user_agent = ua_parse(environ['HTTP_USER_AGENT'])
client['device'].update({
'is_mobile': user_agent.is_mobile,
'is_tablet': user_agent.is_tablet,
'is_touch_capable': user_agent.is_touch_capable,
'is_pc': user_agent.is_pc,
'is_bot': user_agent.is_bot,
'os': {
'family': user_agent.os.family,
'version': user_agent.os.version_string,
},
'device': {
'family': user_agent.device.family,
'brand': user_agent.device.brand,
'model': user_agent.device.model,
},
'browser': {
'family': user_agent.browser.family,
'version': user_agent.browser.version_string,
}
})
except:
logger.warning('Could not parse client user agent (ua=%s)' %
(environ.get('HTTP_USER_AGENT')))
self.context.socket_clients[sid] = client
self.context.queue_event('socket', 'client.connected', payload=payload)
await self.emit('status', self.context.status, room=sid)
def parse_useragent(self, ua):
sys.path.append(self.prj_path + "/script/useragent_parser")
from user_agents import parse as ua_parse
ua_dict = {}
user_agent = ua_parse(ua)
if user_agent.browser.family != None:
ua_dict['browser'] = user_agent.browser.family
if user_agent.browser.version_string != None:
ua_dict['browser_version'] = user_agent.browser.version_string
if user_agent.os.family != None:
ua_dict['os'] = user_agent.os.family
if user_agent.device.family != None:
ua_dict['device'] = user_agent.device.family
if user_agent.device.brand != None:
ua_dict['device_type'] = user_agent.device.brand
if user_agent.device.model != None:
ua_dict['device_version'] = user_agent.device.model
return ua_dict
def __process_pageviews(self, f):
print("BrowserParser - processing pageviews from: " + f)
ua_cache = {}
with open(f) as csvfile:
r = csv.DictReader(csvfile, delimiter=',')
for row in r:
# save UA to cache
user_agent = row['user_agent']
if user_agent not in ua_cache:
ua_cache[user_agent] = ua_parse(user_agent)
# save all user devices for (user <-> device mapping)
browser_id = row['browser_id']
if browser_id not in self.browsers_with_users:
self.browsers_with_users[browser_id] = {'user_ids': set()}
if row['user_id']:
self.browsers_with_users[browser_id]['user_ids'].add(
row['user_id'])
self.browsers_with_users[browser_id]['ua'] = ua_cache[
user_agent]
# continue with pageviews only for subscribers
if row['subscriber'] == 'True':
continue
if row['derived_referer_medium'] == '':
continue
if browser_id not in self.data:
self.data[browser_id] = empty_data_entry(
{'user_ids': set()})
record = self.data[browser_id]
update_record_from_pageviews_row(record, row)
if row['user_id']:
record['user_ids'].add(row['user_id'])
record['ua'] = ua_cache[user_agent]
self.data[browser_id] = record
def parser_ua(self,ua_string):
info = {}
info['spider'] = False
msg = ua_parse(ua_string)
if msg.is_pc:
info['dev'] = 'PC'
elif msg.is_tablet:
info['dev'] = 'Pad'
elif msg.is_mobile:
info['dev'] = 'MObile'
else:
info['dev'] = 'Unknow'
if msg.is_bot:
info['spider'] = True
info["type"] = msg.os.family+' '+str(msg.os.version_string)
info["ua"] = msg.browser.family+' '+str(msg.browser.version_string)
return info
def log_event(self,
msg=None,
event_type=None,
action=None,
key=None,
value=None,
agent=None):
""" This is the primary user-facing logging interface, so there' s a bit
of a high bar for using it.
The basic idea of creating a log entry is that we're doing a bit of the
semantic logging (i.e. strongly typed) thing, so, depending on which
kwargs you use when calling this method, your final outcome/message is
going to vary somewhat.
That said, none of the kwargs here are mandatory, because context.
"""
#
# baseline attributes
#
# for those who still raw-dog it; force to ASCII:
if msg is not None:
msg = msg.encode("ascii", 'ignore')
# 0.) method: determine caller method
curframe = inspect.currentframe()
calframe = inspect.getouterframes(curframe, 2)
method = calframe[2][3]
# 1.) event: determine event type if it's None
if event_type is None:
event_type = method
# 2.) action: figure out the action; set the special action vars
if action is None:
action = method.split("_")[0]
action_word, action_preposition = utils.action_keyword(action)
# 3.) key: default the key if we don't get one
if key is None:
key = " ".join(method.split("_")[1:])
key = key.encode('ascii', 'ignore')
if key == "settlement":
key = ""
# 4.) value; default the value if we don't get one
if value is None:
value = "UNKNOWN"
if type(value) != int:
value = value.encode('ascii', 'ignore')
# set 'created_by'
created_by = None
created_by_email = None
if request:
if hasattr(request, 'User'):
created_by = request.User.user['_id']
created_by_email = request.User.user['login']
if agent is None:
agent = "user"
# set 'attribute_modified'
attribute_modified = {
'key': key,
'value': value,
}
if attribute_modified['key'] is not None:
attribute_modified['key_pretty'] = key.replace("_", " ").replace(
"and", "&").title()
if attribute_modified['value'] is not None:
attribute_modified['value_pretty'] = str(value).replace("_", " ")
d = {
'version': 1.3,
'agent': agent,
"created_on": datetime.now(),
'created_by': created_by,
'created_by_email': created_by_email,
"settlement_id": self.settlement_id,
"ly": self.get_current_ly(),
'event_type': event_type,
'event': msg,
'modified': {
'attribute': attribute_modified
},
}
# survivor, if it's a survivor
if self.collection == 'survivors':
d['survivor_id'] = self.survivor['_id']
# target is the settlement, unless a survivor object calls this method
action_target = "settlement"
if 'survivor_id' in d.keys():
d['modified']['asset'] = {
'type': 'survivor',
'_id': d['survivor_id'],
'name': self.survivor['name'],
'sex': self.get_sex(),
}
action_target = "survivor"
else:
d['modified']['asset'] = {
"type": "settlement",
"name": self.settlement['name'],
'_id': self.settlement_id
}
# create the 'action'
d['action'] = {
'word': action_word,
'preposition': str(action_preposition)
}
# now write the repr, which is like...the simplified sentence of the event
if key is None and value is None:
d['action']['repr'] = " ".join(['modified', action_target])
elif key is not None and value is None:
d['action']['repr'] = " ".join(['modified', action_target, key])
else:
if action_preposition is None:
d['action']['repr'] = action_word
elif action_word in ['set']:
d['action']['repr'] = " ".join([
action_word, action_target, key, action_preposition,
str(value)
])
elif action_word in ['unset']:
d['action']['repr'] = " ".join(
[action_word, action_target, key])
elif action_target == "survivor" and action_preposition is not None:
d['action']['repr'] = " ".join([
action_word,
"'%s'" % value, action_preposition,
str(key)
])
else:
d['action']['repr'] = " ".join([
action_word,
"'%s'" % value, action_preposition, action_target,
str(key)
])
# default a message, if incoming message is none
if msg is None:
# create messages for survivor updates
if d['modified']['asset']['type'] == 'survivor':
if d['agent'] == 'user' and action_preposition is None:
d['event'] = " ".join([
d['created_by_email'],
d['action']['word'],
"%s [%s]" % (self.survivor['name'], self.get_sex()),
])
elif d['agent'] == 'user' and action_preposition is not None:
d['event'] = " ".join([
d['created_by_email'],
d['action']['word'],
d['modified']['attribute']['value_pretty'],
d['action']['preposition'],
"%s [%s]" % (self.survivor['name'], self.get_sex()),
d['modified']['attribute']['key_pretty'],
])
else:
d['event'] = " ".join([
"%s [%s]" % (self.survivor['name'], self.get_sex()),
d['action']['word'],
d['modified']['attribute']['value_pretty'],
d['action']['preposition'],
d['modified']['attribute']['key_pretty'],
])
# create messages for settlements
elif d['modified']['asset']['type'] == 'settlement':
if d['agent'] == 'user' and action_preposition is None:
d['event'] = " ".join([
d['created_by_email'],
d['action']['word'],
self.settlement['name'],
])
else:
d['event'] = " ".join([
str(d['created_by_email']),
str(d['action']['repr']),
])
else:
d['event'] = 'Updated %s' % self
# elif agent == "automation":
# d['event'] = d['action']['repr']
# enforce terminal punctuation on the event "sentence"
if d['event'][-1] not in ['.', '!']:
d['event'] = d['event'].strip()
d['event'] += "."
# d['event'] = d['event'].decode('ascii','replace').encode('utf-8','replace')
# finally, if we had a requester, now that we've settled on a message
# text, update the requester's latest action with it
if 'created_by' is not None:
if request:
ua_string = str(ua_parse(request.user_agent.string))
request.User.set_latest_action(d['event'], ua_string)
# finally, insert the event (i.e. save)
utils.mdb.settlement_events.insert(d)
self.logger.info("%s event: %s" % (self, d['event']))
def ua(self):
if not hasattr(self, '_ua'):
self._ua = ua_parse(self.user_agent)
return self._ua
def parser_ua(ua_string):
'''
解析user-agent
'''
info = {}
info['spider'] = False
try:
msg = ua_parse(ua_string)
if msg.is_pc:
info['dev'] = 'PC'
elif msg.is_tablet:
info['dev'] = 'Pad'
elif msg.is_mobile:
info['dev'] = 'MObile'
else:
info['dev'] = 'Unknow'
if msg.is_bot:
info['spider'] = True
info["type"] = msg.os.family + ' ' + str(msg.os.version_string)
info["ua"] = msg.browser.family + ' ' + str(msg.browser.version_string)
return info
except Exception as e:
return info
info = {}
addr = ""
g1 = GeoIPUtil()
g2 = GeoIP2Util()
if is_intranet(ipaddr):
info["address"] = "局域网内地址"
info['weidu'] = info['jingdu'] = info['country'] = info[
'subdivision'] = info['city'] = ""
else:
try:
longitude, latitude = g1.get_lat_alt(ipaddr)
info['weidu'] = latitude
info['jingdu'] = longitude
country, subdivision, city = g2.get_ip_location(ipaddr)
if country == u"中国":
if not subdivision and not city:
addr = country
elif subdivision.find(u'市') == -1 and subdivision not in [
u'上海'
] and city:
addr = subdivision.strip(u'省') + u"省\t" + city
elif subdivision and not city:
addr = country + ' ' + subdivision
else:
addr = subdivision + "\t" + city
else:
if subdivision in [
u'台北市', u'新北市', u'基隆市', u'新竹市', u'嘉义市', u'台中市', u'台南市',
u'高雄市', u'屏东市'
]:
subdivision = "台湾省" + ' ' + subdivision
if country in [u'香港', u'澳门'] and subdivision:
subdivision = country + ' ' + subdivision
country = '中国'
elif country in [u'香港', u'澳门'] and not subdivision:
subdivision = country
country = '中国'
if not subdivision and not city:
addr = country
elif subdivision and not city:
addr = country + ' ' + subdivision
else:
addr = country.replace(
u'台湾', u'中国') + "\t" + subdivision + "\t" + city
if addr:
info["address"] = addr
info["country"] = country.replace(u'台湾', u'中国')
info["subdivision"] = subdivision
info["city"] = city
except Exception as e:
print str(e)
return info
def ua(self):
if not hasattr(self, '_ua'):
self._ua = ua_parse(self.requests.first().user_agent)
return self._ua
def do_GET(self):
parsed_path = urlparse(self.path)
msg = """CLIENT VALUES:
client_address=%s (%s)
command=%s
path=%s
real path=%s
query=%s
request_version=%s
""" % (
self.client_address,
self.address_string(),
self.command,
self.path,
parsed_path.path,
parsed_path.query,
self.request_version,
)
msg += """SERVER VALUES:
server_version=%s
sys_version=%s
protocol_version=%s
""" % (
self.server_version,
self.sys_version,
self.protocol_version,
)
msg += '\nHEADERS:\n'
for name, value in sorted(self.headers.items()):
msg += ('\t\t%s=%s\n' % (name, value.rstrip()))
#OS guesing feature
if self.headers.has_key('user-agent'):
user_agent = ua_parse(self.headers['user-agent'])
msg += """\nUser-Agent guessing:
Browser: %s %s
OS: %s %s
Device: %s
PC: %s
Mobile: %s
Tablet: %s
Touch: %s
Bot: %s\n""" % \
(user_agent.browser.family, user_agent.browser.version_string,
user_agent.os.family,user_agent.os.version_string,
user_agent.device.family,
user_agent.is_pc, user_agent.is_mobile,user_agent.is_tablet,
user_agent.is_touch_capable,user_agent.is_bot,
)
self.send_response(200)
self.end_headers()
self.wfile.write(msg)
#self.wfile.write('OK')
#logging.info(message)
return