def uas_parser(logRows=[]):
uas_parser = UASparser()
total_rows_in_log = len(logRows)
#only 'ua_icon' or 'os_icon' or both are allowed in entire_url
now = str(date.strftime(datetime.now(), '%d%m%y%H%M'))
out = open('ua_' + now + '.log', 'w')
for row_log in range(total_rows_in_log):
#todo : progress bar
#perc=(float(row_log)/float(total_rows_in_log))*100
#progress(perc)
sys.stdout.write("\rParsing row log #%s of %s" %
(str(row_log), str(total_rows_in_log)))
sys.stdout.flush()
ua_string = str(logRows[row_log][8])
parsed_ua = uas_parser.parse(ua_string)
if (parsed_ua['ua_name'] == "unknown"):
out.write("%s\t%s\t%s\t%s\n" %
(parsed_ua['typ'], parsed_ua['ua_name'],
parsed_ua['os_name'], ua_string))
else:
out.write(
"%s\t%s\t%s\n" %
(parsed_ua['typ'], parsed_ua['ua_name'], parsed_ua['os_name']))
#save in file
out.close()
class UserAgentType:
def __init__(self):
self.uas_parser = UASparser('/home/natty/FCDDOS/UASparserCache')
def isBrowser(self,userAgent):
if not userAgent or userAgent == None:
return True
if userAgent == '-':
return True
#print agentType
result = self.uas_parser.parse(str(userAgent))
agentType = result["typ"]
#print agentType,":::",userAgent,"\n"
if agentType in USER_AGENT_BROWSER_TYPES:
return True
else:
return False
def isRobot(self,userAgent):
if not userAgent or (userAgent == None):
return False
if userAgent == "-" or len(userAgent) ==0:
return False
result = self.uas_parser.parse(str(userAgent))
agentType = result["typ"]
#print agentType,":::",userAgent,"\n"
if agentType == USER_AGENT_ROBOT:
return True
else:
return False
def main():
uas_parser = UASparser('.')
ua = 'SonyEricssonK750i/R1L Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1'
print('User Agent: {0}'.format(ua))
# only 'ua_icon' or 'os_icon' or both are allowed in entire_url
result = uas_parser.parse(ua, entire_url='ua_icon,os_icon')
pprint(result)
def main():
print "www.ua-tracker.org parse script"
ualist = []
fr = open("www.ua-tracker.com user_agents.txt",'r')
for line in fr:
uaString = line.rstrip('\n')
uaString = uaString.rstrip()
uaString = uaString.lstrip()
if uaString != '':
x = uaobject()
x.uaString = uaString
ualist.append(x)
fr.close()
uas_parser = UASparser()
for (i,x) in enumerate(ualist):
# if i > 10: # debug
# break
print x.uaString
print "parsing UA string", i
x = parseua(x,uas_parser)
f = open("uatrack_data.txt",'w')
for x in ualist:
f.write(x+'\n')
f.close()
return 0
def uas_parser(logRows=[]):
uas_parser = UASparser()
total_rows_in_log = len(logRows)
#only 'ua_icon' or 'os_icon' or both are allowed in entire_url
now = str(date.strftime(datetime.now(), '%d%m%y%H%M'))
out = open('ua_' + now + '.log', 'w')
for row_log in range(total_rows_in_log):
#todo : progress bar
#perc=(float(row_log)/float(total_rows_in_log))*100
#progress(perc)
sys.stdout.write("\rParsing row log #%s of %s" % (str(row_log), str(total_rows_in_log)))
sys.stdout.flush()
ua_string = str(logRows[row_log][8])
parsed_ua = uas_parser.parse(ua_string)
if(parsed_ua['ua_name'] == "unknown"):
out.write("%s\t%s\t%s\t%s\n" % (parsed_ua['typ'], parsed_ua['ua_name'], parsed_ua['os_name'], ua_string))
else:
out.write("%s\t%s\t%s\n" % (parsed_ua['typ'], parsed_ua['ua_name'], parsed_ua['os_name']))
#save in file
out.close()
def main():
print "XML Parse Script"
tree = et.parse("allagents.xml")
# these two iterations do the same thing
# for elem in tree.getiterator("String"):
# print elem.text.encode('utf-8')
ualist = []
for elem in tree.getiterator("user-agent"):
stringelem = elem.find("String")
x = uaobject()
x.uaString = stringelem.text.encode('utf-8')
typeelem = elem.find("Type")
if typeelem.text == "R":
x.data['Type'] = "Robot"
elif typeelem.text == "B":
x.data['Type'] = "Browser"
elif typeelem.text == "C":
x.data['Type'] = "Validator"
elif typeelem.text == "P":
x.data['Type'] = "Proxy"
elif typeelem.text == "S":
x.data['Type'] = "Malicious"
elif typeelem.text == "D":
x.data['Type'] = "Downloader"
ualist.append(x)
# run uas parsser
uas_parser = UASparser()
for (i,x) in enumerate(ualist):
print "parsing UA string", i
x = parseua(x,uas_parser)
# result = uas_parser.parse(x.uaString)
# ostype, osversion = osparse(result['os_name'])
#
# if result['typ'] != 'unknown':
# x.data['Type'] = result['typ']
# if result['ua_family'] != 'unknown':
# x.data['Family'] = result['ua_family']
# if ostype != 'unknown':
# x.data['OS'] = ostype
# if osversion != '':
# x.data['OS Version'] = osversion
f = open('xmldata.txt','w')
for x in ualist:
f.write(x + '\n')
f.close()
return 0
def start(self):
"""Stuff that should be done when the server is running as a process,
not just imported as a obj."""
self.external = embedis.embedis()
from uasparser import UASparser
self.logger.info("Loading Browser info")
self.browserdetector = UASparser()
# Start actually logging to file or console
if self.debug:
self.logger.setLevel("DEBUG")
self.logger.addHandler(self.consolehandler)
# logging.getLogger('gnupg').setLevel("DEBUG")
logging.getLogger('gnupg').addHandler(self.consolehandler)
self.logger.addHandler(self.handler_file)
print("Logging Server started at level: " +
str(self.logger.getEffectiveLevel()))
# Pregenerate some users in the background.
self.keygenerator = KeyGenerator.KeyGenerator()
self.keygenerator.start()
if not 'guestacct' in self.serversettings.settings:
self.logger.info("Generating a Guest user acct.")
self.guestacct = User()
self.guestacct.generate(AllowGuestKey=False)
self.serversettings.settings['guestacct'] = {}
self.serversettings.settings['guestacct'][
'pubkey'] = self.guestacct.Keys['master'].pubkey
self.serversettings.saveconfig()
self.guestacct.savemongo()
else:
self.logger.info("Loading the Guest user acct.")
self.guestacct = User()
self.guestacct.load_mongo_by_pubkey(
self.serversettings.settings['guestacct']['pubkey'])
def main():
# start with OS file
f = open('uasOS_example_20101014-01.csv')
ualist = list()
for (i, line) in enumerate(f):
m = re.split(r"\",", line)
m[0] = m[0].replace('"', '') # clean up quotes
m[1] = m[1].replace('"', '')
m[1] = m[1].replace('\n', '')
x = uaobject()
x.uaString = m[1]
# pull in parse from parser on ua-string-info.com
ualist.append(x)
print "number of ua strings from OS file " + str(i)
# now load other file, compare strings as well
f = open('uas_example_20101014-01.csv')
newcnt = 0
for (i, line) in enumerate(f):
m = re.split(r"\",", line)
m[0] = m[0].replace('"', '') # clean up quotes
m[0] = m[0].lower()
m[1] = m[1].replace('"', '')
m[2] = m[2].replace('\n', '')
m[2] = m[2].replace('"', '')
hit = 0
for (j, x) in enumerate(ualist):
if x.uaString == m[2]:
hit = 1
break
if hit == 0:
newcnt = newcnt + 1
x = uaobject()
x.uaString = m[2]
ualist.append(x)
print "number of new ua strings main example file " + str(newcnt)
# now bring in bot csv file THERE ARE NO NEW STRINGS IN BOT FILE
# f = open('botIP.csv')
# newcnt = 0
# for (i,line) in enumerate(f):
# m = re.split(r"\",",line)
# m[0] = m[0].replace('"','')
# m[5] = m[5].replace('"','')
# m[6] = m[6].replace('"','')
# m[6] = m[6].replace('\n','')
# hit = 0
# for (j,x) in enumerate(ualist):
# if x.uaString == m[6]:
# hit = 1
# break
# if hit == 0:
# newcnt = newcnt+1
# x = uaobject()
# x.uaString = m[6]
# ualist.append(x)
# print "number of new UAs in botIP.csv: " + str(newcnt)
uas_parser = UASparser()
for (i, x) in enumerate(ualist):
print "parsing UA string ", i
x = parseua(x, uas_parser)
# result = uas_parser.parse(x.uaString)
# ostype, osversion = osparse(result['os_name'])
# result2 = UserAgent.factory(x.uaString).pretty()
# result2 = UserAgent.parse_pretty(result2)
# version = result2[1:4]
# for (j, k) in enumerate(version):
# if j == 0:
# s = version[0]
# elif k:
# s = ".".join([s, k])
# if s:
# x.data['Family Version'] = s
# if result['typ'] != 'unknown':
# x.data['Type'] = result['typ']
# if result['ua_family'] != 'unknown':
# x.data['Family'] = result['ua_family']
# elif result2[0] != "Other":
# x.data['Family'] = result2[0]
# if ostype != 'unknown':
# x.data['OS'] = ostype
# if osversion != '':
# x.data['OS Version'] = osversion
f = open('csvdata.txt', 'w')
for x in ualist:
f.write(x + '\n')
f.close()
return 0
from uasparser import UASparser
uas_parser = UASparser('/home/natty/FCDDOS/UASparserCache')
userAgents = ["Bimbot/1.0","Mozilla/5.0 (compatible; U; ABrowse 0.6; Syllable) AppleWebKit/420+ (KHTML, like Gecko)","Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30","Bunjalloo/0.7.6(Nintendo DS;U;en)","Wget/1.9+cvs-stable (Red Hat modified)","Mozilla/5.0 (X11; Linux i686; rv:7.0.1) Gecko/20110929 Thunderbird/7.0.1","Mozilla/5.0 (compatible; AbiLogicBot/1.0; +http://www.abilogic.com/bot.html)","EmailSiphon","CSE HTML Validator Lite Online (http://online.htmlvalidator.com/php/onlinevallite.php)","GreatNews/1.0","BinGet/1.00.A (http://www.bin-co.com/php/scripts/load/)","AppEngine-Google; (+http://code.google.com/appengine; appid: unblock4myspace)","AppEngine-Google; (+http://code.google.com/appengine; appid: webetrex)","amaya/11.3.1 libwww/5.4.1"]
userAgents = ['Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x4.90;http://www.Abolimba.de)',
'GetRight/3.2',
'Mozilla/5.0 (X11; U; Linux armv6l; rv: 1.8.1.5pre) Gecko/20070619 Minimo/0.020',
'GcMail Browser/1.0 (compatible; MSIE 5.0; Windows 98) ',
'Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)',
'Klondike/1.50 (HTTP Win32)',
'HTMLParser/1.6',
'Abilon',
'Banshee 1.5.1 (http://banshee-project.org/)',
'ApacheBench/2.3',
'http://Anonymouse.org/ (Unix)',
'Mozilla/5.0 (compatible; WASALive-Bot ; http://blog.wasalive.com/wasalive-bots/)']
for userAgent in userAgents:
result = uas_parser.parse(userAgent)
print result["typ"]
import re
from uasparser import UASparser
from urlparse import urlparse
#set a custom writable cache folder or use the folder where the script locate
uas_parser = UASparser('/tmp')
class entry_processor:
nb_new_request = 0
nb_new_visit = 0
nb_new_visitor = 0
nb_new_linkable = 0
nb_new_useragent = 0
nb_new_referer = 0
nb_skipped = 0
first_request_date = None
last_request_date = None
ua_data = {}
ref_data = {}
visitor_data = {}
request_data = {}
pagename_data = {}
def __init__(self, date, thread, level):
self.date = date
self.thread = thread
self.level = level
self.user_id = -1
import json
from uasparser import UASparser
up = UASparser()
uas_list = []
ua_file = open('user_agents_sample.txt', 'r').read().split('\n')
ua_file += open('user_agents.txt', 'r').read().split('\n')[:10000]
c = 0
for uas in ua_file:
if uas:
c += 1
if c % 1000 == 0:
print c, '/', len(ua_file)
uas_list.append((uas, up.parse(uas)))
json.dump(uas_list, open('uas.json', 'w'))
def __init__(self):
self.uas_parser = UASparser('/home/natty/FCDDOS/UASparserCache')
import json
from uasparser import UASparser
up = UASparser()
uas_list = []
ua_file = open('user_agents_sample.txt', 'r').read().split('\n')
ua_file += open('user_agents.txt', 'r').read().split('\n')[:10000]
c = 0
for uas in ua_file:
if uas:
c += 1
if c % 1000 == 0:
print c, '/', len(ua_file)
uas_list.append((uas, up.parse(uas)))
json.dump(uas_list, open('uas.json', 'w'))
