def uas_parser(logRows=[]): uas_parser = UASparser() total_rows_in_log = len(logRows) #only 'ua_icon' or 'os_icon' or both are allowed in entire_url now = str(date.strftime(datetime.now(), '%d%m%y%H%M')) out = open('ua_' + now + '.log', 'w') for row_log in range(total_rows_in_log): #todo : progress bar #perc=(float(row_log)/float(total_rows_in_log))*100 #progress(perc) sys.stdout.write("\rParsing row log #%s of %s" % (str(row_log), str(total_rows_in_log))) sys.stdout.flush() ua_string = str(logRows[row_log][8]) parsed_ua = uas_parser.parse(ua_string) if (parsed_ua['ua_name'] == "unknown"): out.write("%s\t%s\t%s\t%s\n" % (parsed_ua['typ'], parsed_ua['ua_name'], parsed_ua['os_name'], ua_string)) else: out.write( "%s\t%s\t%s\n" % (parsed_ua['typ'], parsed_ua['ua_name'], parsed_ua['os_name'])) #save in file out.close()
class UserAgentType: def __init__(self): self.uas_parser = UASparser('/home/natty/FCDDOS/UASparserCache') def isBrowser(self,userAgent): if not userAgent or userAgent == None: return True if userAgent == '-': return True #print agentType result = self.uas_parser.parse(str(userAgent)) agentType = result["typ"] #print agentType,":::",userAgent,"\n" if agentType in USER_AGENT_BROWSER_TYPES: return True else: return False def isRobot(self,userAgent): if not userAgent or (userAgent == None): return False if userAgent == "-" or len(userAgent) ==0: return False result = self.uas_parser.parse(str(userAgent)) agentType = result["typ"] #print agentType,":::",userAgent,"\n" if agentType == USER_AGENT_ROBOT: return True else: return False
def main(): uas_parser = UASparser('.') ua = 'SonyEricssonK750i/R1L Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1' print('User Agent: {0}'.format(ua)) # only 'ua_icon' or 'os_icon' or both are allowed in entire_url result = uas_parser.parse(ua, entire_url='ua_icon,os_icon') pprint(result)
def main(): print "www.ua-tracker.org parse script" ualist = [] fr = open("www.ua-tracker.com user_agents.txt",'r') for line in fr: uaString = line.rstrip('\n') uaString = uaString.rstrip() uaString = uaString.lstrip() if uaString != '': x = uaobject() x.uaString = uaString ualist.append(x) fr.close() uas_parser = UASparser() for (i,x) in enumerate(ualist): # if i > 10: # debug # break print x.uaString print "parsing UA string", i x = parseua(x,uas_parser) f = open("uatrack_data.txt",'w') for x in ualist: f.write(x+'\n') f.close() return 0
def uas_parser(logRows=[]): uas_parser = UASparser() total_rows_in_log = len(logRows) #only 'ua_icon' or 'os_icon' or both are allowed in entire_url now = str(date.strftime(datetime.now(), '%d%m%y%H%M')) out = open('ua_' + now + '.log', 'w') for row_log in range(total_rows_in_log): #todo : progress bar #perc=(float(row_log)/float(total_rows_in_log))*100 #progress(perc) sys.stdout.write("\rParsing row log #%s of %s" % (str(row_log), str(total_rows_in_log))) sys.stdout.flush() ua_string = str(logRows[row_log][8]) parsed_ua = uas_parser.parse(ua_string) if(parsed_ua['ua_name'] == "unknown"): out.write("%s\t%s\t%s\t%s\n" % (parsed_ua['typ'], parsed_ua['ua_name'], parsed_ua['os_name'], ua_string)) else: out.write("%s\t%s\t%s\n" % (parsed_ua['typ'], parsed_ua['ua_name'], parsed_ua['os_name'])) #save in file out.close()
def main(): print "XML Parse Script" tree = et.parse("allagents.xml") # these two iterations do the same thing # for elem in tree.getiterator("String"): # print elem.text.encode('utf-8') ualist = [] for elem in tree.getiterator("user-agent"): stringelem = elem.find("String") x = uaobject() x.uaString = stringelem.text.encode('utf-8') typeelem = elem.find("Type") if typeelem.text == "R": x.data['Type'] = "Robot" elif typeelem.text == "B": x.data['Type'] = "Browser" elif typeelem.text == "C": x.data['Type'] = "Validator" elif typeelem.text == "P": x.data['Type'] = "Proxy" elif typeelem.text == "S": x.data['Type'] = "Malicious" elif typeelem.text == "D": x.data['Type'] = "Downloader" ualist.append(x) # run uas parsser uas_parser = UASparser() for (i,x) in enumerate(ualist): print "parsing UA string", i x = parseua(x,uas_parser) # result = uas_parser.parse(x.uaString) # ostype, osversion = osparse(result['os_name']) # # if result['typ'] != 'unknown': # x.data['Type'] = result['typ'] # if result['ua_family'] != 'unknown': # x.data['Family'] = result['ua_family'] # if ostype != 'unknown': # x.data['OS'] = ostype # if osversion != '': # x.data['OS Version'] = osversion f = open('xmldata.txt','w') for x in ualist: f.write(x + '\n') f.close() return 0
def start(self): """Stuff that should be done when the server is running as a process, not just imported as a obj.""" self.external = embedis.embedis() from uasparser import UASparser self.logger.info("Loading Browser info") self.browserdetector = UASparser() # Start actually logging to file or console if self.debug: self.logger.setLevel("DEBUG") self.logger.addHandler(self.consolehandler) # logging.getLogger('gnupg').setLevel("DEBUG") logging.getLogger('gnupg').addHandler(self.consolehandler) self.logger.addHandler(self.handler_file) print("Logging Server started at level: " + str(self.logger.getEffectiveLevel())) # Pregenerate some users in the background. self.keygenerator = KeyGenerator.KeyGenerator() self.keygenerator.start() if not 'guestacct' in self.serversettings.settings: self.logger.info("Generating a Guest user acct.") self.guestacct = User() self.guestacct.generate(AllowGuestKey=False) self.serversettings.settings['guestacct'] = {} self.serversettings.settings['guestacct'][ 'pubkey'] = self.guestacct.Keys['master'].pubkey self.serversettings.saveconfig() self.guestacct.savemongo() else: self.logger.info("Loading the Guest user acct.") self.guestacct = User() self.guestacct.load_mongo_by_pubkey( self.serversettings.settings['guestacct']['pubkey'])
def main(): # start with OS file f = open('uasOS_example_20101014-01.csv') ualist = list() for (i, line) in enumerate(f): m = re.split(r"\",", line) m[0] = m[0].replace('"', '') # clean up quotes m[1] = m[1].replace('"', '') m[1] = m[1].replace('\n', '') x = uaobject() x.uaString = m[1] # pull in parse from parser on ua-string-info.com ualist.append(x) print "number of ua strings from OS file " + str(i) # now load other file, compare strings as well f = open('uas_example_20101014-01.csv') newcnt = 0 for (i, line) in enumerate(f): m = re.split(r"\",", line) m[0] = m[0].replace('"', '') # clean up quotes m[0] = m[0].lower() m[1] = m[1].replace('"', '') m[2] = m[2].replace('\n', '') m[2] = m[2].replace('"', '') hit = 0 for (j, x) in enumerate(ualist): if x.uaString == m[2]: hit = 1 break if hit == 0: newcnt = newcnt + 1 x = uaobject() x.uaString = m[2] ualist.append(x) print "number of new ua strings main example file " + str(newcnt) # now bring in bot csv file THERE ARE NO NEW STRINGS IN BOT FILE # f = open('botIP.csv') # newcnt = 0 # for (i,line) in enumerate(f): # m = re.split(r"\",",line) # m[0] = m[0].replace('"','') # m[5] = m[5].replace('"','') # m[6] = m[6].replace('"','') # m[6] = m[6].replace('\n','') # hit = 0 # for (j,x) in enumerate(ualist): # if x.uaString == m[6]: # hit = 1 # break # if hit == 0: # newcnt = newcnt+1 # x = uaobject() # x.uaString = m[6] # ualist.append(x) # print "number of new UAs in botIP.csv: " + str(newcnt) uas_parser = UASparser() for (i, x) in enumerate(ualist): print "parsing UA string ", i x = parseua(x, uas_parser) # result = uas_parser.parse(x.uaString) # ostype, osversion = osparse(result['os_name']) # result2 = UserAgent.factory(x.uaString).pretty() # result2 = UserAgent.parse_pretty(result2) # version = result2[1:4] # for (j, k) in enumerate(version): # if j == 0: # s = version[0] # elif k: # s = ".".join([s, k]) # if s: # x.data['Family Version'] = s # if result['typ'] != 'unknown': # x.data['Type'] = result['typ'] # if result['ua_family'] != 'unknown': # x.data['Family'] = result['ua_family'] # elif result2[0] != "Other": # x.data['Family'] = result2[0] # if ostype != 'unknown': # x.data['OS'] = ostype # if osversion != '': # x.data['OS Version'] = osversion f = open('csvdata.txt', 'w') for x in ualist: f.write(x + '\n') f.close() return 0
from uasparser import UASparser uas_parser = UASparser('/home/natty/FCDDOS/UASparserCache') userAgents = ["Bimbot/1.0","Mozilla/5.0 (compatible; U; ABrowse 0.6; Syllable) AppleWebKit/420+ (KHTML, like Gecko)","Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30","Bunjalloo/0.7.6(Nintendo DS;U;en)","Wget/1.9+cvs-stable (Red Hat modified)","Mozilla/5.0 (X11; Linux i686; rv:7.0.1) Gecko/20110929 Thunderbird/7.0.1","Mozilla/5.0 (compatible; AbiLogicBot/1.0; +http://www.abilogic.com/bot.html)","EmailSiphon","CSE HTML Validator Lite Online (http://online.htmlvalidator.com/php/onlinevallite.php)","GreatNews/1.0","BinGet/1.00.A (http://www.bin-co.com/php/scripts/load/)","AppEngine-Google; (+http://code.google.com/appengine; appid: unblock4myspace)","AppEngine-Google; (+http://code.google.com/appengine; appid: webetrex)","amaya/11.3.1 libwww/5.4.1"] userAgents = ['Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x4.90;http://www.Abolimba.de)', 'GetRight/3.2', 'Mozilla/5.0 (X11; U; Linux armv6l; rv: 1.8.1.5pre) Gecko/20070619 Minimo/0.020', 'GcMail Browser/1.0 (compatible; MSIE 5.0; Windows 98) ', 'Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)', 'Klondike/1.50 (HTTP Win32)', 'HTMLParser/1.6', 'Abilon', 'Banshee 1.5.1 (http://banshee-project.org/)', 'ApacheBench/2.3', 'http://Anonymouse.org/ (Unix)', 'Mozilla/5.0 (compatible; WASALive-Bot ; http://blog.wasalive.com/wasalive-bots/)'] for userAgent in userAgents: result = uas_parser.parse(userAgent) print result["typ"]
import re from uasparser import UASparser from urlparse import urlparse #set a custom writable cache folder or use the folder where the script locate uas_parser = UASparser('/tmp') class entry_processor: nb_new_request = 0 nb_new_visit = 0 nb_new_visitor = 0 nb_new_linkable = 0 nb_new_useragent = 0 nb_new_referer = 0 nb_skipped = 0 first_request_date = None last_request_date = None ua_data = {} ref_data = {} visitor_data = {} request_data = {} pagename_data = {} def __init__(self, date, thread, level): self.date = date self.thread = thread self.level = level self.user_id = -1
import json from uasparser import UASparser up = UASparser() uas_list = [] ua_file = open('user_agents_sample.txt', 'r').read().split('\n') ua_file += open('user_agents.txt', 'r').read().split('\n')[:10000] c = 0 for uas in ua_file: if uas: c += 1 if c % 1000 == 0: print c, '/', len(ua_file) uas_list.append((uas, up.parse(uas))) json.dump(uas_list, open('uas.json', 'w'))
def __init__(self): self.uas_parser = UASparser('/home/natty/FCDDOS/UASparserCache')