def get(self):
ua = self.request.headers['User-Agent']
if 'ua' in self.request.GET:
ua = self.request.GET['ua']
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.content_type = 'application/json'
self.response.write(json.dumps(woothee.parse(ua)))
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--sql',
action='store',
default=None,
help='the filename of the SQLite DB to create')
parser.add_argument('infile',
nargs='?',
type=argparse.FileType('r'),
default=sys.stdin)
parser.add_argument('outfile',
nargs='?',
type=argparse.FileType('w'),
default=sys.stdout)
args = parser.parse_args(args)
if args.sql:
writer = new_sqlite_writer(args.sql)
else:
writer = new_csv_writer(args.outfile)
while True:
line = args.infile.readline().strip()
if not line:
break
log_entry = json.loads(line.encode('utf-8'))
device_type = log_entry['client']['deviceType']
ua_string = log_entry['clientRequest']['userAgent']
woot = woothee.parse(ua_string)
writer.writerow([
ua_string, woot['os'], woot['os_version'], device_type,
woot['name'], woot['version']
])
def test_testsets(self):
import woothee
check_attributes = ('name', 'category', 'os', 'version', 'vendor',
'os_version')
def _can_assert(attribute, e):
if attribute in ('name', 'category'):
return True
check_attributes = ('os', 'version', 'vendor', 'os_version')
if attribute in check_attributes and attribute in e:
return True
return False
for filename, groupname in TARGETS:
with open(os.path.join(TESTSET_DIR, filename), 'rb') as fp:
for es in yaml.load_all(fp):
for e in es:
r = woothee.parse(e['target'])
for attribute in check_attributes:
testname = groupname + (' test({0}): {1}'.format(
attribute, e['target']))
if _can_assert(attribute, e):
self.assertEqual(r[attribute], e[attribute],
testname)
def test_testsets(self):
import woothee
check_attributes = ('name', 'category', 'os',
'version', 'vendor', 'os_version')
def _can_assert(attribute, e):
if attribute in ('name', 'category'):
return True
check_attributes = ('os', 'version', 'vendor', 'os_version')
if attribute in check_attributes and attribute in e:
return True
return False
for filename, groupname in TARGETS:
with open(os.path.join(TESTSET_DIR, filename), 'rb') as fp:
for es in yaml.load_all(fp):
for e in es:
r = woothee.parse(e['target'])
for attribute in check_attributes:
testname = groupname + (' test({0}): {1}'.format(
attribute, e['target']
))
if _can_assert(attribute, e):
self.assertEqual(
r[attribute],
e[attribute],
testname
)
def define_useragent(useragents, output='score'):
"""Define individual elements of hte useragent.
Useragents should be a list of suspect useragents to evaluate.
Default output will be JSON Blob containing descriptive information.
Set output='score' to receive a threat score for each UA.
If you're calling this from your own application, be sure 'useragents' is a list.
If you're expecting score to be returned, leave output alone, if you want the
features, change output to json.
"""
response_dict = {'results': {}}
allowlist = ('curl', 'mobileasset', 'microsoft ncsi'
) # Always permit these agents
blocklist = ('mozilla/4.0') # Always block these user agents
for agent in useragents:
pua = woothee.parse(agent)
open_count = len([x for x in list(agent) if x in ['(', '[']])
close_count = len([x for x in list(agent) if x in [')', ']']])
response_dict['results'].update({agent: {}})
allow = block = False
if agent.split(' ')[0].lower() in allowlist:
allow = True
elif agent.split(' ')[0].lower() in blocklist:
block = True
response_dict['results'][agent].update(pua)
response_dict['results'][agent].update({'allowlisted': alllow})
response_dict['results'][agent].update({'blocklisted': block})
response_dict['results'][agent].update(
{'tokens': len(agent.split(' '))})
response_dict['results'][agent].update(
{'ngrams': [x for x in pyngram.calc_ngram(agent, 2) if x[1] > 1]})
if open_count != close_count: # unbalanced
response_dict['results'][agent].update({'unbalanced': True})
else:
response_dict['results'][agent].update({'unbalanced': False})
if ';' in agent and '; ' not in agent: # Malformed, should be '; ' between settings
response_dict['results'][agent].update(
{'malformed_semicolon': True})
else:
response_dict['results'][agent].update(
{'malformed_semicolon': False})
if '/' in agent and ' ' in agent and '(' not in agent:
response_dict['results'][agent].update({'malformed_noparen': True})
else:
response_dict['results'][agent].update(
{'malformed_noparen': False})
if '==' in agent or '<' in agent or '>' in agent or '`' in agent: # SQLi/XSS Tactics
response_dict['results'][agent].update(
{'malformed_hacklang': True})
else:
response_dict['results'][agent].update(
{'malformed_hacklang': False})
response_dict['results'][agent].update(
{'length': len(agent)}) # Length is kinda interesting
if output == 'json':
return response_dict
else:
return _score(response_dict)
def register():
#パラメータ取得
agent_cd = request.form['agent_cd']
agent_name = request.form['agent_name']
#パラメータチェック
if not agent_cd or not agent_name:
flash('入力してください。')
return redirect(url_for('login'))
#UserAgent取得
header = request.headers.get('User-Agent')
ua = woothee.parse(header)
category = ua.get('category')
name = ua.get('name')
version = ua.get('version')
os_name = ua.get('os')
vendor = ua.get('vendor')
os_version = ua.get('os_version')
#DB登録処理
user_agent = UserAgent(agent_cd=agent_cd, agent_name=agent_name, category=category, name=name,
version=version, os_name=os_name, vendor=vendor, os_version=os_version)
try:
db_session.add(user_agent)
db_session.commit()
except Exception as e:
db_session.rollback()
flash('既に登録済みです。')
return redirect(url_for('login'))
finally:
db_session.close()
return render_template('register.html')
def test_testsets(self):
import woothee
for filename, groupname in TARGETS:
f = open(os.path.join(TESTSET_DIR, filename), 'rb')
try:
for es in yaml.load_all(f):
for e in es:
r = woothee.parse(e['target'])
for attribute in ('name', 'category', 'os', 'version', 'vendor'):
testname = groupname + (' test(%s): %s' % (attribute, e['target']))
if attribute in ('name', 'category') or \
(attribute in ('os', 'version', 'vendor') and \
attribute in e):
#print r[attribute], e[attribute]
self.assertEqual(r[attribute], e[attribute], testname)
finally:
f.close()
def test_testsets(self):
import woothee
for filename, groupname in TARGETS:
f = open(os.path.join(TESTSET_DIR, filename), 'rb')
try:
for es in yaml.load_all(f):
for e in es:
r = woothee.parse(e['target'])
for attribute in ('name', 'category', 'os', 'version', 'vendor'):
testname = groupname + (' test(%s): %s' % (attribute, e['target']))
if attribute in ('name', 'category') or \
(attribute in ('os', 'version', 'vendor') and \
attribute in e):
#print r[attribute], e[attribute]
self.assertEqual(r[attribute], e[attribute], testname)
finally:
f.close()
def extract_features_from_log(line, source):
if source == 'training_data':
assert len(line) == 4
status=(get_status( line[3].split('\n')[0]))
elif source == 'live_data':
assert len(line) == 3
# Parse IP
ip=(line[0])
j = ip.split('.')
ip_1 = int(j[0])
ip_2 = int(j[1])
ip_3 = int(j[2])
ip_4 = int(j[3])
# Parse User Agent
user_agent=( urllib.unquote(line[1]) )
p = woothee.parse(user_agent.replace('+',' '))
category = get_category ( p['category'].lower())
os_version = get_os_version( p['os_version'].lower() )
vendor = get_vendor( p['vendor'].lower())
name = get_name( p['name'].lower())
os = get_os_family( p['os'].lower())
version = get_version(p['version'].lower())
# Parse Referer
referer=( urllib.unquote(line[2]) )
p = urlparse(referer)
scheme= get_scheme( p.scheme)
hostname = get_hostname( p.hostname)
alexa_top_million = get_alexa_top_million(p.hostname)
len_path = len( p.path)
len_query = len(p.query)
len_host = str(p.hostname).count('.')
if source == 'training_data':
return [ip_1, ip_2, ip_3, ip_4, category, os_version, version, vendor, name, os,
scheme, hostname, alexa_top_million, len_path, len_query, len_host, status]
elif source == 'live_data':
return [ip_1, ip_2, ip_3, ip_4, category, os_version, version, vendor, name, os,
scheme, hostname, alexa_top_million, len_path, len_query, len_host]
def modify_row_td(request):
ua = request.headers.get(u"User-Agent")
ua_parse = woothee.parse(ua)
headers = request.headers
modified = str(request.params.get(u"modified"))
row = json.loads(base64.b64decode(request.params.get(u"data")))
for i in [u"td_path", u"td_referrer", u"td_url"]:
if (row.get(i)):
row[i] = urllib2.unquote(row[i].encode('utf8'))
row.update({
u"time": float(".".join([modified[0:10], modified[10:13]])),
u"bqid": generate_bqid(request.cookies),
u"td_ip": request.remote_addr,
u"td_browser": ua_parse.get(u"name"),
u"td_browser_version": ua_parse.get(u"version"),
u"td_os": ua_parse.get(u"os"),
u"td_os_version": ua_parse.get(u"os_version"),
u"user_agent": ua,
u"x_country": headers.get(u"X-Appengine-Country"),
u"x_region": headers.get(u"X-Appengine-Region"),
u"x_city": headers.get(u"X-Appengine-City"),
u"x_citylatlong": headers.get(u"X-Appengine-Citylatlong"),
})
return row
def modify_row_beacon(request):
ua = request.headers.get(u"User-Agent")
ua_parse = woothee.parse(ua)
headers = request.headers
params = request.params
row = {}
for key in params.keys():
row[key] = params[key]
row.update({
u"time": time.time(),
u"bqid": generate_bqid(request.cookies),
u"referrer": headers.get(u"Referer"),
u"td_ip": request.remote_addr,
u"td_browser": ua_parse.get(u"name"),
u"td_browser_version": ua_parse.get(u"version"),
u"td_os": ua_parse.get(u"os"),
u"td_os_version": ua_parse.get(u"os_version"),
u"user_agent": ua,
u"x_country": headers.get(u"X-Appengine-Country"),
u"x_region": headers.get(u"X-Appengine-Region"),
u"x_city": headers.get(u"X-Appengine-City"),
u"x_citylatlong": headers.get(u"X-Appengine-Citylatlong"),
})
return row
def modify_row_td(request):
ua = request.headers.get(u"User-Agent")
ua_parse = woothee.parse(ua)
headers = request.headers
modified = str(request.params.get(u"modified"))
row = json.loads(base64.b64decode(request.params.get(u"data")))
for i in [u"td_path", u"td_referrer", u"td_url"]:
if ( row.get(i) ):
row[i] = urllib2.unquote(row[i].encode('utf8'))
row.update({
u"time": float(".".join([modified[0:10], modified[10:13]])),
u"bqid": generate_bqid(request.cookies),
u"td_ip": request.remote_addr,
u"td_browser": ua_parse.get(u"name"),
u"td_browser_version": ua_parse.get(u"version"),
u"td_os": ua_parse.get(u"os"),
u"td_os_version": ua_parse.get(u"os_version"),
u"user_agent": ua,
u"x_country": headers.get(u"X-Appengine-Country"),
u"x_region": headers.get(u"X-Appengine-Region"),
u"x_city": headers.get(u"X-Appengine-City"),
u"x_citylatlong": headers.get(u"X-Appengine-Citylatlong"),
})
return row
def modify_row_beacon(request):
ua = request.headers.get(u"User-Agent")
ua_parse = woothee.parse(ua)
headers = request.headers
params = request.params
row = {}
for key in params.keys():
row[key] = params[key]
row.update({
u"time": time.time(),
u"bqid": generate_bqid(request.cookies),
u"referrer": headers.get(u"Referer"),
u"td_ip": request.remote_addr,
u"td_browser": ua_parse.get(u"name"),
u"td_browser_version": ua_parse.get(u"version"),
u"td_os": ua_parse.get(u"os"),
u"td_os_version": ua_parse.get(u"os_version"),
u"user_agent": ua,
u"x_country": headers.get(u"X-Appengine-Country"),
u"x_region": headers.get(u"X-Appengine-Region"),
u"x_city": headers.get(u"X-Appengine-City"),
u"x_citylatlong": headers.get(u"X-Appengine-Citylatlong"),
})
return row
def test_non_provide_testsets(self):
# This test pattern that does not exist in testsets.
# The main purpose is that each logic to pass.
# UserAgent is a dummy that does not exist in the world.
import woothee
# 48 line in lib/woothee/appliance.py
self.assertEqual(
{
"name": "Nintendo DSi",
"version": "UNKNOWN",
"os": "Nintendo DSi",
"os_version": "UNKNOWN",
"category": "appliance",
"vendor": "Nintendo",
}, woothee.parse("(Nintendo DSi; U; ja)"))
# 50 line in lib/woothee/appliance.py
self.assertEqual(
{
"name": "Nintendo Wii",
"version": "UNKNOWN",
"os": "Nintendo Wii",
"os_version": "UNKNOWN",
"category": "appliance",
"vendor": "Nintendo",
}, woothee.parse("(Nintendo Wii; U; ; 3642; ja)"))
# 26 line lib/woothee/browser.py
self.assertEqual(
{
"name": "Internet Explorer",
"version": "11.0",
"os": "Windows Phone OS",
"os_version": "8.1",
"category": "smartphone",
"vendor": "Microsoft",
},
woothee.parse(
"Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0;"
" Touch; IEMobile/11.0; NOKIA; Lumia 930) like Gecko"))
# 159 line lib/woothee/crawler.py
self.assertEqual(
{
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "UNKNOWN",
"os_version": "UNKNOWN",
"category": "UNKNOWN",
"vendor": "UNKNOWN",
}, woothee.parse("Data-Hotel-Cat/1.1"))
# 74-75 line lib/woothee/mobilephone.py
self.assertEqual(
{
"name": "SymbianOS",
"version": "UNKNOWN",
"os": "SymbianOS",
"os_version": "UNKNOWN",
"category": "mobilephone",
"vendor": "UNKNOWN",
}, woothee.parse("SymbianOS/9.2;"))
# 78-80 line lib/woothee/mobilephone.py
self.assertEqual(
{
"name": "Mobile Transcoder",
"version": "Hatena",
"os": "Mobile Transcoder",
"os_version": "UNKNOWN",
"category": "mobilephone",
"vendor": "UNKNOWN",
},
woothee.parse("(compatible; Hatena-Mobile-Gateway/1.2;"
" +http://mgw.hatena.ne.jp/help)"))
# 25-27 line lib/woothee/os.py
self.assertEqual(
{
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "Windows UNKNOWN Ver",
"os_version": "UNKNOWN",
"category": "pc",
"vendor": "UNKNOWN",
},
woothee.parse(
"Mozilla/5.0 (Windows ; rv:8.0) Gecko/20111105 Thunderbird/8.0"
))
# 49 line lib/woothee/os.py
self.assertEqual(
{
"name": "Internet Explorer",
"version": "UNKNOWN",
"os": "Windows NT 4.0",
"os_version": "NT 4.0",
"category": "pc",
"vendor": "Microsoft",
},
woothee.parse(
"Mozilla/4.0 (compatible; MSIE 6.0b; Windows NT 4.0)"))
# 51 line lib/woothee/os.py
self.assertEqual(
{
"name": "Internet Explorer",
"version": "6.0",
"os": "Windows 98",
"os_version": "98",
"category": "pc",
"vendor": "Microsoft",
}, woothee.parse("Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"))
# 53 line lib/woothee/os.py
self.assertEqual(
{
"name": "Internet Explorer",
"version": "5.50",
"os": "Windows 95",
"os_version": "95",
"category": "pc",
"vendor": "Microsoft",
},
woothee.parse(
"Mozilla/4.0 (compatible; MSIE 5.50; Windows 95; SiteKiosk 4.8)"
))
# 121 line lib/woothee/os.py
self.assertEqual(
{
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "iPad",
"os_version": "UNKNOWN",
"category": "smartphone",
"vendor": "UNKNOWN",
}, woothee.parse("Mozilla/5.0 (iPad; "))
# 123 line lib/woothee/os.py
self.assertEqual(
{
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "iPod",
"os_version": "UNKNOWN",
"category": "smartphone",
"vendor": "UNKNOWN",
}, woothee.parse("Mozilla/5.0 (iPod; "))
# 183-185 line lib/woothee/os.py
self.assertEqual(
{
"name": "Mobile Transcoder",
"version": "Naver",
"os": "Mobile Transcoder",
"os_version": "UNKNOWN",
"category": "mobilephone",
"vendor": "UNKNOWN",
}, woothee.parse("Naver Transcoder"))
def is_smartphone(ua):
category = woothee.parse(ua)['category']
return category.endswith('phone')
def output(queue, keu2):
for line in iter(queue.get, sentinel):
if line:
if (line.HTTP_USER_AGENT and len(line.HTTP_USER_AGENT) > 5
and "decode" not in line.HTTP_USER_AGENT):
try:
user_agent = woothee.parse(line.HTTP_USER_AGENT)
line.ua_os = user_agent['os']
line.ua_name = user_agent['name']
line.ua_category = user_agent['category']
except:
line.ua_os = ""
line.ua_name = ""
line.ua_category = ""
else:
line.ua_os = ""
line.ua_name = ""
line.ua_category = ""
if (len(line.REMOTE_ADDR) < 16 and len(line.REMOTE_ADDR) > 3):
try:
line.victime_country = countrydb.country_name_by_addr(
line.REMOTE_ADDR)
except:
line.victime_country = "unknown"
if not line.victime_country:
line.victime_country = "unknown"
try:
city = citydb.record_by_addr(line.REMOTE_ADDR)
except:
line.victime_city = "unknown"
try:
line.asn_victime = asndb.org_by_name(line.REMOTE_ADDR)
except:
line.asn_victime = "unknown"
if not line.asn_victime:
line.asn_victime = "unknown"
else:
line.victime_country = "unknown"
line.asn_victime = "unknown"
if city and "unknown" not in city:
line.victime_city = city['city']
if not line.victime_city:
line.victime_city = "unknown"
else:
line.victime_city = "unknown"
try:
line.website_tld = get_tld(line.HTTP_HOST,
fix_protocol=True,
as_object=True).suffix
except:
line.website_tld = line.HTTP_HOST
keu2.put(line)
keu2.put(sentinel2)
def test_non_provide_testsets(self):
# This test pattern that does not exist in testsets.
# The main purpose is that each logic to pass.
# UserAgent is a dummy that does not exist in the world.
import woothee
# 48 line in lib/woothee/appliance.py
self.assertEqual({
"name": "Nintendo DSi",
"version": "UNKNOWN",
"os": "Nintendo DSi",
"os_version": "UNKNOWN",
"category": "appliance",
"vendor": "Nintendo",
}, woothee.parse("(Nintendo DSi; U; ja)"))
# 50 line in lib/woothee/appliance.py
self.assertEqual({
"name": "Nintendo Wii",
"version": "UNKNOWN",
"os": "Nintendo Wii",
"os_version": "UNKNOWN",
"category": "appliance",
"vendor": "Nintendo",
}, woothee.parse("(Nintendo Wii; U; ; 3642; ja)"))
# 26 line lib/woothee/browser.py
self.assertEqual({
"name": "Internet Explorer",
"version": "11.0",
"os": "Windows Phone OS",
"os_version": "8.1",
"category": "smartphone",
"vendor": "Microsoft",
}, woothee.parse(
"Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0;"
" Touch; IEMobile/11.0; NOKIA; Lumia 930) like Gecko"
))
# 159 line lib/woothee/crawler.py
self.assertEqual({
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "UNKNOWN",
"os_version": "UNKNOWN",
"category": "UNKNOWN",
"vendor": "UNKNOWN",
}, woothee.parse("Data-Hotel-Cat/1.1"))
# 74-75 line lib/woothee/mobilephone.py
self.assertEqual({
"name": "SymbianOS",
"version": "UNKNOWN",
"os": "SymbianOS",
"os_version": "UNKNOWN",
"category": "mobilephone",
"vendor": "UNKNOWN",
}, woothee.parse("SymbianOS/9.2;"))
# 78-80 line lib/woothee/mobilephone.py
self.assertEqual({
"name": "Mobile Transcoder",
"version": "Hatena",
"os": "Mobile Transcoder",
"os_version": "UNKNOWN",
"category": "mobilephone",
"vendor": "UNKNOWN",
}, woothee.parse(
"(compatible; Hatena-Mobile-Gateway/1.2;"
" +http://mgw.hatena.ne.jp/help)"
))
# 25-27 line lib/woothee/os.py
self.assertEqual({
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "Windows UNKNOWN Ver",
"os_version": "UNKNOWN",
"category": "pc",
"vendor": "UNKNOWN",
}, woothee.parse(
"Mozilla/5.0 (Windows ; rv:8.0) Gecko/20111105 Thunderbird/8.0"
))
# 49 line lib/woothee/os.py
self.assertEqual({
"name": "Internet Explorer",
"version": "UNKNOWN",
"os": "Windows NT 4.0",
"os_version": "NT 4.0",
"category": "pc",
"vendor": "Microsoft",
}, woothee.parse(
"Mozilla/4.0 (compatible; MSIE 6.0b; Windows NT 4.0)"
))
# 51 line lib/woothee/os.py
self.assertEqual({
"name": "Internet Explorer",
"version": "6.0",
"os": "Windows 98",
"os_version": "98",
"category": "pc",
"vendor": "Microsoft",
}, woothee.parse(
"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"
))
# 53 line lib/woothee/os.py
self.assertEqual({
"name": "Internet Explorer",
"version": "5.50",
"os": "Windows 95",
"os_version": "95",
"category": "pc",
"vendor": "Microsoft",
}, woothee.parse(
"Mozilla/4.0 (compatible; MSIE 5.50; Windows 95; SiteKiosk 4.8)"
))
# 121 line lib/woothee/os.py
self.assertEqual({
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "iPad",
"os_version": "UNKNOWN",
"category": "smartphone",
"vendor": "UNKNOWN",
}, woothee.parse("Mozilla/5.0 (iPad; "))
# 123 line lib/woothee/os.py
self.assertEqual({
"name": "UNKNOWN",
"version": "UNKNOWN",
"os": "iPod",
"os_version": "UNKNOWN",
"category": "smartphone",
"vendor": "UNKNOWN",
}, woothee.parse("Mozilla/5.0 (iPod; "))
# 183-185 line lib/woothee/os.py
self.assertEqual({
"name": "Mobile Transcoder",
"version": "Naver",
"os": "Mobile Transcoder",
"os_version": "UNKNOWN",
"category": "mobilephone",
"vendor": "UNKNOWN",
}, woothee.parse("Naver Transcoder"))
def is_smartphone(ua):
category = woothee.parse(ua)['category']
return category.endswith('phone')
