def tokenize_json(json_dict, token_type, token_time):
"""
:param json_dict: Python object to walk
and make a new token for every item
:param token_type: The type for the token to create
:param token_time: The saphireTime for the token to create
:return The number of tokens recognized
"""
recognized = 0
all_strings = []
walk(json_dict, all_strings) # break the dict
all_strings = list(set(all_strings)) # unique-ify
for el in all_strings:
if isinstance(el, bool):
continue # ignore True/False
try:
t = Token(token_type, token_time, ('', unicode(el)))
t.match_and_insert(global_vars.tokens)
recognized += 1
except IgnoredTokenException:
continue
return recognized
def recognize_tokens():
if global_vars.debug:
print termcolor.colored("[+] %d Entries in for token recognition" %
len(global_vars.req_resp),
color='blue')
global_vars.common_headers = []
with open('common_headers.txt') as infile:
for line in infile:
global_vars.common_headers.append(
line.lower().replace('\n', '') if '#' not in line else None)
if global_vars.debug:
print "[+] Read in " + str(len(
global_vars.common_headers)) + ' common headers to ignore'
for e in global_vars.req_resp:
recognized = 0
try:
###### url params
for p in e['request']['queryString']:
t = Token('url', e['saphireTime'], (p['name'], p['value']))
t.match_and_insert(global_vars.tokens)
recognized += 1
except IgnoredTokenException:
pass
except KeyError:
pass
try:
###### cookies
if e['request']['cookies'] != []:
for c in e['request']['cookies']:
t = Token('cookie', e['saphireTime'],
(c['name'], c['value']))
t.match_and_insert(global_vars.tokens)
recognized += 1
else: # check the header
cookie_string = [
h['value'] for h in e['request']['headers']
if h['name'].lower() == 'cookie'
][0]
for c in cookie_string.split('; '):
t = Token('cookie', e['saphireTime'],
(c.split('=')[0], c.split('=')[1]))
t.match_and_insert(global_vars.tokens)
recognized += 1
except IgnoredTokenException:
pass
except KeyError:
pass
except IndexError:
pass
try:
###### form fields
if e['request']['method'] == 'POST':
header_values = [h['value'] for h in e['request']['headers']]
for v in header_values:
if 'application/x-www-form-urlencoded' in v: # TODO what about application/form-multipart
if e['request']['postData']['params'] != []:
for f in e['request']['postData']['params']:
t = Token('form', e['saphireTime'],
(f['name'], f['value']))
t.match_and_insert(global_vars.tokens)
recognized += 1
else:
for f in e['request']['postData']['text'].split(
'&'):
ttuple = (f.split('=')[0],
f.split('=')[1]) if '=' in f else (
'', f)
t = Token('form', e['saphireTime'], ttuple)
t.match_and_insert(global_vars.tokens)
recognized += 1
if 'application/json' in v \
or 'application/x-javascript' in v \
or 'text/javascript' in v:
body = e['request']['postData']['text']
post_json = utils.get_json(body)
e['request'][
'saphireJson'] = post_json # add the full dict, might need later...
count = utils.tokenize_json(post_json, 'json',
e['saphireTime'])
recognized += count
except IgnoredTokenException:
pass
except KeyError:
pass
try:
###### headers
for h in e['request']['headers']:
h['name'] = h['name'].lower()
h['value'] = h['value']
if h['name'] in global_vars.common_headers:
continue
t = Token('req_header', e['saphireTime'],
(h['name'], h['value']))
t.match_and_insert(global_vars.tokens)
recognized += 1
except IgnoredTokenException:
pass
except KeyError:
pass
try:
for h in e['response']['headers']:
h['name'] = h['name'].lower()
h['value'] = h['value']
if h['name'] in global_vars.common_headers:
continue
t = Token('rsp_header', e['saphireTime'],
(h['name'], h['value']))
t.match_and_insert(global_vars.tokens)
recognized += 1
if str(e['response']['status'])[0] == '3':
location = [
h['value'] for h in e['response']['headers']
if h['name'].lower() == 'location'
][0]
t = Token('rsp_header', e['saphireTime'],
('location', location))
t.match_and_insert(global_vars.tokens)
recognized += 1
except IgnoredTokenException:
pass
except KeyError:
pass
except IndexError:
pass
try:
###### json in resp body
header_values = [h['value'] for h in e['response']['headers']]
for v in header_values:
if 'application/json' in v \
or 'application/x-javascript' in v \
or 'text/javascript' in v:
body = e['response']['content']['text']
resp_json = utils.get_json(body)
e['response'][
'saphireJson'] = resp_json # add the full dict, might need later...
count = utils.tokenize_json(resp_json, 'json',
e['saphireTime'])
recognized += count
except KeyError:
pass
try:
###### resp cookies
if e['response']['cookies'] != []:
for c in e['response']['cookies']:
t = Token('set_cookie', e['saphireTime'],
(c['name'], c['value']))
t.match_and_insert(global_vars.tokens)
recognized += 1
else:
cookie_string = [
h['value'] for h in e['response']['headers']
if h['name'].lower() == 'set-cookie'
][0]
for c in cookie_string.split('\n'):
t = Token(
'set_cookie', e['saphireTime'],
(c.split('=')[0], c.split('=')[1].split('; ')[0]))
t.match_and_insert(global_vars.tokens)
recognized += 1
except IgnoredTokenException:
pass
except KeyError:
pass
except IndexError:
pass
try:
###### html input fields
if 'text/html' == e['response']['content'][
'mimeType']: # this also appears on XHTML
html = e['response']['content']['text']
soup = bs4.BeautifulSoup(html, 'html.parser')
for form_input in soup.find_all('input'):
input_type = form_input.attrs['type']
input_name = form_input.attrs['name']
input_id = ''
try: # 'id' scraping optional
input_id = form_input.attrs['id']
except KeyError:
pass
tuple = (input_type, input_name,
input_id) if input_id else (input_type,
input_name)
t = Token('html', e['saphireTime'], tuple)
t.match_and_insert(global_vars.tokens)
recognized += 1
except IgnoredTokenException:
pass
except KeyError:
pass
try:
###### JWTs # discovered from match_and_insert > smart_decode call from the prev. ones
jwt_header = e['request']['saphireJWT']['header']
count = utils.tokenize_json(jwt_header, 'jwt_header',
e['saphireTime'])
recognized += count
except KeyError:
pass
try:
jwt_payload = e['request']['saphireJWT']['payload']
count = utils.tokenize_json(jwt_payload, 'jwt_payload',
e['saphireTime'])
recognized += count
except KeyError:
pass
try:
jwt_header = e['response']['saphireJWT']['header']
count = utils.tokenize_json(jwt_header, 'jwt_header',
e['saphireTime'])
recognized += count
except KeyError:
pass
try:
jwt_payload = e['response']['saphireJWT']['payload']
count = utils.tokenize_json(jwt_payload, 'jwt_payload',
e['saphireTime'])
recognized += count
except KeyError:
pass
if global_vars.debug:
print termcolor.colored(
'[+] Recognized %d tokens in req with saphireTime %0.3f' %
(recognized, e['saphireTime']),
color='green')
if global_vars.debug:
ans = raw_input(
termcolor.colored('Print 10 random tokens?(y/N): ',
color='yellow'))
if ans == 'y':
idx = 0
for i in range(10):
global_vars.tokens[idx % len(global_vars.tokens)].dump()
idx += 87
return
