def tokenize_json(json_dict, token_type, token_time): """ :param json_dict: Python object to walk and make a new token for every item :param token_type: The type for the token to create :param token_time: The saphireTime for the token to create :return The number of tokens recognized """ recognized = 0 all_strings = [] walk(json_dict, all_strings) # break the dict all_strings = list(set(all_strings)) # unique-ify for el in all_strings: if isinstance(el, bool): continue # ignore True/False try: t = Token(token_type, token_time, ('', unicode(el))) t.match_and_insert(global_vars.tokens) recognized += 1 except IgnoredTokenException: continue return recognized
def recognize_tokens(): if global_vars.debug: print termcolor.colored("[+] %d Entries in for token recognition" % len(global_vars.req_resp), color='blue') global_vars.common_headers = [] with open('common_headers.txt') as infile: for line in infile: global_vars.common_headers.append( line.lower().replace('\n', '') if '#' not in line else None) if global_vars.debug: print "[+] Read in " + str(len( global_vars.common_headers)) + ' common headers to ignore' for e in global_vars.req_resp: recognized = 0 try: ###### url params for p in e['request']['queryString']: t = Token('url', e['saphireTime'], (p['name'], p['value'])) t.match_and_insert(global_vars.tokens) recognized += 1 except IgnoredTokenException: pass except KeyError: pass try: ###### cookies if e['request']['cookies'] != []: for c in e['request']['cookies']: t = Token('cookie', e['saphireTime'], (c['name'], c['value'])) t.match_and_insert(global_vars.tokens) recognized += 1 else: # check the header cookie_string = [ h['value'] for h in e['request']['headers'] if h['name'].lower() == 'cookie' ][0] for c in cookie_string.split('; '): t = Token('cookie', e['saphireTime'], (c.split('=')[0], c.split('=')[1])) t.match_and_insert(global_vars.tokens) recognized += 1 except IgnoredTokenException: pass except KeyError: pass except IndexError: pass try: ###### form fields if e['request']['method'] == 'POST': header_values = [h['value'] for h in e['request']['headers']] for v in header_values: if 'application/x-www-form-urlencoded' in v: # TODO what about application/form-multipart if e['request']['postData']['params'] != []: for f in e['request']['postData']['params']: t = Token('form', e['saphireTime'], (f['name'], f['value'])) t.match_and_insert(global_vars.tokens) recognized += 1 else: for f in e['request']['postData']['text'].split( '&'): ttuple = (f.split('=')[0], f.split('=')[1]) if '=' in f else ( '', f) t = Token('form', e['saphireTime'], ttuple) t.match_and_insert(global_vars.tokens) recognized += 1 if 'application/json' in v \ or 'application/x-javascript' in v \ or 'text/javascript' in v: body = e['request']['postData']['text'] post_json = utils.get_json(body) e['request'][ 'saphireJson'] = post_json # add the full dict, might need later... count = utils.tokenize_json(post_json, 'json', e['saphireTime']) recognized += count except IgnoredTokenException: pass except KeyError: pass try: ###### headers for h in e['request']['headers']: h['name'] = h['name'].lower() h['value'] = h['value'] if h['name'] in global_vars.common_headers: continue t = Token('req_header', e['saphireTime'], (h['name'], h['value'])) t.match_and_insert(global_vars.tokens) recognized += 1 except IgnoredTokenException: pass except KeyError: pass try: for h in e['response']['headers']: h['name'] = h['name'].lower() h['value'] = h['value'] if h['name'] in global_vars.common_headers: continue t = Token('rsp_header', e['saphireTime'], (h['name'], h['value'])) t.match_and_insert(global_vars.tokens) recognized += 1 if str(e['response']['status'])[0] == '3': location = [ h['value'] for h in e['response']['headers'] if h['name'].lower() == 'location' ][0] t = Token('rsp_header', e['saphireTime'], ('location', location)) t.match_and_insert(global_vars.tokens) recognized += 1 except IgnoredTokenException: pass except KeyError: pass except IndexError: pass try: ###### json in resp body header_values = [h['value'] for h in e['response']['headers']] for v in header_values: if 'application/json' in v \ or 'application/x-javascript' in v \ or 'text/javascript' in v: body = e['response']['content']['text'] resp_json = utils.get_json(body) e['response'][ 'saphireJson'] = resp_json # add the full dict, might need later... count = utils.tokenize_json(resp_json, 'json', e['saphireTime']) recognized += count except KeyError: pass try: ###### resp cookies if e['response']['cookies'] != []: for c in e['response']['cookies']: t = Token('set_cookie', e['saphireTime'], (c['name'], c['value'])) t.match_and_insert(global_vars.tokens) recognized += 1 else: cookie_string = [ h['value'] for h in e['response']['headers'] if h['name'].lower() == 'set-cookie' ][0] for c in cookie_string.split('\n'): t = Token( 'set_cookie', e['saphireTime'], (c.split('=')[0], c.split('=')[1].split('; ')[0])) t.match_and_insert(global_vars.tokens) recognized += 1 except IgnoredTokenException: pass except KeyError: pass except IndexError: pass try: ###### html input fields if 'text/html' == e['response']['content'][ 'mimeType']: # this also appears on XHTML html = e['response']['content']['text'] soup = bs4.BeautifulSoup(html, 'html.parser') for form_input in soup.find_all('input'): input_type = form_input.attrs['type'] input_name = form_input.attrs['name'] input_id = '' try: # 'id' scraping optional input_id = form_input.attrs['id'] except KeyError: pass tuple = (input_type, input_name, input_id) if input_id else (input_type, input_name) t = Token('html', e['saphireTime'], tuple) t.match_and_insert(global_vars.tokens) recognized += 1 except IgnoredTokenException: pass except KeyError: pass try: ###### JWTs # discovered from match_and_insert > smart_decode call from the prev. ones jwt_header = e['request']['saphireJWT']['header'] count = utils.tokenize_json(jwt_header, 'jwt_header', e['saphireTime']) recognized += count except KeyError: pass try: jwt_payload = e['request']['saphireJWT']['payload'] count = utils.tokenize_json(jwt_payload, 'jwt_payload', e['saphireTime']) recognized += count except KeyError: pass try: jwt_header = e['response']['saphireJWT']['header'] count = utils.tokenize_json(jwt_header, 'jwt_header', e['saphireTime']) recognized += count except KeyError: pass try: jwt_payload = e['response']['saphireJWT']['payload'] count = utils.tokenize_json(jwt_payload, 'jwt_payload', e['saphireTime']) recognized += count except KeyError: pass if global_vars.debug: print termcolor.colored( '[+] Recognized %d tokens in req with saphireTime %0.3f' % (recognized, e['saphireTime']), color='green') if global_vars.debug: ans = raw_input( termcolor.colored('Print 10 random tokens?(y/N): ', color='yellow')) if ans == 'y': idx = 0 for i in range(10): global_vars.tokens[idx % len(global_vars.tokens)].dump() idx += 87 return