def parse_log(log_file, token):
requests = get_file_to_array(log_file)
for request in requests:
url_info = request.split(
'======================================================')[1]
request_raw = request.split(
'======================================================')[2]
url_info_parsed = parse_url_info(url_info)
request_info_parsed = parse_request_info(request_raw) # raw to dict
request_parsed = dict(url_info_parsed, **request_info_parsed)
request_parsed['token'] = token
if request_filter(request_parsed, '', black_rules):
insert_result = insert_request(
request_parsed,
request_raw) # Insert Burp requests into Database
if insert_result: # 如果插入/更新成功，就将数据包存入队列，不具备去重
if request_raw.startswith('\r\n'):
request_raw = request_raw.lstrip('\r\n')
if not is_duplicate('results', getRid(
request_raw)): # 去重，如果以及扫描过了，就不再进行扫描(不区分token去重))
data = {
'token': 'parse_burp_log',
'body': request_raw
} # 给parse_log 设置一个专门的token叫parse_burp_log
sendToMQ(data) # 用json序列化字典
def saveRequest(token, request_raw):
request_info = parse_request_service(request_raw)
rid = getRid(request_raw)
request_info['rid'] = rid
request_info['token'] = token
request_info['update_time'] = time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))
return request_info
def insert_request(request, raw):
if raw.startswith('\\r\\n'):
raw = raw.lstrip('\\r\\n')
try:
# feeds = []
# for key, value in request.items():
# if key != 'time':
# feeds.append(escape_content(value))
# feeds_str = ",".join(feeds)
# rid = hashlib.sha256(feeds_str).hexdigest()
rid = getRid(raw)
if not is_duplicate('requests', rid):
now = str(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
table_request = 'requests'
request['rid'] = rid
request['update_time'] = now
request['raw'] = raw
table_response = 'responses'
args_response = {}
args_response['rid'] = rid
args_response['update_time'] = now
flag = 'insert'
else:
now = str(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
table_request = 'requests'
args_request = {}
args_request['update_time'] = now
cons_request = {}
cons_request['rid'] = rid
table_response = 'responses'
args_response = {}
args_response['update_time'] = now
cons_response = {}
cons_response['rid'] = rid
flag = 'update'
if flag == 'insert':
if db_insert(table_request, request) and db_insert(
table_response, args_response):
print highlight(
'[+] {} request rid: {}, url: {}://{}{}'.format(
flag, rid, request['protocol'], request['host'],
request['path']), 'green')
return True
else:
return False
except Exception, e:
print highlight('[!] {}'.format(str(e)), 'red')
return False
def scan():
token = request.query.token
if token:
postdata = request.body.read()
flag1 = request_filter(parse_request_service(postdata), '',
black_rules) #url 去重
flag2 = is_duplicate(
'results',
getRid(postdata)) # results 表去重，如果以及扫描过了，就不再进行扫描(不区分token去重))
flag3 = is_duplicate('requests', getRid(postdata)) # requests 表去重
if flag1 == True:
if flag2 == flag3 == False:
data = {'token': token, 'body': postdata}
sendToMQ(data)
return 'send to burp scan'
else:
return 'fail:repeat scan'
else:
return 'fail: %s' % flag1
def sendToScan(token, body):
url = scan_api % token
try:
res = requests.post(url, data=body,timeout=5).text
if res:
request_info = saveRequest(token,body);
request_info['scan_burp'] = 0
if not is_duplicate('requests',getRid(body)):
mydb.insert('requests', request_info)
return res
else:
pass
except Exception,e:
print e