def gen_new_info(info, resp):
if isinstance(resp, Exception):
info['reason'] = str(resp.args)
info['request'] = 0
info['alive'] = 0
return info
info['reason'] = resp.reason
code = resp.status_code
info['status'] = code
info['request'] = 1
if code == 400 or code >= 500:
info['alive'] = 0
else:
info['alive'] = 1
headers = resp.headers
if settings.enable_banner_identify:
info['banner'] = utils.get_sample_banner(headers)
info['header'] = json.dumps(dict(headers))
history = resp.history
info['history'] = json.dumps(get_jump_urls(history))
text = utils.decode_resp_text(resp)
title = get_html_title(text).strip()
info['title'] = utils.remove_invalid_string(title)
info['response'] = utils.remove_invalid_string(text)
return info
def find_subdomains(domain, data):
subdomains = set()
js_urls = set()
db = Database()
for infos in data:
jump_history = infos.get('history')
req_url = infos.get('url')
subdomains.update(find_in_history(domain, req_url, jump_history))
rsp_html = db.get_resp_by_url(domain, req_url)
if not rsp_html:
logger.log(
'DEBUG',
f'an abnormal response occurred in the request {req_url}')
continue
subdomains.update(find_in_resp(domain, req_url, rsp_html))
js_urls.update(find_js_urls(domain, req_url, rsp_html))
req_data = convert_to_dict(js_urls)
resp_data = request.bulk_request(domain, req_data, ret=True)
while not resp_data.empty():
_, resp = resp_data.get()
if not isinstance(resp, Response):
continue
text = utils.decode_resp_text(resp)
subdomains.update(find_in_resp(domain, resp.url, text))
return subdomains
def find_subdomains(domain, data):
subdomains = set()
js_urls = set()
for infos in data:
jump_history = infos.get('history')
req_url = infos.get('url')
subdomains.update(find_in_history(domain, req_url, jump_history))
rsp_html = infos.get('response')
if not rsp_html:
logger.log(
'DEBUG',
f'an abnormal response occurred in the request {req_url}')
continue
subdomains.update(find_in_resp(domain, req_url, rsp_html))
js_urls.update(find_js_urls(domain, req_url, rsp_html))
resp_data = request.bulk_request(js_urls)
for _, resp in resp_data:
if not isinstance(resp, Response):
continue
text = utils.decode_resp_text(resp)
subdomains.update(find_in_resp(domain, resp.url, text))
return subdomains