def response_dump(resp):
# save request and response raw data to Database
try:
caller_poc_object = inspect.currentframe(
).f_back.f_back.f_back.f_locals['self']
caller_poc_object_scan_info = caller_poc_object.scan_info
caller_poc_object_poc_info = caller_poc_object.poc_info
data = {}
data["task_id"] = caller_poc_object_scan_info.get("TaskId", "")
data["target"] = caller_poc_object_scan_info.get("Target", "")
data["vid"] = caller_poc_object_poc_info.get("poc",
dict()).get("Id", "")
except Exception as e:
# log.warn(e)
return
if not (data["task_id"] and data["target"] and data["vid"]):
return
dump_data = dump.dump_all(resp)
# dump_data has request_prefix=b'< ' and response_prefix=b'> ', use them to split line if necessary
for codec in ["utf-8", "gbk"]:
try:
data["response"] = dump_data.decode(codec)
break
except Exception as e:
continue
if not data.get("response", ""):
data["response"] = str(dump_data)
response = data.pop("response")
db = DBClient()
db.save_response(data, response)
def get_task_result(cls, task_id):
cls.db = DBClient()
results = cls.db.get_result(task_id)
if results:
return results
else:
return False
def __init__(self, fb):
self.fb = fb
self.db = DBClient()
def check_task_exist(cls, task_id):
cls.db = DBClient()
results = cls.db.get_result(task_id)
return True if results else False
class PoCManager:
def __init__(self, fb):
self.fb = fb
self.db = DBClient()
def _load_poc(self, poc_vid):
specify_pocs = {}
poc_names = [name for name in os.listdir(self.fb.poc_setting.dir_name) if name.startswith("vb_") and name.endswith(".py")]
if poc_vid == ["all"]:
for poc_name in poc_names:
vid = "_".join(poc_name.split("_")[:3])
specify_pocs[vid] = poc_name
else:
for vid in poc_vid:
for poc_name in poc_names:
if vid in poc_name:
specify_pocs[vid] = poc_name
break
poc_classes = collections.defaultdict(list)
# poc_classes = {vid1: [name1, class1], vid2: [name2, class2], vid3: [name3, class3]}
for vid, poc in specify_pocs.items():
try:
module_name = "{}.{}".format(self.fb.poc_setting.dir_name, poc)[:-3].split("/")[-1]
__import__(module_name)
tmp = sys.modules[module_name]
poc_classes[vid] = [module_name, getattr(tmp, getattr(tmp, "POC_NAME"))]
except ImportError as e:
log.warn("Failed to import PoC {}. {}".format(poc, e))
continue
except Exception as e:
log.warn("Failed to load PoC {}. {}".format(poc, e))
continue
return poc_classes
def run(self):
"""
:return: result dict
result = {task_id: {
poc_name1: {"poc_info": {}, "scan_info": [{}, {}, {}]},
poc_name2: {"poc_info": {}, "scan_info": [{}, {}, {}]},
poc_name3: "ERROR MESSAGE"
}}
"""
task_id = self.fb.task_id
targets = self.fb.targets
vids = self.fb.vids
mode = self.fb.mode
verbose = self.fb.verbose
if not task_id:
task_flag = False
task_id = str(uuid.uuid4()).replace("-", "")
else:
task_flag = True
start_time = time.time()
log.info("Get task [{task_id}]: targets{targets}, mode[{mode}], verbose[{verbose}], poc_id{vid}".format(
task_id=task_id, targets=targets, mode=mode, verbose=verbose, vid=vids))
if task_flag:
_now = now()[:-4]
self.db.create_task(task_id, _now)
self.db.create_basic(task_id, mode, _now, targets, vids)
poc_classes = self._load_poc(vids)
result_queue = queue.Queue()
task_queue = queue.Queue()
for target in targets:
for poc_class in poc_classes.items():
task_queue.put_nowait([target, poc_class])
# [target, (vid1, [name1, class1])]
task_info = {"TaskId": task_id, "Mode": mode, "Verbose": verbose}
run_poc = [RunPoc(task_queue, task_info, result_queue, self.fb) for i in range(self.fb.poc_setting.thread_num)]
def task():
p = run_poc.pop()
try:
p.start()
finally:
run_poc.append(p)
start_gevent_pool_skip_empty(self.fb.poc_setting.thread_num, task)
results_list = []
results = {task_id: collections.defaultdict(dict)}
while not result_queue.empty():
# tmp = [poc_name, poc.poc_info, poc.scan_info]
tmp = result_queue.get_nowait()
results_list.append(tmp)
results[task_id][tmp[0]] = {"poc_info": {}, "scan_info": []}
for result in results_list:
results[task_id][result[0]]["scan_info"].append(result[2])
if results[task_id][result[0]]["poc_info"]:
continue
results[task_id][result[0]]["poc_info"] = result[1]
stop_time = time.time()
log.info("Finish task [{task_id}](used {time}s): targets{targets}, mode[{mode}], verbose[{verbose}]".format(
task_id=task_id, time=round(stop_time - start_time, 2), targets=targets, mode=mode, verbose=verbose))
if task_flag:
_now = now()[:-4]
self.db.save_result(task_id, results[task_id], _now)
self.db.save_basic(task_id, _now)
else:
return results[task_id]
@classmethod
def get_task_result(cls, task_id):
cls.db = DBClient()
results = cls.db.get_result(task_id)
if results:
return results
else:
return False
@classmethod
def exit_write2db(cls, task_id):
cls.db = DBClient()
cls.db.save_result(task_id, "Process timeout and has terminated.", now()[:-4])
@classmethod
def check_task_exist(cls, task_id):
cls.db = DBClient()
results = cls.db.get_result(task_id)
return True if results else False
def exit_write2db(cls, task_id):
cls.db = DBClient()
cls.db.save_result(task_id, "Process timeout and has terminated.", now()[:-4])
class RunPoc:
def __init__(self, task_queue, task_info, result_queue, fb):
self.task_queue = task_queue
self.result = result_queue
self.task_id = task_info["TaskId"]
self.verbose = task_info["Verbose"]
self.mode = task_info["Mode"]
self.fb = fb
def start(self):
task = self.task_queue.get(block=False)
# [target, (vid1, [name1, class1])]
target = task[0]
poc_vid = task[1][0]
poc_name = task[1][1][0].split(".")[-1]
poc = task[1][1][1]()
poc.scan_info = {
'TaskId': self.task_id,
'Target': target,
'Verbose': self.verbose,
'Error': '',
'Mode': self.mode,
'Success': False,
'Ret': tree(),
"risk_category": poc.scan_info.get('risk_category', '')
}
poc.poc_info["poc"]["Class"] = task[1][1][1].__name__
timeout = Timeout(self.fb.poc_setting.timeout)
timeout.start()
try:
log.info("{} - {} start...".format(poc_vid, target))
poc.run(fb=self.fb)
log.info("{} - {} finish.".format(poc_vid, target))
except Timeout:
poc.scan_info['Error'] = "PoC run timeout."
poc.scan_info['Success'] = False
log.error("{} - {} error: PoC run timeout.".format(
poc_vid, target))
except (requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as e:
poc.scan_info['Error'] = str(e)
poc.scan_info['Success'] = False
log.error("{} - {} error: {}.".format(poc_vid, target, e))
except Exception:
import traceback
err = traceback.format_exc()
poc.scan_info['Error'] = err
poc.scan_info['Success'] = False
log.error("{} - {} error: {}.".format(poc_vid, target, err))
finally:
timeout.cancel()
if not poc.scan_info.get("Success", False):
return
if self.fb.poc_setting.return_resp:
poc.scan_info["req_resp"] = self._get_http_data(poc_vid, target)
self.result.put_nowait([poc_name, poc.poc_info, poc.scan_info])
def _get_http_data(self, vid, target):
self.db = DBClient()
data = {"task_id": self.task_id, "target": target, "vid": vid}
results = self.db.get_response(data)
return results.get("response", "") if results else None
def _get_http_data(self, vid, target):
self.db = DBClient()
data = {"task_id": self.task_id, "target": target, "vid": vid}
results = self.db.get_response(data)
return results.get("response", "") if results else None
import inspect
import urllib3
from lib.core.req import requests
from functools import wraps
from thirdparty.requests_toolbelt.utils import dump
from lib.core.db import DBClient
from lib.log import logger as log
from settings import save_http_dump, req_timeout
urllib3.disable_warnings()
db = DBClient()
PROXIES_DICT = ""
def request_config(func, *args, **kwargs):
pass
def response_dump(resp):
# save request and response raw data to Database
try:
caller_poc_object = inspect.currentframe().f_back.f_back.f_back.f_locals['self']
caller_poc_object_scan_info = caller_poc_object.scan_info
caller_poc_object_poc_info = caller_poc_object.poc_info
data = {}
data["task_id"] = caller_poc_object_scan_info.get("TaskId", "")
data["target"] = caller_poc_object_scan_info.get("Target", "")
data["vid"] = caller_poc_object_poc_info.get("poc", dict()).get("Id", "")
except Exception as e:
# log.warn(e)