def count_status():
red = getredis()
while True:
try:
time.sleep(int(scan_set.get("status_flush_time", 30)))
burpdata_undo = red.llen("burpdata")
vuln = red.llen("vuln_all")
data = red.hmget("count_all", "doned", "request", "block_host",
"request_fail")
burpdata_doned, request, block_host, request_fail = list(
map(lambda x: x.decode(), data))
reverse_count = 0
res, resdata = query_reverse("myscan_total")
if res:
reverse_count = int(resdata.get("total"))
logger.warning(
"do/undo:{}/{} req_succ:{}/fail:{} blockhost:{} vuln:{}/reverse:{}"
.format(burpdata_doned, burpdata_undo, request, request_fail,
block_host, vuln, reverse_count),
text="STATUS")
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
except Exception as ex:
logger.warning("Count stat moudle get error:{}".format(ex))
pass
def is_wildcard_dns(domain, istopdomain=False, level=1):
'''
domain: like baidu.com or www.baidu.com
topdomain: True--> domain is baidu.com,False--> domain is www.baidu.com
return :
True:
False:
None: error
'''
if not istopdomain:
domain = ".".join(domain.split(".")[1:])
if domain == "":
return None #
red = getredis()
key = getmd5(domain)
if red.sismember("dns_wildcard_true", key):
return True
if red.sismember("dns_wildcard_false", key):
return False
try:
r = dns.resolver.Resolver(configure=False)
r.nameservers = others.dns_servers
answers = r.query('myscan-not-%s-test.%s' %
(get_random_str(4).lower(), domain))
ips = ', '.join(sorted([answer.address for answer in answers]))
if level == 1:
wildcard_test('any-sub-to.%s' % domain, istopdomain, 2)
elif level == 2:
red.sadd("dns_wildcard_true", key)
return True
except Exception as e:
red.sadd("dns_wildcard_false", key)
return False
def run_python_plugin():
red = getredis()
try:
while True:
try:
workdata = red.lpop("plugin_data_py")
if workdata:
workdata_ = pickle.loads(workdata)
signal.signal(signal.SIGALRM, handler_plugin)
signal.alarm(int(scan_set.get("poc_timeout")))
# workdata_ will like this
# {
# "id": id,
# "poc": poc,
# }
logger.debug("Python plugin get one data")
p = python_plugin(workdata_)
p.run()
signal.alarm(0)
else:
time.sleep(random.uniform(1, 2))
except Exception as ex:
traceback.print_exc()
logger.warning("Run_python scan get error:{}".format(ex))
pass
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
except Exception as ex:
traceback.print_exc()
logger.warning("Run_python main scan get error:{}".format(ex))
def writeresults():
red = getredis()
total_write = 0
if "." not in cmd_line_options.html_output:
cmd_line_options.html_output = cmd_line_options.html_output + ".html"
while True:
try:
results = []
while True:
id = red.lpop("vuln_all_write")
if id:
pickle_data = red.get(id)
if pickle_data:
results.append(pickle.loads(pickle_data))
else:
if results:
for result in results:
total_write += 1
current = int(total_write /
scan_set.get("max_html_output", 10))
outfilename = "{}{}.html".format(
'.'.join(
cmd_line_options.html_output.split(".")
[:-1]), current)
check(outfilename)
out = htmlexport([result], outfilename)
out.save()
results = []
time.sleep(5)
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
except Exception as ex:
traceback.print_exc()
logger.warning(ex)
pass
def count_status():
red = getredis()
while True:
try:
time.sleep(int(scan_set.get("status_flush_time", 30)))
burpdata_undo = red.llen("burpdata")
if scan_set.get("random_test", False):
# workdata = red.spop("work_data_py_set")
unactive = red.scard("work_data_py_set")
else:
# red.lpush("work_data_py", pickledata)
# workdata = red.lpop("work_data_py")
unactive = red.llen("work_data_py")
vuln = red.llen("vuln_all")
data = red.hmget("count_all", "doned", "request", "block_host",
"request_fail", "active")
burpdata_doned, request, block_host, request_fail, active = list(
map(lambda x: x.decode(), data))
reverse_count = 0
res, resdata = query_reverse("myscan_total")
if res:
reverse_count = int(resdata.get("total"))
if cmd_line_options.command == "hostscan":
logger.warning(
"do/undo/active/unactive:{}/{}/{}/{} vuln:{}/reverse:{}".
format(burpdata_doned, burpdata_undo, active, unactive,
vuln, reverse_count),
text="STATUS")
elif cmd_line_options.command == "webscan":
if cmd_line_options.allow_plugin:
undoplugin = red.llen("plugin_data_py")
logger.warning(
"do/undo/active/unactive/undoplugin:{}/{}/{}/{}/{} req_total/fail:{}/{} blockhost:{} vuln:{}/reverse:{}"
.format(burpdata_doned, burpdata_undo, active,
unactive, undoplugin, request, request_fail,
block_host, vuln, reverse_count),
text="STATUS")
else:
logger.warning(
"do/undo/active/unactive:{}/{}/{}/{} req_total/fail:{}/{} blockhost:{} vuln:{}/reverse:{}"
.format(burpdata_doned, burpdata_undo, active,
unactive, request, request_fail, block_host,
vuln, reverse_count),
text="STATUS")
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
except Exception as ex:
logger.warning("Count stat moudle get error:{}".format(ex))
traceback.print_exc()
pass
def find_ip(self):
if is_ipaddr(self.domain):
return self.domain
red = getredis()
key = getmd5("domain_to_ip_" + self.domain)
res = red.get(key)
if res:
return res.decode()
mythread(self.query, copy.deepcopy(others.dns_servers), 6)
data = ",".join(list(self.msg))
red.set(key, data)
return data
def request(**kwargs_sour):
kwargs = copy.deepcopy(kwargs_sour)
red = getredis()
# print("start:",kwargs)
if not kwargs.get("verify", None):
kwargs["verify"] = False
if not kwargs.get("timeout", None):
kwargs["timeout"] = 8
if not kwargs.get("headers", None):
kwargs["headers"] = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0"
}
if cmd_line_options.proxy:
kwargs["proxies"] = cmd_line_options.proxy
if cmd_line_options.timeout:
kwargs["timeout"] = cmd_line_options.timeout
# print("end:",kwargs)
if kwargs.get('data', None):
if isinstance(kwargs.get("data"), str):
kwargs["data"] = kwargs["data"].encode("utf-8", "ignore")
r = None
red.hincrby("count_all", "request", amount=1)
h, p = gethostportfromurl(kwargs.get("url"))
block = block_info(h, p)
# retry
for x in range(cmd_line_options.retry + 1):
try:
r = requests.request(**kwargs)
block.push_result_status(0)
break
except requests.exceptions.ConnectTimeout:
pass
# logger.debug("request connect timeout :{}".format(kwargs["url"]))
except requests.exceptions.ReadTimeout:
pass
# logger.debug("request read timeout :{}".format(kwargs["url"]))
except Exception as ex:
# print(kwargs)
logger.debug("Request error url:{} error:{}".format(
kwargs["url"], ex))
block.push_result_status(1)
sleep(uniform(0, 0.2))
if r != None:
if scan_set.get("search_open", False):
s = searchmsg(r)
s.verify()
s.saveresult()
else:
red.hincrby("count_all", "request_fail", amount=1)
return r
def output(self, msg, insert=False):
msg = "/".join(self.url.split("/")[:3]) + " " + msg
msgmd5 = getmd5(msg)[10:18]
red = getredis()
if insert == False:
if not red.sismember("myscan_max_output", msgmd5):
return True # 可以输出
else:
# logger.debug("sql boolen moudle : {} 输出个数已达{}上限，不再测试输出".format(msg, self.verify_count))
return False # 不可以继续输出
else:
# red.hincrby("myscan_max_output", msgmd5, amount=1)
red.sadd("myscan_max_output", msgmd5)
def set_conn():
try:
redis_conn()
red=getredis()
if not red.ping():
error_msg = "redis ping error . will exit program"
logger.warning(error_msg)
sys.exit()
else:
logger.info("Redis ping success")
except Exception as ex:
error_msg =" connnect redis get error {}:please use --redis pass@host:port:db ,if pass is none ,like --redis @host:port:db".format(ex)
logger.warning(error_msg)
sys.exit()
def __init__(self, dictdata):
self.dictdata = dictdata
self.red = getredis()
# self.red = ""
data = copy.deepcopy(dictdata)
self.url = data.get("url")
self.request = data.get("request")
self.request_bodyoffset = int(self.dictdata.get("request").get("bodyoffset"))
self.response_bodyoffset = int(self.dictdata.get("response").get("bodyoffset"))
self.response = data.get("response")
self.keys = {
"perfile": "doned_perfile",
"perfolder": "doned_perfolder",
"perscheme": "doned_perscheme",
}
def can_output(self, msg, insert=False):
'''
msg : should url+somename
'''
msgmd5 = getmd5(msg)
red = getredis()
if insert == False:
if not red.sismember("myscan_max_output", msgmd5):
return True # 可以输出
else:
logger.debug("{} 输出个数已达一次，不再测试输出".format(msg))
return False # 不可以继续输出
else:
# red.hincrby("myscan_max_output", msgmd5, amount=1)
red.sadd("myscan_max_output", msgmd5)
def start_input():
if cmd_line_options.command == "hostscan":
try:
red = getredis()
if cmd_line_options.input_nmaptext:
datas = get_data_from_textfile(cmd_line_options.input_nmaptext)
logger.info("input {} lines from nmap_text".format(len(datas)))
for data in datas:
red.lpush("hostdata", json.dumps(data))
if cmd_line_options.input_jsonfile:
datas = get_data_from_jsonfile(cmd_line_options.input_jsonfile)
logger.info("input {} lines from nmap_json".format(len(datas)))
for data in datas:
red.lpush("hostdata", json.dumps(data))
except Exception as ex:
traceback.print_exc()
logger.warning("input target to hostdata get error:{}".format(ex))
def cleandb():
# red = redis.StrictRedis(connection_pool=conn.redis)
red=getredis()
if None in red.hmget("count_all", "doned", "request", "block_host","request_fail"):
count_all = {
"block_host": 0, # 被封的host_port
'doned': 0, # 已经做过的burpdata条数
"request": 0, # request 次数
"request_fail": 0, # request fail次数
}
red.hmset("count_all", count_all)
if cmd_line_options.clean:
red.flushall()
count_all = {
"block_host": 0, # 被封的host_port
'doned': 0, # 已经做过的burpdata条数
"request": 0, # request 次数
"request_fail":0, #request fail次数
}
red.hmset("count_all", count_all)
def saveresult(self, result_data, info):
red = getredis()
if not result_data.get("createtime", None):
result_data["createtime"] = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
parsehash = hash(
str(result_data.get("detail").get("parse")) +
result_data.get("url") + result_data.get("name"))
hosthash = "saerch_" + str(
hash(parse.urlparse(result_data.get("url")).netloc.split(":")[0]))
if not red.sismember(hosthash, parsehash):
self.output(info.get("vulmsg"), insert=True)
red.sadd(hosthash, parsehash)
logger.critical(result_data)
random_id = get_random_str(9)
red.set("result_" + random_id, pickle.dumps(result_data))
red.lpush("vuln_" + result_data["name"].replace(" ", "_"),
"result_" + random_id)
red.lpush("vuln_all", "result_" + random_id)
red.lpush("vuln_all_write",
"result_" + random_id) # 保存结果到html,save线程取
def run_python_poc():
red = getredis()
try:
while True:
try:
if scan_set.get("random_test", False):
workdata = red.spop("work_data_py_set")
else:
# red.lpush("work_data_py", pickledata)
workdata = red.lpop("work_data_py")
if workdata:
workdata_ = pickle.loads(workdata)
signal.signal(signal.SIGALRM, handler)
signal.alarm(int(scan_set.get("poc_timeout")))
# workdata_ will like this
# {
# "id": id,
# "data": None, perscheme 为None
# "poc": poc,
# "type": "perscheme"
# }
logger.debug("Python poc get one data, type:" +
workdata_.get("type"))
p = python_poc(workdata_)
p.run()
signal.alarm(0)
else:
time.sleep(random.uniform(1, 2))
except Exception as ex:
traceback.print_exc()
logger.warning("Run_python scan get error:{}".format(ex))
pass
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
except Exception as ex:
traceback.print_exc()
logger.warning("Run_python main scan get error:{}".format(ex))
def run_python():
red = getredis()
try:
while True:
try:
workdata = red.lpop("work_data_py")
if workdata:
workdata = pickle.loads(workdata)
logger.debug("Python poc get one data, type:" +
workdata.get("type"))
p = python_poc(workdata)
p.run()
else:
time.sleep(random.uniform(1, 2))
except Exception as ex:
traceback.print_exc()
logger.warning("Run_python process get error:{}".format(ex))
pass
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
except Exception as ex:
traceback.print_exc()
logger.warning("Run_python main process get error:{}".format(ex))
def is_cdn_domain(domain):
'''
return True ,False
'''
red = getredis()
key = getmd5(domain)
if red.sismember("domain_cdn_true", key):
return True
if red.sismember("domain_cdn_false", key):
return False
try:
r = dns.resolver.Resolver(configure=False)
r.nameservers = others.dns_servers
answers = r.query('myscan-not-%s-test.%s' %
(get_random_str(4).lower(), domain))
ips = ', '.join(sorted([answer.address for answer in answers]))
if level == 1:
wildcard_test('any-sub-to.%s' % domain, istopdomain, 2)
elif level == 2:
red.sadd("dns_wildcard_true", key)
return True
except Exception as e:
red.sadd("dns_wildcard_false", key)
return False
def writeresults():
red = getredis()
while True:
try:
results = []
while True:
id = red.lpop("vuln_all_write")
if id:
pickle_data = red.get(id)
if pickle_data:
results.append(pickle.loads(pickle_data))
else:
if results:
check()
out = htmlexport(results)
out.save()
results = []
time.sleep(5)
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
except Exception as ex:
traceback.print_exc()
logger.warning(ex)
pass
def __init__(self, workdata):
self.workdata = workdata
self.red = getredis()
def __init__(self, dictdata):
self.dictdata = dictdata
self.redis_key = "plugin_extractdomain"
self.red = getredis()
def start():
logger.info("Myscan Python Moudle Listen ...")
red = getredis()
try:
while True:
try:
if cmd_line_options.command == "webscan":
data = red.rpop("burpdata")
if data:
red.hincrby("count_all", "doned", amount=1)
logger.debug("Get one data from burpdata")
dictdata = None
try:
dictdata = json.loads(data)
except Exception as ex:
logger.warning(
"Process burpdata to json get error:" +
str(ex))
continue
if dictdata is not None:
# 把dictdata分配一个id
id = get_random_str(10) + str(get_random_num(5))
toredisdatas = []
# 开启plugin
if cmd_line_options.allow_plugin:
for pluginhash, plugin_info in cmd_line_options.allow_plugin.items(
):
toredisdatas.append(
("plugin_data_py",
pickle.dumps({
"id":
id,
"pochash":
pluginhash,
"poc":
plugin_info.get("poc")
})))
is_filter = dictdata.get("filter")
host = dictdata.get("url").get("host")
port = dictdata.get("url").get("port")
block = block_info(host, port)
if allow_host(host) and not block.is_block():
# 是否启动被动搜索模式
if scan_set.get("search_open", False):
s = searchmsg(dictdata)
s.verify()
# s.saveresult()
data_parser = dictdata_parser(dictdata)
# perfile
if cmd_line_options.pocs_perfile:
if not is_filter or not data_parser.is_perfile_doned(
):
logger.debug(
data_parser.getperfile(
).capitalize() +
" is_perfile_doned res:False")
for poc in cmd_line_options.pocs_perfile:
toredisdatas.append(
("work_data_py",
pickle.dumps({
"id":
id,
"data":
data_parser.getperfile(),
"poc":
poc,
"type":
"perfile"
})))
else:
logger.debug(
data_parser.getperfile(
).capitalize() +
" is_perfile_doned res:True")
# perfolder
if cmd_line_options.pocs_perfoler:
if not is_filter:
folders = data_parser.getperfolders()
else:
folders = data_parser.is_perfolder_doned(
)
if folders != []:
for folder in folders:
for poc in cmd_line_options.pocs_perfoler:
# red.lpush("work_data_py", pickle.dumps({
# "data": folder,
# "dictdata": dictdata,
# "type": "perfolder"
# }))
toredisdatas.append(
("work_data_py",
pickle.dumps({
"id":
id,
"data":
folder,
"poc":
poc,
"type":
"perfolder"
})))
# scheme
if cmd_line_options.pocs_perscheme:
if not is_filter or not data_parser.is_perscheme_doned(
):
logger.debug(
data_parser.getperfile(
).capitalize() +
" is_perscheme_doned res:False")
for poc in cmd_line_options.pocs_perscheme:
toredisdatas.append((
"work_data_py",
pickle.dumps({
"id":
id,
"data":
None, # 这里没有data字段，无关data字段了
"poc":
poc,
"type":
"perscheme"
})))
else:
logger.debug(
data_parser.getperfile(
).capitalize() +
" is_perscheme_doned res:True")
else:
logger.debug("Host block:" + host)
# 分发
if toredisdatas:
# 给id新建一个hash
red.hmset(id, {
'data': data,
'count': len(toredisdatas)
})
for key, pickledata in toredisdatas:
if key == "plugin_data_py":
red.lpush("plugin_data_py", pickledata)
else:
if scan_set.get("random_test", False):
red.sadd("work_data_py_set",
pickledata)
else:
red.lpush("work_data_py",
pickledata)
else:
time.sleep(random.uniform(0, 1))
elif cmd_line_options.command == "hostscan":
data = red.rpop("hostdata")
if data:
red.hincrby("count_all", "doned", amount=1)
logger.debug("Get one data from hostdata")
dictdata = None
try:
dictdata = json.loads(data)
except Exception as ex:
logger.warning(
"Process hostdata to json get error:" +
str(ex))
continue
if dictdata is not None:
# 开启plugin
if cmd_line_options.plugins:
plugin(dictdata)
if "all" in cmd_line_options.disable:
continue
is_filter = dictdata.get("filter")
host = dictdata.get("addr")
port = dictdata.get("port")
block = block_info(host, port)
id = get_random_str(10) + str(get_random_num(5))
if allow_host(host):
toredisdatas = []
if is_filter:
if not block.is_block():
block.block_it()
else:
continue
for poc in cmd_line_options.pocs_perserver:
toredisdatas.append(
pickle.dumps({
"id": id,
"data":
None, # 这里没有data字段，无关data字段了
"poc": poc,
"type": "perserver"
}))
if toredisdatas:
red.hmset(id, {
'data': data,
'count': len(toredisdatas)
})
red.hmset(id, {
'data': data,
'count': len(toredisdatas)
})
for pickledata in toredisdatas:
if scan_set.get("random_test", False):
red.sadd("work_data_py_set",
pickledata)
else:
red.lpush("work_data_py", pickledata)
else:
time.sleep(random.uniform(1, 2))
except Exception as ex:
logger.debug("Run start get error:{}".format(ex))
traceback.print_exc()
continue
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
def __init__(self, dictdata):
self.dictdata = dictdata
self.dict_host_ip = {}
self.hosts = set()
self.red = getredis()
def start():
logger.info("Myscan Python Moudle Listen ...")
red = getredis()
try:
while True:
data = red.lpop("burpdata")
if data:
red.hincrby("count_all", "doned", amount=1)
logger.debug("Get one data from burpdata")
dictdata = ""
try:
dictdata = json.loads(data)
except Exception as ex:
logger.warning("Process burpdata to json get error:" +
str(ex))
continue
if dictdata != "":
# 开启plugin
if cmd_line_options.plugins:
plugin(dictdata)
if "all" in cmd_line_options.disable:
continue
is_filter = dictdata.get("filter")
host = dictdata.get("url").get("host")
port = dictdata.get("url").get("port")
block = block_info(host, port)
if allow_host(host) and not block.is_block():
# 是否启动被动搜索模式
if scan_set.get("search_open", False):
s = searchmsg(dictdata)
s.verify()
s.saveresult()
data_parser = dictdata_parser(dictdata)
# perfile
if cmd_line_options.pocs_perfile:
if not is_filter or not data_parser.is_perfile_doned(
):
logger.debug(
data_parser.getperfile().capitalize() +
" is_perfile_doned res:False")
red.lpush(
"work_data_py",
pickle.dumps({
"data":
data_parser.getperfile(),
"dictdata":
dictdata,
"type":
"perfile"
}))
else:
logger.debug(
data_parser.getperfile().capitalize() +
" is_perfile_doned res:True")
# perfolder
if cmd_line_options.pocs_perfoler:
if not is_filter:
folders = data_parser.getperfolders()
else:
folders = data_parser.is_perfolder_doned()
if folders != []:
for folder in folders:
red.lpush(
"work_data_py",
pickle.dumps({
"data": folder,
"dictdata": dictdata,
"type": "perfolder"
}))
# scheme
if cmd_line_options.pocs_perscheme:
if not is_filter or not data_parser.is_perscheme_doned(
):
logger.debug(
data_parser.getperfile().capitalize() +
" is_perscheme_doned res:False")
red.lpush(
"work_data_py",
pickle.dumps({
"dictdata":
dictdata, # 这里没有data字段，无关data字段了
"type": "perscheme"
}))
else:
logger.debug(
data_parser.getperfile().capitalize() +
" is_perscheme_doned res:True")
else:
logger.debug("Host block:" + host)
else:
time.sleep(random.uniform(1, 2))
except KeyboardInterrupt as ex:
logger.warning("Ctrl+C was pressed ,aborted program")
def __init__(self, host, port):
self.red = getredis()
self.host_port = "{}_{}".format(host, port)
self.count_res_key = "count_res_{}".format(self.host_port) # list
self.block_key = "block" # set
