def dict2str(dictionary):
try:
if type(dictionary) == str:
return dictionary
return json.dumps(dictionary)
except TypeError as e:
log.exception("conv dict failed : %s" % e)
def str2dict(string):
try:
if type(string) == dict:
return string
return json.loads(string)
except TypeError as e:
log.exception("conv string failed : %s" % e)
def run(self):
while True:
try:
ack_key = redis.get_key("ack_scan_*")
if len(ack_key):
timeout_data = redis.zrangebyscore(ack_key[0], "-INF", time() - 60 * 5, 0, 1)
if timeout_data:
log.debug("触发ack:%s" % timeout_data[0])
redis.zrem(ack_key[0], timeout_data[0])
redis.put(ack_key[0].replace("ack_scan_", "scan_"), timeout_data[0])
scan_key = redis.get_key("scan_*")
if len(scan_key):
scanning_num = load_setting().get("scanning_num", 5)
if len(multiprocessing.active_children()) < scanning_num:
log.debug("【nmap】指定最大并发进程数%s，当前空闲进程数：%s，当前nmap待检测任务数：%s" % (
scanning_num, scanning_num - len(multiprocessing.active_children()),
redis.qsize(scan_key[0])))
log.debug("子进程数目：%s" % len(multiprocessing.active_children()))
scan_data = redis.get(scan_key[0])
p = ChildProcess(scan_key[0], scan_data)
p.start()
else:
sleep(1)
else:
sleep(1)
break
except Exception as e:
log.exception(e)
sleep(60 * 5)
def run_nmap(scan_key, scan_data):
try:
redis.zadd("ack_" + scan_key, {scan_data: time()})
nm = Nmap()
scan_data_dict = str2dict(scan_data)
ip = scan_data_dict['ip']
port = str(scan_data_dict['port'])
log.info("pid=%s,nmap开始扫描:%s" % (os.getpid(), scan_data))
timeout = int(globals().get("SCAN_TIMEOUT", 300))
if FIND_HOST:
nm.scan(hosts=ip,
arguments='-sV -p%s -T4 --version-intensity 4' % port,
timeout=timeout)
else:
nm.scan(hosts=ip,
arguments='-sV -PS445,22 -p%s -T4 --version-intensity 4' %
port,
timeout=timeout)
nmap_result_list = nm.scan_result()
if nmap_result_list:
mongo = Mongodb(host=MONGO_IP,
port=MONGO_PORT,
username=MONGO_USER,
password=MONGO_PWD)
mongo_scan_result = mongo.conn[MONGO_DB_NAME][
MONGO_RESULT_COLL_NAME]
for nmap_result in nmap_result_list:
nmap_result['port_status'] = 'open'
nmap_result['base_task_id'] = ObjectId(
scan_data_dict['base_task_id'])
nmap_result['create_time'] = datetime.datetime.now().strftime(
'%Y-%m-%d')
nmap_result['ip_port'] = "%s:%s" % (nmap_result['ip'],
str(nmap_result['port']))
if VULSCAN_KEY:
redis.sadd(
VULSCAN_KEY,
json.dumps({
"protocol": nmap_result["service"],
"info_id": 0,
"finger": nmap_result["version_info"],
"type": "portscan",
"port": nmap_result["port"],
"url": nmap_result["ip"]
}))
mongo_scan_result.insert_many(nmap_result_list, ordered=False)
mongo.conn.close()
redis.zrem("ack_" + scan_key, scan_data)
except KeyboardInterrupt as e:
log.exception(scan_data)
except Exception as e:
log.exception(scan_data)
redis.zrem("ack_" + scan_key, scan_data)
def save_setting():
setting = {
}
while True:
try:
mongo = Mongodb(host=MONGO_IP, port=MONGO_PORT, username=MONGO_USER, password=MONGO_PWD)
result = mongo.conn[MONGO_DB_NAME][MONGO_SETTING_COLL_NAME].find_one({})
if result:
setting_path = get_setting_path()
for key in ["mail_enable", "scanning_num", "email_address", "email_pwd", "email_server", "sender", "send_type"]:
setting[key] = result.get(key, "")
with open(setting_path, "w+") as fp:
fp.write(dict2str(setting))
except Exception as e:
log.exception("save setting failed : %s" % e)
sleep(60)
else:
sleep(1)
else:
sleep(1)
break
except Exception as e:
log.exception(e)
sleep(60 * 5)
if __name__ == "__main__":
# test = Nmap()
# task_process()
for func in [task_process, save_setting, check_heartbeat]:
t1 = threading.Thread(target=func)
t1.setDaemon(True)
t1.start()
log.info("扫描开始")
while True:
try:
p = ParentsProcess()
p.start()
p.join()
except Exception as e:
log.exception(e)
def diff_port():
while True:
try:
check_heartbeat()
mongo = Mongodb(host=MONGO_IP, port=MONGO_PORT, username=MONGO_USER, password=MONGO_PWD)
mongo_task = mongo.conn[MONGO_DB_NAME][MONGO_TASKS_COLL_NAME]
mongo_scan_result = mongo.conn[MONGO_DB_NAME][MONGO_RESULT_COLL_NAME]
cron_task_running = mongo_task.find(
{"task_status": "running"})
if cron_task_running.count():
for t in cron_task_running:
if not redis.get_key("scan_" + str(t["_id"])) and not redis.get_key(
"ack_scan_" + str(t["_id"])):
log.info("扫描结束")
mongo_task.update_one({"_id": t["_id"]},
{"$set": {"task_status": "finish", "end_time": datetime.datetime.now()}})
task_names = mongo_task.aggregate(
[{"$match": {"task_type": "loop", "task_status": "finish", "diff_result.diff": 0}},
{"$group": {"_id": "$name"}}])
for task_name in task_names:
tasks = mongo_task.find(
{"name": task_name['_id'], "task_status": "finish"}
).sort([('create_time', -1)]).limit(3)
if tasks.count() > 2:
new_task = tasks[0]
lasttime_task_0 = tasks[1]
lasttime_task_1 = tasks[2]
new_ip_ports, old_ip_ports_1, old_ip_ports_0 = set(), set(), set()
new_ips, old_ips_0, old_ips_1 = set(), set(), set()
results = mongo_scan_result.find({"base_task_id": new_task['_id']})
for result in results:
new_ip_ports.add("%s:%s" % (result['ip'], result['port']))
new_ips.add(result['ip'])
results = mongo_scan_result.find({"base_task_id": lasttime_task_0['_id']})
for result in results:
old_ip_ports_0.add("%s:%s" % (result['ip'], result['port']))
old_ips_0.add(result['ip'])
results = mongo_scan_result.find({"base_task_id": lasttime_task_1['_id']})
for result in results:
old_ip_ports_1.add("%s:%s" % (result['ip'], result['port']))
old_ips_1.add(result['ip'])
add_ports = new_ip_ports - old_ip_ports_0 - old_ip_ports_1
add_ips = new_ips - old_ips_0 - old_ips_1
date = strftime('%Y-%m-%d', localtime())
del_ports = old_ip_ports_0 - new_ip_ports
del_ips = old_ips_0 - new_ips
mongo_task.update_one({"_id": new_task['_id']}, {
"$set": {"diff_result.diff": 1, "diff_result.add_ips": list(add_ips),
"diff_result.add_ports": list(add_ports)}})
mongo_task.update_many({"name": task_name['_id'], "diff_result.diff": 0}, {
"$set": {"diff_result.diff": 1}})
setting = load_setting()
if setting["mail_enable"] == "on":
contents = format_html(scan_time=date, add_ips_count=len(add_ips),
add_ports_count=len(add_ports), del_ips_count=len(del_ips),
add_ips=add_ips, add_ports=add_ports, del_ips=del_ips)
send_mail(subject="【%s】端口对比结果" % date, contents=contents, host=setting["email_server"],
use_ssl=True,
sender=setting["sender"], pwd=setting["email_pwd"],
email_address=setting["email_address"])
else:
log.info("任务是第一次扫描")
for task in tasks:
mongo_task.update_one({"_id": task['_id']}, {"$set": {"diff_result.diff": 1}})
except Exception as e:
log.exception(e)
sleep(60)
def task_process():
while True:
try:
mongo = Mongodb(host=MONGO_IP, port=MONGO_PORT, username=MONGO_USER, password=MONGO_PWD)
mongo_task = mongo.conn[MONGO_DB_NAME][MONGO_TASKS_COLL_NAME]
mongo_scan_result = mongo.conn[MONGO_DB_NAME][MONGO_RESULT_COLL_NAME]
cron_task_running = mongo_task.find(
{"task_status": "running"})
if cron_task_running.count():
for t in cron_task_running:
if not redis.get_key("scan_" + str(t["_id"])) and not redis.get_key(
"ack_scan_" + str(t["_id"])):
log.info("扫描结束")
mongo_task.update_one({"_id": t["_id"]},
{"$set": {"task_status": "finish", "end_time": datetime.datetime.now()}})
task_names = mongo_task.aggregate(
[{"$match": {"task_status": "finish",
"$or": [{"$and": [ {"diff_result.diff": 0}]},
{"monitor_result.monitor": 0}]}},
{"$group": {"_id": "$name", "last_doc": {"$last": "$$ROOT"}}}])
for task_name in task_names:
setting = load_setting()
date = strftime('%Y-%m-%d', localtime())
if task_name["last_doc"]["task_type"] == "monitor_task":
ip_port = set()
results = mongo_scan_result.find({"base_task_id": task_name['last_doc']['_id']})
for result in results:
ip_port.add("%s:%s" % (result['ip'], result['port']))
if setting["mail_enable"] == "on":
if len(ip_port) or setting["send_type"]=="always":
mail_contents = format_monitor_html(scan_time=date, ips_count=len(ip_port), ips=ip_port)
if "," in setting["email_address"]:
setting["email_address"] = setting["email_address"].split(",")
send_mail(subject="【%s】【%s】监控端口开放结果" % (date, task_name['_id']), contents=mail_contents,
host=setting["email_server"],
use_ssl=True,
sender=setting["sender"], pwd=setting["email_pwd"],
email_address=setting["email_address"])
else:
log.info("端口开放数为零，不发送邮件")
mongo_task.update_one({"_id": task_name['last_doc']['_id']}, {
"$set": {"monitor_result.monitor": 1, "monitor_result.ip_port": list(ip_port),
}})
elif task_name["last_doc"]["task_type"] in ["diff_task", "loop"]:
tasks = mongo_task.find(
{"name": task_name['_id'], "task_status": "finish"}
).sort([('create_time', -1)]).limit(3)
if tasks.count() > 2:
new_task = tasks[0]
lasttime_task_0 = tasks[1]
lasttime_task_1 = tasks[2]
new_ip_ports, old_ip_ports_1, old_ip_ports_0 = set(), set(), set()
new_ips, old_ips_0, old_ips_1 = set(), set(), set()
results = mongo_scan_result.find({"base_task_id": new_task['_id']})
for result in results:
new_ip_ports.add("%s:%s" % (result['ip'], result['port']))
new_ips.add(result['ip'])
results = mongo_scan_result.find({"base_task_id": lasttime_task_0['_id']})
for result in results:
old_ip_ports_0.add("%s:%s" % (result['ip'], result['port']))
old_ips_0.add(result['ip'])
results = mongo_scan_result.find({"base_task_id": lasttime_task_1['_id']})
for result in results:
old_ip_ports_1.add("%s:%s" % (result['ip'], result['port']))
old_ips_1.add(result['ip'])
add_ports = new_ip_ports - old_ip_ports_0 - old_ip_ports_1
add_ips = new_ips - old_ips_0 - old_ips_1
del_ports = old_ip_ports_0 - new_ip_ports
del_ips = old_ips_0 - new_ips
mongo_task.update_one({"_id": new_task['_id']}, {
"$set": {"diff_result.diff": 1, "diff_result.add_ips": list(add_ips),
"diff_result.add_ports": list(add_ports)}})
mongo_task.update_many({"name": task_name['_id'], "diff_result.diff": 0}, {
"$set": {"diff_result.diff": 1}})
if setting["mail_enable"] == "on":
if len(add_ports) or setting["send_type"]=="always":
contents = format_diff_html(scan_time=date, add_ips_count=len(add_ips),
add_ports_count=len(add_ports), del_ips_count=len(del_ips),
add_ips=add_ips, add_ports=add_ports, del_ips=del_ips)
if "," in setting["email_address"]:
setting["email_address"] = setting["email_address"].split(",")
send_mail(subject="【%s】【%s】端口对比结果" % (date, task_name['_id']), contents=contents,
host=setting["email_server"],
use_ssl=True,
sender=setting["sender"], pwd=setting["email_pwd"],
email_address=setting["email_address"])
else:
log.info("无新增端口，不发送邮件")
else:
log.info("任务是第一次扫描")
for task in tasks:
mongo_task.update_one({"_id": task['_id']}, {"$set": {"diff_result.diff": 1}})
else:
pass
except Exception as e:
log.exception(e)
sleep(60)
