def __init__(self, txt_filename=""):
# 将要读取的文本文件路径
self.txt_filename = txt_filename
self.database_name = Config.database_name
self.table_name = Config.domains_table_name
console(__name__, self.database_name, self.table_name)
pass
def run(self):
# 连接本地SQLite数据库
console(__name__, "connecting local SQLite", self.database_local)
self.sqlite_local = Sqldb(self.database_local)
console(__name__, "local SQLite", "connected")
# 连接远程PostgreSQL数据库
console(__name__, "connecting remote PostgreSQL", self.host_remote + ":" + self.port_remote)
self.postgresql_remote = Psqldb(database=self.database_remote, user=self.user_remote,
password=self.password_remote, host=self.host_remote, port=self.port_remote)
console(__name__, "remote PostgreSQL", "connected")
# 同步websites_detail表
self.sync_websites_detail_table()
# 同步domains_list表
self.sync_domains_list_table()
# 同步ports_detail表
self.sync_ports_detail_table()
# 同步ports_list表
self.sync_ports_list_table()
# 同步ips_list表
self.sync_ips_list_table()
# 关闭远端数据库
self.postgresql_remote.close()
# 关闭本地数据库
self.sqlite_local.close()
console(__name__, "synchronization has been completed!")
def fetchone(self, sql, values=()):
result = None
while True:
try:
console(__name__, "fetchone")
cursor = self.conn.cursor()
cursor.execute(sql, values)
result = cursor.fetchone()
break
except psycopg2.OperationalError as e:
logging.exception(e)
str_exception = str(e)
if is_timeout_or_closed_error(str_exception):
# 重新连接数据库
self.re_connect()
else:
break
except Exception as e:
# 其他错误，记录，退出
logging.exception(e)
str_exception = str(e)
if is_timeout_or_closed_error(str_exception):
# 重新连接数据库
self.re_connect()
else:
break
return result
def close(self):
try:
console(__name__, "close")
self.conn.close()
except psycopg2.OperationalError as e:
logging.exception(e)
except Exception as e:
logging.exception(e)
def run(self):
console(__name__, "connecting", self.database_name)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
# 更新数据用
sqldb_update = Sqldb(self.database_name)
console(__name__, "SQLite", "connected")
# 当前行数
row_index = 0
# 总行数
rows_total = self.get_file_rows_count()
# 逐行读取txt文件
with open(self.txt_filename, "r", encoding="utf-8") as f:
while True:
line = f.readline()
if not line:
break
# 解析字符串，写入数据库
dict_tmp = eval(line)
str_org_name = dict_tmp["name"]
for key, value in dict_tmp["domain2ip"].items():
str_domain_name = str(key)
str_ip = str(value)
sql_str = "SELECT domain FROM " + self.table_name + " WHERE domain=? and org_name=? LIMIT 1"
sql_value = (str_domain_name, str_org_name)
result_query = sqldb_query.fetchone(sql_str, sql_value)
# 生成时间字符串
datetime = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
# 如果有相同记录，则更新
if result_query and len(result_query) > 0:
sql_str = "UPDATE " + self.table_name + " SET ips=?, time=? WHERE domain=? and org_name=?"
sql_value = (str_ip, datetime, str_domain_name,
str_org_name)
else:
# 没有相同记录，则新增
sql_str = "INSERT INTO " + self.table_name + " (domain,org_name,ips,time) VALUES (?,?,?,?)"
sql_value = (str_domain_name, str_org_name, str_ip,
datetime)
sqldb_update.execute_non_query(sql_str, sql_value)
console_progress(row_index, rows_total, __name__,
str_org_name, str_domain_name)
sqldb_update.commit()
row_index = 1 + row_index
f.close()
sqldb_update.close()
sqldb_query.close()
def sync_domains_list_table(self):
# 同步domains_list表
data_index = 0
# 查询总行数
sql_str = "SELECT count(domain) FROM " + self.domains_table_name_local + " WHERE synced=0"
result_count = self.sqlite_local.fetchone(sql_str)
# 总行数
if result_count and int(result_count[0]) > 0:
data_total = int(result_count[0])
console(__name__, "synchronizing domains_list table, rows count:", str(data_total))
# 读取本地domains_list表synced=0的数据
sql_str = "SELECT domain,scan_times,time FROM " + self.domains_table_name_local + \
" WHERE synced=0 ORDER BY domain ASC"
result_query_local = self.sqlite_local.fetchall(sql_str)
if result_query_local and len(result_query_local) > 0:
for domain_tmp, scan_times_tmp, time_tmp in result_query_local:
if len(str(scan_times_tmp)) > 0:
# 更新远程domains_list表，synced=-1
update_sql_str_remote = "UPDATE " + self.domains_table_name_remote + \
" SET scan_times=%s, synced=-1," \
" time=%s WHERE domain=%s"
update_sql_value_remote = (scan_times_tmp, time_tmp, domain_tmp)
self.postgresql_remote.execute_non_query(update_sql_str_remote, update_sql_value_remote)
# 当前行
data_index = data_index + 1
# 打印进度和更新的数据
console_progress(data_index, data_total, __name__, domain_tmp, "scan_times: " +
str(scan_times_tmp))
pass
pass
pass
# 提交数据
self.postgresql_remote.commit()
# 更新本地domains_list表，synced=1
sql_str = "UPDATE " + self.domains_table_name_local + " SET synced=1 WHERE synced=0"
self.sqlite_local.execute_non_query(sql_str)
self.sqlite_local.commit()
pass
pass
pass
def sync_ips_list_table(self):
# 同步ips_list表
data_index = 0
# 查询总行数
sql_str = "SELECT count(ip) FROM " + self.ips_table_name_local + " WHERE synced=0"
result_count = self.sqlite_local.fetchone(sql_str)
# 总行数
if result_count and int(result_count[0]) > 0:
data_total = int(result_count[0])
console(__name__, "synchronizing ips_list table, rows count:", str(data_total))
# 读取本地ips_list表synced=0的数据
sql_str = "SELECT ip,list_times,detail_times,time FROM " + self.ips_table_name_local + \
" WHERE synced=0 ORDER BY ip ASC"
result_query_local = self.sqlite_local.fetchall(sql_str)
if result_query_local and len(result_query_local) > 0:
for ip_tmp, list_times_tmp, detail_times_tmp, time_tmp in result_query_local:
if len(str(list_times_tmp)) > 0 and len(str(detail_times_tmp)) > 0:
# 更新远程ips_list表
update_sql_str_remote = "UPDATE " + self.ips_table_name_remote + \
" SET list_times=%s, detail_times=%s, synced=-1," \
" time=%s WHERE ip=%s"
update_sql_value_remote = (list_times_tmp, detail_times_tmp, time_tmp, ip_tmp)
self.postgresql_remote.execute_non_query(update_sql_str_remote, update_sql_value_remote)
# 当前行
data_index = data_index + 1
# 打印进度和更新的数据
console_progress(data_index, data_total, __name__, ip_tmp, "list_times: " +
str(list_times_tmp) + " | detail_times: " + str(detail_times_tmp))
pass
pass
pass
# 提交数据
self.postgresql_remote.commit()
# 更新本地ips_list表
sql_str = "UPDATE " + self.ips_table_name_local + " SET synced=1 WHERE synced=0"
self.sqlite_local.execute_non_query(sql_str)
self.sqlite_local.commit()
pass
pass
pass
def sync_ports_list_table(self):
# 当前行数
data_index = 0
# 查询总行数
sql_str = "SELECT count(ip) FROM " + self.ports_table_name_local + " WHERE synced=0"
result_count = self.sqlite_local.fetchone(sql_str)
# 总行数
if result_count and int(result_count[0]) > 0:
data_total = int(result_count[0])
console(__name__, "synchronizing ports_list table, rows count:", str(data_total))
# 读取本地ports_list表synced=0的数据
sql_str = "SELECT ip,list FROM " + self.ports_table_name_local + \
" WHERE synced=0 GROUP BY ip,list ORDER BY ip ASC"
result_query_local = self.sqlite_local.fetchall(sql_str)
if result_query_local and len(result_query_local) > 0:
for ip_tmp, list_tmp in result_query_local:
if list_tmp and len(list_tmp) > 0:
# 更新远程ports_list表，synced=-1
insert_sql_str_remote = "INSERT INTO " + self.ports_table_name_remote + \
" (ip,list,synced) VALUES (%s,%s,-1)"
insert_sql_value = (ip_tmp, list_tmp)
self.postgresql_remote.execute_non_query(insert_sql_str_remote, insert_sql_value)
# 当前行
data_index = data_index + 1
# 打印进度和插入的数据
console_progress(data_index, data_total, __name__, ip_tmp, list_tmp)
pass
pass
pass
# 提交数据
self.postgresql_remote.commit()
# 更新本地ports_list表
sql_str = "UPDATE " + self.ports_table_name_local + \
" SET synced=1 WHERE synced=0"
self.sqlite_local.execute_non_query(sql_str)
self.sqlite_local.commit()
pass
pass
pass
def sync_websites_detail_table(self):
# 当前行数
data_index = 0
# 查询总行数
sql_str = "SELECT count(domain) FROM " + self.websites_table_name_local + " WHERE synced=0"
result_count = self.sqlite_local.fetchone(sql_str)
# 总行数
if result_count and int(result_count[0]) > 0:
data_total = int(result_count[0])
console(__name__, "synchronizing websites_detail table, rows count:", str(data_total))
# 读取本地websites_detail表synced=0的数据
sql_str = "SELECT domain,detail FROM " + self.websites_table_name_local + \
" WHERE synced=0 GROUP BY domain,detail ORDER BY domain ASC"
result_query_local = self.sqlite_local.fetchall(sql_str)
if result_query_local and len(result_query_local) > 0:
for domain_tmp, detail_tmp in result_query_local:
if detail_tmp and len(detail_tmp) > 0:
# 更新远程websites_detail表，synced=-1
insert_sql_str_remote = "INSERT INTO " + self.websites_table_name_remote + \
" (domain,detail,synced) VALUES (%s,%s,-1)"
insert_sql_value = (domain_tmp, detail_tmp)
self.postgresql_remote.execute_non_query(insert_sql_str_remote, insert_sql_value)
# 当前行
data_index = data_index + 1
# 打印进度和插入的数据
console_progress(data_index, data_total, __name__, domain_tmp, str(len(detail_tmp)))
# 提交数据
self.postgresql_remote.commit()
# 更新本地websites_detail表，synced=0
sql_str = "UPDATE " + self.websites_table_name_local + \
" SET synced=1 WHERE synced=0"
self.sqlite_local.execute_non_query(sql_str)
self.sqlite_local.commit()
pass
pass
pass
def re_connect(self):
# 如果连接数据库超时，则循环再次连接
while True:
try:
console(__name__, "re_connect")
self.conn = psycopg2.connect(database=self.database, user=self.user,
password=self.password, host=self.host, port=self.port)
# 如果连接正常，则退出循环
break
except psycopg2.OperationalError as e:
logging.exception(e)
except Exception as e:
# 其他错误，记录，退出
logging.exception(e)
break
# sleep 5秒后重新连接
console(__name__, "re-connecting PostgreSQL in 5 seconds")
time.sleep(5)
def show_workers_progress(self):
# 假设数据库路径为 sec-blocks-1、2、3/sqlite.db
for i in range(1, 4):
str_db_path = '../sec-blocks-' + str(i) + '/sqlite.db'
str_id = str(i)
# 判断数据库文件是否存在
if os.path.exists(str_db_path):
sqldb_query = Sqldb(str_db_path)
sql_str = "SELECT count(domain) FROM " + Config.domains_table_name + \
" WHERE worker_id=? AND target=1 AND synced=1"
# 假设worker_id=1、2、3
sql_value = (str_id, )
result_count = sqldb_query.fetchone(sql_str, sql_value)
console(__name__, str_db_path, str(result_count[0]))
sqldb_query.close()
else:
console(__name__, str_db_path, "does not exist")
pass
pass
pass
def run(self):
try:
self.scan()
except Exception as e:
str_exception = str(e)
if "Max retries exceeded with url" in str_exception:
# 找到端口号
# HTTPConnectionPool(host='127.0.0.1', port=50501): Max
# retries exceeded with url: http://zap/JSON/core/action/
str_begin = 'HTTPConnectionPool(host=\'127.0.0.1\', port='
str_end = '): Max retries exceeded with url:'
str_port = get_middle_string(str_exception, str_begin, str_end)
if str_port and len(str_port) > 0:
docker_container_name = 'zap' + str_port
# 重启zap的docker
restart_docker_zap(docker_container_name)
# 睡眠60秒，等待容器
console(__name__, 'time.sleep(120)')
time.sleep(120)
# 重启程序
self.run()
else:
logging.exception(e)
console(__name__, 'restart docker failed!', str(e))
pass
else:
# 其他错误打印出来
logging.exception(e)
console(__name__, 'zap dead, again...', str_exception)
pass
pass
def execute_non_query(self, sql, values=()):
result = False
while True:
try:
console(__name__, "execute_non_query")
cursor = self.conn.cursor()
cursor.execute(sql, values)
# self.commit()
result = True
break
except psycopg2.OperationalError as e:
logging.exception(e)
str_exception = str(e)
if is_timeout_or_closed_error(str_exception):
# 重新连接数据库
self.re_connect()
else:
break
except Exception as e:
# 其他错误，记录，退出
logging.exception(e)
str_exception = str(e)
if is_timeout_or_closed_error(str_exception):
console(__name__, "re-connecting PostgreSQL immediately")
# 重新连接数据库
self.re_connect()
else:
break
# sleep 5秒后重新连接
console(__name__, "re-connecting PostgreSQL in 5 seconds")
time.sleep(5)
return result
def batch_scan(self, target_dict_ip_port, arguments="-O -n -sS -sV -T4"):
# 扫描结果
dict_return = {}
# 解析ip
for target_ip_tmp, target_port_tmp in target_dict_ip_port.items():
# 尝试读或扫描结果
dict_scan_result_tmp = None
# 解析此ip下，要扫描的端口，例如 '1-65535'，或'8009'
try:
console(
__name__, "scanning " + target_ip_tmp + " , ports: " +
target_port_tmp, arguments)
dict_scan_result_tmp = self.nm.scan(hosts=target_ip_tmp,
ports=target_port_tmp,
arguments=arguments)
except Exception as e:
# network is unreachable.
if "network is unreachable." == str(e):
# 如果只是无法访问，则忽略
console(__name__, target_ip_tmp, str(e))
pass
else:
# 其他错误打印出来
logging.exception(e)
console(__name__, target_ip_tmp, str(e))
# 扫描结果是open还是close，还是什么
if dict_scan_result_tmp is not None:
# 尝试解析ip和端口号，然后存储入数据库
try:
dict_return.update(dict_scan_result_tmp["scan"])
except Exception as e:
console(__name__, target_ip_tmp, str(e))
logging.exception(e)
pass
return dict_return
def get_websites_detail_job_info(self,
str_job_table_name='websites_detail'):
console(__name__, "connecting", self.database_name)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
console(__name__, self.database_name, "connected")
# 根据程序配置文件中的客户端id和业务类型，查询要处理的任务信息
sql_str = "SELECT name,job_data_count FROM " + self.table_name + " WHERE id=? AND job_table_name=?"
sql_value = (self.id, str_job_table_name)
result_worker = sqldb_query.fetchone(sql_str, sql_value)
# 客户端名称，例如“#1, zap:docker, secblocks:kali, kali:pc”
self.worker_name = result_worker[0]
# 分配给此客户端的任务总数，例如1000个
self.w_job_data_count = int(result_worker[1])
# 关闭数据库
sqldb_query.close()
console(__name__, self.worker_name, str(self.w_job_data_count))
def run(self):
console(__name__, "connecting", self.database_name)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
console(__name__, "SQLite", "connected")
# 当前行数
data_index = 0
# target=是否为扫描目标, list_times=port扫描次数, time=数据更新的日期时间
sql_str = "SELECT count(ip) FROM " + self.ips_table_name + \
" WHERE target=1 AND synced=1 AND org_name LIKE ?"
sql_value = ("%公司", )
result_count = sqldb_query.fetchone(sql_str, sql_value)
# 总行数
data_total = int(result_count[0])
# 关闭数据库
sqldb_query.close()
# 打印总ip数量
console(__name__, "target ip count", str(data_total))
# 每页数据数量
page_size = 20
# 页码
page_index = 0
# 翻页循环的flag
loop_flag = True
mas = MasscanBlock()
# 从ip_all表中，分页读取IP和端口，用masscan扫描
while loop_flag:
# 分页读取，表ips_all里的数据，筛选"公司"的ip
sql_str = "SELECT ip FROM " + self.ips_table_name + \
" WHERE target=1 AND synced=1 AND org_name LIKE ? " \
" ORDER BY list_times ASC, id ASC LIMIT ? OFFSET ?*?"
sql_value = ("%公司", page_size, page_index, page_size)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
result_query = sqldb_query.fetchall(sql_str, sql_value)
if result_query and len(result_query) > 0:
# 组成ip的list，传递给masscan，批量扫描
list_ips = []
for ip_tmp in result_query:
list_ips.append(ip_tmp[0])
pass
# 关闭数据库
sqldb_query.close()
# 获取到ip，传递给masscan扫描1-65535端口
dict_temp = mas.batch_scan(list_ips, "1-65535",
"--max-rate 10000")
# 更新数据用
sqldb_update = Sqldb(self.database_name)
# 生成时间字符串
datetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if dict_temp and len(dict_temp) > 0:
# 扫描结果，insert到表ports_list表的list字段
for key1, value1 in dict_temp.items():
# ips_list存储打开的端口号list[]
list_open_ports = list(value1["tcp"].keys())
insert_sql_str = "INSERT INTO " + self.ports_table_name + \
" (ip,list,synced,time) VALUES (?,?,0,?)"
insert_sql_value = (key1, str(list_open_ports),
datetime)
sqldb_update.execute_non_query(insert_sql_str,
insert_sql_value)
# 打印扫描到的端口
console(__name__, key1, str(list_open_ports))
# 更新扫描次数
for ip_item in list_ips:
# 序号自增
data_index = data_index + 1
# 扫描次数自增+1
# 同步
update_sql_str = "UPDATE " + self.ips_table_name + \
" SET list_times=list_times+1,synced=0,time=? WHERE ip=?"
update_sql_value = (datetime, ip_item)
sqldb_update.execute_non_query(update_sql_str,
update_sql_value)
# 提交数据
sqldb_update.commit()
# 关闭数据库
sqldb_update.close()
# 打印进度
console_progress(data_index, data_total, __name__)
else:
loop_flag = False
# 关闭数据库
sqldb_query.close()
console(__name__, "jobs", "done!")
def scan(self):
self.zap = ZapBlock(api_key=Config.zap_api_key,
local_proxy_ip=Config.zap_proxy_ip,
local_proxy_port=Config.zap_proxy_port)
# 初始化客户端
worker_obj = WebsitesWorker()
worker_obj.get_websites_detail_job_info(self.websites_table_name)
console(__name__, "connecting", self.database_name)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
console(__name__, self.database_name, "connected")
# 当前行数
data_index = 0
# target=是否为扫描目标, list_times=zap扫描次数，synced=是否已经与服务器端postgresql同步
sql_str = "SELECT count(domain) FROM " + self.domains_table_name + \
" WHERE worker_id=? AND target=1 AND synced=1"
sql_value = (worker_obj.id, )
result_count = sqldb_query.fetchone(sql_str, sql_value)
# 总行数
data_total = int(result_count[0])
# 关闭数据库
sqldb_query.close()
# 打印目标domain数量
console(__name__, "target domain count", str(data_total))
# 每页数据数量
page_size = 10
# 页码
page_index = 0
# 翻页循环的flag
loop_flag = True
# 从websites_detail表中，分页读取domain，用zap扫描
# 因为扫描一次的速度较慢，所以这里的分页其实没有什么用处，复制粘贴来的代码，索性不改了
while loop_flag:
# 根据wordker_id，筛选出目标
sql_str = "SELECT domain FROM " + self.domains_table_name + \
" WHERE worker_id=? AND target=1 AND synced=1 " \
" ORDER BY scan_times ASC, id ASC LIMIT ? OFFSET ?*?"
sql_value = (worker_obj.id, page_size, page_index, page_size)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
result_query = sqldb_query.fetchall(sql_str, sql_value)
if result_query and len(result_query) > 0:
# 缓存到list里
domains_list = []
for domain_tmp in result_query:
domains_list.append(domain_tmp[0])
# 关闭数据库
sqldb_query.close()
# domains_list
for domain_item in domains_list:
# 测试用：
# domain_item = 'http://192.168.31.1/'
# 从数据库里读取domains，扫描，html报告的结果存入数据库
str_html = self.zap.single_scan(target_url=domain_item)
# 更新数据用
sqldb_update = Sqldb(self.database_name)
# 生成时间字符串
datetime = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
if str_html and len(str_html) > 0:
# 扫描结果，insert到domains_list表
insert_sql_str = "INSERT INTO " + self.websites_table_name + \
" (domain,detail,synced,time) VALUES (?,?,0,?)"
insert_sql_value = (domain_item, str_html, datetime)
sqldb_update.execute_non_query(insert_sql_str,
insert_sql_value)
# 打印扫描到的端口
console(__name__, domain_item, str(len(str_html)))
# 更新扫描次数
# 序号自增
data_index = data_index + 1
# 扫描次数自增+1
# 同步
update_sql_str = "UPDATE " + self.domains_table_name + \
" SET scan_times=scan_times+1,synced=0,time=? WHERE domain=?"
update_sql_value = (datetime, domain_item)
sqldb_update.execute_non_query(update_sql_str,
update_sql_value)
# 提交数据
sqldb_update.commit()
# 关闭数据库
sqldb_update.close()
# 打印进度
console_progress(data_index, data_total, __name__)
else:
loop_flag = False
# 关闭数据库
sqldb_query.close()
pass
console(__name__, "jobs", "done!")
pass
def run(self):
console(__name__, "connecting", self.database_name)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
console(__name__, "SQLite", "connected")
# 当前行数
data_index = 0
# target=是否为扫描目标
# sql_str = "SELECT count(ip) FROM " + self.ips_table_name + " WHERE target=1 AND org_name LIKE %s"
sql_str = "SELECT COUNT(" + self.ports_table_name + ".ip) FROM " + self.ports_table_name + \
" INNER JOIN " + self.ips_table_name + " ON " + self.ports_table_name + ".ip=" + \
self.ips_table_name + ".ip AND " + self.ips_table_name + ".target=1 AND " + \
self.ips_table_name + ".synced=1 AND " + self.ips_table_name + ".org_name LIKE ?"
sql_value = ("%公司", )
result_count = sqldb_query.fetchone(sql_str, sql_value)
# 总行数
data_total = int(result_count[0])
# 关闭数据库
sqldb_query.close()
# 打印总ip数量
console(__name__, "target ip total", str(data_total))
# 每页数据数量
page_size = 10
# 页码
page_index = 0
# 翻页循环的flag
loop_flag = True
nm = NmapBlock()
# 从ip_all表中，分页读取IP和端口，用nmap扫描
while loop_flag:
# 分页读取
sql_str = "SELECT " + self.ports_table_name + ".ip, " + self.ports_table_name + ".list FROM " + \
self.ports_table_name + " INNER JOIN " + self.ips_table_name + " ON " + \
self.ports_table_name + ".ip=" + self.ips_table_name + ".ip AND " + \
self.ips_table_name + ".target=1 AND " + self.ips_table_name + \
".synced=1 AND " + self.ips_table_name + ".org_name LIKE ? ORDER BY " + \
self.ips_table_name + ".detail_times, " + self.ports_table_name + \
".id LIMIT ? OFFSET ?*? "
# SELECT ports_list.ip, ports_list.list FROM ports_list INNER JOIN ips_list ON
# ports_list.ip=ips_list.ip AND ips_list.target=1 AND ips_list.synced=1 AND
# ips_list.org_name LIKE '%公司' ORDER BY ips_list.detail_times, ports_list.id
sql_value = ("%公司", page_size, page_index, page_size)
# 查询数据用
sqldb_query = Sqldb(self.database_name)
result_query = sqldb_query.fetchall(sql_str, sql_value)
if result_query and len(result_query) > 0:
# 关闭数据库
sqldb_query.close()
for ip_tmp, ports_tmp in result_query:
# open_ports存储打开的端口号list[]
dict_ports_info = {}
# if len(ports_tmp) > 1000:
# # 字符串太长的，开放大量端口的主机，不扫描
# console(__name__, "too many open ports", str(len(ports_tmp)))
# else:
# 获取到ip，传递给nmap扫描端口
# ip_tmp = "192.168.31.1"
dict_temp = nm.single_scan(target_ip=ip_tmp,
target_port=ports_tmp,
arguments="-O -n -sS -sV -T4")
# 更新数据用
sqldb_update = Sqldb(self.database_name)
# 生成时间字符串
datetime = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
if dict_temp and len(dict_temp) > 0:
# 因为只扫描一个ip，所以循环一次即获得全部值
for value1 in dict_temp.values():
dict_ports_info = value1
break
insert_sql_str = "INSERT INTO " + self.detail_table_name + \
" (ip,detail,synced,time) VALUES (?,?,0,?)"
insert_sql_value = (ip_tmp, str(dict_ports_info),
datetime)
sqldb_update.execute_non_query(insert_sql_str,
insert_sql_value)
# 打印扫描到的端口
console(__name__, ip_tmp, str(dict_ports_info))
# 同步 detail_times 端口详细信息的扫描次数
update_sql_str = "UPDATE " + self.ips_table_name + \
" SET detail_times=detail_times+1,synced=0," \
"time=? WHERE ip=?"
update_sql_value = (datetime, ip_tmp)
sqldb_update.execute_non_query(update_sql_str,
update_sql_value)
# 提交数据
sqldb_update.commit()
# 关闭数据库
sqldb_update.close()
# 序号自增
data_index = data_index + 1
# 打印进度
console_progress(data_index, data_total, __name__)
else:
loop_flag = False
# 关闭数据库
sqldb_query.close()
def loop_scan(self, target_dict_ip_port, arguments="--max-rate 5000"):
# 扫描结果
dict_return = {}
# 解析ip
for target_ip_tmp, target_port_tmp in target_dict_ip_port.items():
# 尝试读或扫描结果
dict_scan_result_tmp = None
# 解析此ip下，要扫描的端口，例如 '1-65535'，或'8009'
try:
console(__name__, "scanning " + target_ip_tmp + " : " + target_port_tmp, arguments)
self.mas.scan(target_ip_tmp, target_port_tmp, arguments=arguments)
try:
dict_scan_result_tmp = self.mas.scan_result
except Exception as e:
# Do a scan before trying to get result !
if "Do a scan before trying to get result !" == str(e):
# 如果只是没扫到开放的端口，则忽略
console(__name__, target_ip_tmp, str(e))
pass
else:
# 其他错误打印出来
logging.exception(e)
console(__name__, target_ip_tmp, str(e))
except Exception as e:
# network is unreachable.
if "network is unreachable." == str(e):
# 如果只是无法访问，则忽略
console(__name__, target_ip_tmp, str(e))
pass
else:
# 其他错误打印出来
logging.exception(e)
console(__name__, target_ip_tmp, str(e))
# 扫描结果是open还是close，还是什么
if dict_scan_result_tmp is not None:
# 尝试解析ip和端口号，然后存储入数据库
try:
dict_return.update(dict_scan_result_tmp["scan"])
# for ip_result, item_result in dict_scan_result_tmp["scan"].items():
# dict_return[ip_result] = item_result["tcp"]
except Exception as e:
logging.exception(e)
console(__name__, target_ip_tmp, str(e))
pass
return dict_return
def batch_scan(self, list_target_ip, target_port="1-65535", arguments="--max-rate 5000"):
# 扫描结果
dict_return = {}
# 用换行符 join list，写入临时文件
str_txt_content = "\n".join(list_target_ip)
f_temp = open(self.temp_file, 'w')
f_temp.write(str_txt_content)
f_temp.close()
# 尝试读或扫描结果
dict_scan_result_tmp = None
# 解析此ip下，要扫描的端口，例如 '1-65535'，或'8009'
try:
# str_arguments = ' --includefile {0} {1} '.format("1.txt", arguments)
str_arguments = ' --includefile {0} {1} '.format(self.temp_file, arguments)
console(__name__, "scanning " + str(len(list_target_ip)) + " hosts, ports: " + target_port, str_arguments)
self.mas.scan('', ports=target_port, arguments=str_arguments)
try:
dict_scan_result_tmp = self.mas.scan_result
except Exception as e:
# Do a scan before trying to get result !
if "Do a scan before trying to get result !" == str(e):
# 如果只是没扫到开放的端口，则忽略
console(__name__, "exception", str(e))
pass
else:
# 其他错误打印出来
logging.exception(e)
console(__name__, "error", str(e))
except Exception as e:
# network is unreachable.
if "network is unreachable." == str(e):
# 如果只是无法访问目标主机，则忽略
console(__name__, "exception", str(e))
pass
else:
# 其他错误打印出来
logging.exception(e)
console(__name__, "error", str(e))
# 扫描结果存入dict
if dict_scan_result_tmp is not None:
try:
dict_return.update(dict_scan_result_tmp["scan"])
except Exception as e:
logging.exception(e)
console(__name__, "error", str(e))
pass
pass
if os.path.exists(self.temp_file):
os.remove(self.temp_file)
else:
console(__name__, self.temp_file, "file does not exist")
return dict_return