def update_reverse_whois_info(sql):
"""更新需要反查的whois记录"""
DB = DataBase()
DB2 = DataBase()
DB.db_connect()
DB2.db_connect()
DB2.execute("""USE malicious_domain_sys""")
for result_list in DB.execute_Iterator(sql):
for result in result_list:
SQL_search_whois_info = """SELECT reg_name,reg_email,reg_phone FROM malicious_domain_sys.whois WHERE ID = {id}""".format(
id=result[0])
# print SQL_search_whois_info
type_count = 2
DATA = DB2.execute(SQL_search_whois_info)
if DATA:
for info in DATA[0]:
if info:
s = str(info)
SQL_insert_into_reverse_table = """INSERT IGNORE INTO info_reverse_search SET info = '{i}',info_type = '{t}' """.format(
i=s, t=type_count)
DB2.execute(SQL_insert_into_reverse_table)
# DB2.db_commit()
type_count += 1
DB2.db_commit()
DB.db_close()
DB2.db_close()
def __socks_refresh(self):
"""代理刷新"""
self.sign_refresh = 0 # 刷新标志
log_whois.info('代理IP列表刷新')
MySQL = DataBase()
MySQL.db_connect()
GET_PROXY_SQL = SQL_generate.PROXY_INFO(Static.PROXY_SOCKS_TABLE)
self.init_data = MySQL.execute(GET_PROXY_SQL) # 初始数据库读取内容
MySQL.db_close()
self.__socks_synthesis() # 合成代理IP表
def __init__(self):
"""数据初始化"""
server_addr_dict = {}
MySQL = DataBase()
MySQL.db_connect()
SQL = SQL_generate.TLD_WHOIS_ADDR_INFO(Static.WHOIS_TLD_TABLE)
results = MySQL.execute(SQL)
for result in results:
key = result[0]
if not result[1]: # 删除无记录的whois数据
continue
values = result[1].split(',')
server_addr_dict.setdefault(key, values)
self.server_addr_dict = server_addr_dict
MySQL.db_close()
def __whoisAddrDictInit(self):
"""whois服务器ip字典数据初始化"""
DB = DataBase()
DB.db_connect()
whois_ip_dict = {}
GET_WHOIS_ADDR_SQL = SQL_generate.WHOIS_SRV_INFO(Static.SRVIP_TABLE)
results = DB.execute(GET_WHOIS_ADDR_SQL)
# 将whois服务器ip数据读入内存(字典)中
for result in results:
key = result[0]
if not result[1]:
continue
ip_list = result[1].split(',')
values = []
if result[2]:
port_available_list = list(result[2])
for i, ip in enumerate(ip_list):
if port_available_list[i] == '1':
values.append(ip)
whois_ip_dict.setdefault(key, values)
DB.db_close()
self.whois_ip_dict = whois_ip_dict
return 5
elif domain[0:1] == 'b' or domain[0:1] == 'j' or domain[0:1] == 'o':
return 6
elif domain[0:1] == 'p' or domain[0:1] == 'g':
return 7
elif domain[0:1] == 'd' or domain[0:1] == 'l' or domain[0:1].isdigit():
return 8
elif domain[0:1] == 'f' or domain[0:1] == 'w' or domain[
0:1] == 'u' or domain[0:1] == 'y' or domain[0:1] == 'z':
return 9
elif domain[0:1] == 'e' or domain[0:1] == 'r' or domain[0:1] == 'k':
return 10
else:
return 7
if __name__ == "__main__":
DB = DataBase()
SQL = SQL_generate()
DB.db_connect()
DB.execute_no_return(
"""USE {database}""".format(database=str(Static.DATABASE_NAME)))
domain_list = get_domain_file()
for i in range(len(domain_list)):
domain = domain_list[i]
domain_table = 'domain_' + str(domain_divide(domain))
sql = SQL.GET_DOMAIN_DIVIDE(domain_table, domain)
DB.execute(sql)
DB.db_commit()
DB.db_close()
def main(Worktype=0):
"""
主流程函数,获取whois并更新到数据库中
:param Worktype: 工作方式 0 - 获取新增域名
1 - 重新获取所有域名数据
-1 - 获取失败域名
SQL - 特定获取域名的SQL语句
"""
# 全局变量声明
global Process_Num, Ban_Setting
global DomainQueue, WhoisQueue
# 初始化操作对象
DB = DataBase()
DB.db_connect()
# 填充域名队列
SQL = """SELECT domain, whois_flag FROM {DB}.{domainTable} """.format(
DB=Static.DATABASE_NAME, domainTable=Static.DOMAIN_TABLE)
if Worktype == 0:
SQL += """WHERE whois_flag = -99 """
log_main.error("新增域名whois获取任务开始...")
elif Worktype == 1:
pass
elif Worktype == -1:
Process_Num = Static.PROCESS_NUM_LOW # 修改进程数
Ban_Setting = True # 开启防ban设置
SQL += """WHERE whois_flag < 0 """
log_main.error("定时探测任务开始...")
elif type(Worktype) == str:
SQL = Worktype
Process_Num = Static.PROCESS_NUM_LOW # 修改进程数
Ban_Setting = True # 开启防ban设置
log_main.error("特定探测任务开始... + " + Worktype)
else:
log_main.error("unexcept arg - main function")
return
domian_list = DB.execute(SQL)
if domian_list is None:
log_main.error("未获取到域名..")
return
for result_list in DB.execute_Iterator(SQL): # 使用<迭代器>减缓数据库IO
for result in result_list:
domain = result[0]
flag = result[1]
DomainQueue.put([str(domain).strip(), flag])
log_main.error("域名获取完成 共 " + str(DomainQueue.qsize()) + " 个域名")
# 开始多线程获取域名
thread_list = []
for i in range(Process_Num):
get_whois_thread = threading.Thread(target=GetWhois)
get_whois_thread.setDaemon(True)
get_whois_thread.start()
thread_list.append(get_whois_thread)
# 开始域名字典,更新数据库
sleep(Static.SOCKS_TIMEOUT) # 等待队列填充
update_whois_thread = threading.Thread(target=WriteWhoisInfo(DB))
update_whois_thread.setDaemon(True)
update_whois_thread.start()
thread_list.append(update_whois_thread)
# 挂起进程直到结束
for update_whois_thread in thread_list:
update_whois_thread.join()
log_main.error("结束")
# 清空队列
while not WhoisQueue.empty():
WhoisQueue.get()
while not DomainQueue.empty():
DomainQueue.get()
DB.db_commit()
DB.db_close()