def update_reverse_whois_info(sql): """更新需要反查的whois记录""" DB = DataBase() DB2 = DataBase() DB.db_connect() DB2.db_connect() DB2.execute("""USE malicious_domain_sys""") for result_list in DB.execute_Iterator(sql): for result in result_list: SQL_search_whois_info = """SELECT reg_name,reg_email,reg_phone FROM malicious_domain_sys.whois WHERE ID = {id}""".format( id=result[0]) # print SQL_search_whois_info type_count = 2 DATA = DB2.execute(SQL_search_whois_info) if DATA: for info in DATA[0]: if info: s = str(info) SQL_insert_into_reverse_table = """INSERT IGNORE INTO info_reverse_search SET info = '{i}',info_type = '{t}' """.format( i=s, t=type_count) DB2.execute(SQL_insert_into_reverse_table) # DB2.db_commit() type_count += 1 DB2.db_commit() DB.db_close() DB2.db_close()
def main(Worktype=0): """ 主流程函数,获取whois并更新到数据库中 :param Worktype: 工作方式 0 - 获取新增域名 1 - 重新获取所有域名数据 -1 - 获取失败域名 SQL - 特定获取域名的SQL语句 """ # 全局变量声明 global Process_Num, Ban_Setting global DomainQueue, WhoisQueue # 初始化操作对象 DB = DataBase() DB.db_connect() # 填充域名队列 SQL = """SELECT domain, whois_flag FROM {DB}.{domainTable} """.format( DB=Static.DATABASE_NAME, domainTable=Static.DOMAIN_TABLE) if Worktype == 0: SQL += """WHERE whois_flag = -99 """ log_main.error("新增域名whois获取任务开始...") elif Worktype == 1: pass elif Worktype == -1: Process_Num = Static.PROCESS_NUM_LOW # 修改进程数 Ban_Setting = True # 开启防ban设置 SQL += """WHERE whois_flag < 0 """ log_main.error("定时探测任务开始...") elif type(Worktype) == str: SQL = Worktype Process_Num = Static.PROCESS_NUM_LOW # 修改进程数 Ban_Setting = True # 开启防ban设置 log_main.error("特定探测任务开始... + " + Worktype) else: log_main.error("unexcept arg - main function") return domian_list = DB.execute(SQL) if domian_list is None: log_main.error("未获取到域名..") return for result_list in DB.execute_Iterator(SQL): # 使用<迭代器>减缓数据库IO for result in result_list: domain = result[0] flag = result[1] DomainQueue.put([str(domain).strip(), flag]) log_main.error("域名获取完成 共 " + str(DomainQueue.qsize()) + " 个域名") # 开始多线程获取域名 thread_list = [] for i in range(Process_Num): get_whois_thread = threading.Thread(target=GetWhois) get_whois_thread.setDaemon(True) get_whois_thread.start() thread_list.append(get_whois_thread) # 开始域名字典,更新数据库 sleep(Static.SOCKS_TIMEOUT) # 等待队列填充 update_whois_thread = threading.Thread(target=WriteWhoisInfo(DB)) update_whois_thread.setDaemon(True) update_whois_thread.start() thread_list.append(update_whois_thread) # 挂起进程直到结束 for update_whois_thread in thread_list: update_whois_thread.join() log_main.error("结束") # 清空队列 while not WhoisQueue.empty(): WhoisQueue.get() while not DomainQueue.empty(): DomainQueue.get() DB.db_commit() DB.db_close()