def GetDomainWhois(domain): whois_dict = get_domain_whois(domain) DB = DataBase() DB.db_connect() WhoisRecord(DB).Update(whois_dict, -99) DB.db_commit() DB.db_close()
def update_reverse_whois_info(sql): """更新需要反查的whois记录""" DB = DataBase() DB2 = DataBase() DB.db_connect() DB2.db_connect() DB2.execute("""USE malicious_domain_sys""") for result_list in DB.execute_Iterator(sql): for result in result_list: SQL_search_whois_info = """SELECT reg_name,reg_email,reg_phone FROM malicious_domain_sys.whois WHERE ID = {id}""".format( id=result[0]) # print SQL_search_whois_info type_count = 2 DATA = DB2.execute(SQL_search_whois_info) if DATA: for info in DATA[0]: if info: s = str(info) SQL_insert_into_reverse_table = """INSERT IGNORE INTO info_reverse_search SET info = '{i}',info_type = '{t}' """.format( i=s, t=type_count) DB2.execute(SQL_insert_into_reverse_table) # DB2.db_commit() type_count += 1 DB2.db_commit() DB.db_close() DB2.db_close()
def __socks_refresh(self): """代理刷新""" self.sign_refresh = 0 # 刷新标志 log_whois.info('代理IP列表刷新') MySQL = DataBase() MySQL.db_connect() GET_PROXY_SQL = SQL_generate.PROXY_INFO(Static.PROXY_SOCKS_TABLE) self.init_data = MySQL.execute(GET_PROXY_SQL) # 初始数据库读取内容 MySQL.db_close() self.__socks_synthesis() # 合成代理IP表
def __init__(self): """数据初始化""" server_addr_dict = {} MySQL = DataBase() MySQL.db_connect() SQL = SQL_generate.TLD_WHOIS_ADDR_INFO(Static.WHOIS_TLD_TABLE) results = MySQL.execute(SQL) for result in results: key = result[0] if not result[1]: # 删除无记录的whois数据 continue values = result[1].split(',') server_addr_dict.setdefault(key, values) self.server_addr_dict = server_addr_dict MySQL.db_close()
def main(work_type=0): """ 主流程 :param work_type: 工作类型 0:获取全部域名数据 1:获取新数据 -1: 重新探测失败数据 """ # 初始化 global TaskQueue, ResultQueue DB = DataBase() DB.db_connect() init_sql = """USE {DB}""".format(DB=Static.HTTP_DATEBASE) DB.execute_no_return(init_sql) # 获取任务域名 GetDomain(DB, work_type) # 开始获取域名解析情况 thread_list = [] for i in range(Static.HTTP_PROCESS_NUM): get_whois_thread = threading.Thread(target=work) get_whois_thread.setDaemon(True) get_whois_thread.start() thread_list.append(get_whois_thread) # 开始域名字典,更新数据库 sleep(Static.HTTP_TIME_OUT) # 等待队列填充 update_whois_thread = threading.Thread(target=UpdateDate(DB)) update_whois_thread.setDaemon(True) update_whois_thread.start() thread_list.append(update_whois_thread) # 挂起进程直到结束 for update_whois_thread in thread_list: update_whois_thread.join() print "域名解析情况获取结束" # 清空队列 while not TaskQueue.empty(): TaskQueue.get() while not ResultQueue.empty(): ResultQueue.get() DB.db_commit() DB.db_close()
def __whoisAddrDictInit(self): """whois服务器ip字典数据初始化""" DB = DataBase() DB.db_connect() whois_ip_dict = {} GET_WHOIS_ADDR_SQL = SQL_generate.WHOIS_SRV_INFO(Static.SRVIP_TABLE) results = DB.execute(GET_WHOIS_ADDR_SQL) # 将whois服务器ip数据读入内存(字典)中 for result in results: key = result[0] if not result[1]: continue ip_list = result[1].split(',') values = [] if result[2]: port_available_list = list(result[2]) for i, ip in enumerate(ip_list): if port_available_list[i] == '1': values.append(ip) whois_ip_dict.setdefault(key, values) DB.db_close() self.whois_ip_dict = whois_ip_dict
def main(domain_list): """ 主流程函数,获取whois并更新到数据库中 :param Worktype: 工作方式 0 - 获取新增域名 1 - 重新获取所有域名数据 -1 - 获取失败域名 SQL - 特定获取域名的SQL语句 """ Static.init() log_main = Static.LOGGER # 全局变量声明 global whois_result_dict whois_result_dict = [] global DomainQueue, WhoisQueue DomainQueue = Queue.Queue() # 任务域名队列 WhoisQueue = Queue.Queue() # whois结果队列 global Process_Num, Ban_Setting Process_Num = Static.PROCESS_NUM # 获取进程数 Ban_Setting = False # 防Ban设置 # 初始化操作对象 DB = DataBase() DB.db_connect() # 填充域名队列 for domain in domain_list: # print domain DomainQueue.put(str(domain).strip()) # 开始多线程获取域名 thread_list = [] for i in range(Process_Num): get_whois_thread = threading.Thread(target=GetWhois) get_whois_thread.setDaemon(True) get_whois_thread.start() thread_list.append(get_whois_thread) # 开始域名字典,更新数据库 sleep(Static.SOCKS_TIMEOUT) # 等待队列填充 update_whois_thread = threading.Thread(target=WriteWhoisInfo(DB)) update_whois_thread.setDaemon(True) update_whois_thread.start() thread_list.append(update_whois_thread) # 挂起进程直到结束 for update_whois_thread in thread_list: update_whois_thread.join() log_main.error("结束") # 清空队列 while not WhoisQueue.empty(): whois_result_dict.append(WhoisQueue.get()) while not DomainQueue.empty(): DomainQueue.get() DB.db_commit() DB.db_close() return whois_result_dict # if __name__ == '__main__': # print "whois探测开始" # domain_list = ['sina.com'] # result = main(domain_list) # # # print result
return 5 elif domain[0:1] == 'b' or domain[0:1] == 'j' or domain[0:1] == 'o': return 6 elif domain[0:1] == 'p' or domain[0:1] == 'g': return 7 elif domain[0:1] == 'd' or domain[0:1] == 'l' or domain[0:1].isdigit(): return 8 elif domain[0:1] == 'f' or domain[0:1] == 'w' or domain[ 0:1] == 'u' or domain[0:1] == 'y' or domain[0:1] == 'z': return 9 elif domain[0:1] == 'e' or domain[0:1] == 'r' or domain[0:1] == 'k': return 10 else: return 7 if __name__ == "__main__": DB = DataBase() SQL = SQL_generate() DB.db_connect() DB.execute_no_return( """USE {database}""".format(database=str(Static.DATABASE_NAME))) domain_list = get_domain_file() for i in range(len(domain_list)): domain = domain_list[i] domain_table = 'domain_' + str(domain_divide(domain)) sql = SQL.GET_DOMAIN_DIVIDE(domain_table, domain) DB.execute(sql) DB.db_commit() DB.db_close()
def main(Worktype=0): """ 主流程函数,获取whois并更新到数据库中 :param Worktype: 工作方式 0 - 获取新增域名 1 - 重新获取所有域名数据 -1 - 获取失败域名 SQL - 特定获取域名的SQL语句 """ # 全局变量声明 global Process_Num, Ban_Setting global DomainQueue, WhoisQueue # 初始化操作对象 DB = DataBase() DB.db_connect() # 填充域名队列 SQL = """SELECT domain, whois_flag FROM {DB}.{domainTable} """.format( DB=Static.DATABASE_NAME, domainTable=Static.DOMAIN_TABLE) if Worktype == 0: SQL += """WHERE whois_flag = -99 """ log_main.error("新增域名whois获取任务开始...") elif Worktype == 1: pass elif Worktype == -1: Process_Num = Static.PROCESS_NUM_LOW # 修改进程数 Ban_Setting = True # 开启防ban设置 SQL += """WHERE whois_flag < 0 """ log_main.error("定时探测任务开始...") elif type(Worktype) == str: SQL = Worktype Process_Num = Static.PROCESS_NUM_LOW # 修改进程数 Ban_Setting = True # 开启防ban设置 log_main.error("特定探测任务开始... + " + Worktype) else: log_main.error("unexcept arg - main function") return domian_list = DB.execute(SQL) if domian_list is None: log_main.error("未获取到域名..") return for result_list in DB.execute_Iterator(SQL): # 使用<迭代器>减缓数据库IO for result in result_list: domain = result[0] flag = result[1] DomainQueue.put([str(domain).strip(), flag]) log_main.error("域名获取完成 共 " + str(DomainQueue.qsize()) + " 个域名") # 开始多线程获取域名 thread_list = [] for i in range(Process_Num): get_whois_thread = threading.Thread(target=GetWhois) get_whois_thread.setDaemon(True) get_whois_thread.start() thread_list.append(get_whois_thread) # 开始域名字典,更新数据库 sleep(Static.SOCKS_TIMEOUT) # 等待队列填充 update_whois_thread = threading.Thread(target=WriteWhoisInfo(DB)) update_whois_thread.setDaemon(True) update_whois_thread.start() thread_list.append(update_whois_thread) # 挂起进程直到结束 for update_whois_thread in thread_list: update_whois_thread.join() log_main.error("结束") # 清空队列 while not WhoisQueue.empty(): WhoisQueue.get() while not DomainQueue.empty(): DomainQueue.get() DB.db_commit() DB.db_close()