def get_domain(self): domain_queue = Queue(-1) i = 1 while i <= DOMAIN_SOURCE_TABLE_NUM: for domain in Static.whois_db.get_not_deal_domains( i, Static.FINISHED_TLD): domain_queue.put(domain) i += 1 print Static.get_now_time(), '提取出域名: ', domain_queue.qsize() return domain_queue
def update_not_deal_method(domain, exist): if exist: Static.whois_db.set_whois_flag(domain, -98) else: domain_whois = { 'domain_hash': hash(domain), # 域名哈希值 "domain": domain, # 域名 "flag": -98, # 状态标记 "domain_status": "", # 域名状态 "sponsoring_registrar": "", # 注册商 "top_whois_server": "", # 顶级域名服务器 "sec_whois_server": "", # 二级域名服务器 "reg_name": "", # 注册姓名 "reg_phone": "", # 注册电话 "reg_email": "", # 注册email "org_name": "", # 注册公司名称 "creation_date": "", # 创建时间 "expiration_date": "", # 到期时间 "updated_date": "", # 更新时间 "insert_time": Static.get_now_time(), # 信息插入时间 "details": "", # 细节 "name_server": "", # 域名服务器 "whois_hash": 0, # whois信息哈希值 } Static.whois_db.db_whois_insert(**domain_whois)
def whois_pull(self, ch, method, properties, body): recv_info_from_client = json.loads(body) if type(recv_info_from_client) == unicode: recv_info_from_client = json.loads(recv_info_from_client) # print 'whois pull:', recv_info_from_client['domain'] update_record(recv_info_from_client) ch.basic_ack(delivery_tag=method.delivery_tag) # 返回处理完成确认信息 if self.count_lock.acquire(): self.count += 1 if self.count > Static.COMMIT_NUM: Static.whois_db.db_commit() # commit 操作 print 'Commit: ', Static.get_now_time() self.count = 0 self.count_lock.release()
def update_record(recv_info_from_client): result_flag = recv_info_from_client['result_flag'] func_name = recv_info_from_client['func_name'] exist = recv_info_from_client['exist'] domain_whois = { 'domain_hash': hash(recv_info_from_client['domain']), # 域名哈希值 "domain": recv_info_from_client['domain'], # 域名 "flag": 0, # 状态标记 "domain_status": "", # 域名状态 "sponsoring_registrar": "", # 注册商 "top_whois_server": recv_info_from_client['top_whois_server'], # 顶级域名服务器 "sec_whois_server": recv_info_from_client['sec_whois_server'], # 二级域名服务器 "reg_name": "", # 注册姓名 "reg_phone": "", # 注册电话 "reg_email": "", # 注册email "org_name": "", # 注册公司名称 "creation_date": "", # 创建时间 "expiration_date": "", # 到期时间 "updated_date": "", # 更新时间 "insert_time": Static.get_now_time(), # 信息插入时间 "details": recv_info_from_client['whois_details'], # 细节 "name_server": "", # 域名服务器 "whois_hash": 0, # whois信息哈希值 } domain_whois = get_deal_result(domain_whois, func_name, result_flag) if not domain_whois: return # 第一次跑数据 if exist: # 数据库中存在该记录 if not is_exist(domain_whois.get('domain_status')): # 域名为未存在 Static.whois_db.delete_whois_info(domain_whois['domain']) else: Static.whois_db.update_whois_info(**domain_whois) else: # 数据库中不存在该记录 if is_exist(domain_whois.get('domain_status')): # 域名存在 Static.whois_db.db_whois_insert(**domain_whois) return domain_whois
def main(): print Static.get_now_time(), ' Start' # api_main.open_search_api() # 打开api服务器 Static.whois_db.connect() # 数据库连接 WorkDispatch().open_dispatch() # 开启任务分发 WhoisQueueMonitor().open_monitor() # 开启whois队列监控
def open_search_api(): print Static.get_now_time(), '打开查询api 端口:', Static.WEB_SERVER_PORT app.run()