def get_domain(self):
domain_queue = Queue(-1)
i = 1
while i <= DOMAIN_SOURCE_TABLE_NUM:
for domain in Static.whois_db.get_not_deal_domains(
i, Static.FINISHED_TLD):
domain_queue.put(domain)
i += 1
print Static.get_now_time(), '提取出域名: ', domain_queue.qsize()
return domain_queue
def update_not_deal_method(domain, exist):
if exist:
Static.whois_db.set_whois_flag(domain, -98)
else:
domain_whois = {
'domain_hash': hash(domain), # 域名哈希值
"domain": domain, # 域名
"flag": -98, # 状态标记
"domain_status": "", # 域名状态
"sponsoring_registrar": "", # 注册商
"top_whois_server": "", # 顶级域名服务器
"sec_whois_server": "", # 二级域名服务器
"reg_name": "", # 注册姓名
"reg_phone": "", # 注册电话
"reg_email": "", # 注册email
"org_name": "", # 注册公司名称
"creation_date": "", # 创建时间
"expiration_date": "", # 到期时间
"updated_date": "", # 更新时间
"insert_time": Static.get_now_time(), # 信息插入时间
"details": "", # 细节
"name_server": "", # 域名服务器
"whois_hash": 0, # whois信息哈希值
}
Static.whois_db.db_whois_insert(**domain_whois)
def whois_pull(self, ch, method, properties, body):
recv_info_from_client = json.loads(body)
if type(recv_info_from_client) == unicode:
recv_info_from_client = json.loads(recv_info_from_client)
# print 'whois pull:', recv_info_from_client['domain']
update_record(recv_info_from_client)
ch.basic_ack(delivery_tag=method.delivery_tag) # 返回处理完成确认信息
if self.count_lock.acquire():
self.count += 1
if self.count > Static.COMMIT_NUM:
Static.whois_db.db_commit() # commit 操作
print 'Commit: ', Static.get_now_time()
self.count = 0
self.count_lock.release()
def update_record(recv_info_from_client):
result_flag = recv_info_from_client['result_flag']
func_name = recv_info_from_client['func_name']
exist = recv_info_from_client['exist']
domain_whois = {
'domain_hash': hash(recv_info_from_client['domain']), # 域名哈希值
"domain": recv_info_from_client['domain'], # 域名
"flag": 0, # 状态标记
"domain_status": "", # 域名状态
"sponsoring_registrar": "", # 注册商
"top_whois_server": recv_info_from_client['top_whois_server'], # 顶级域名服务器
"sec_whois_server": recv_info_from_client['sec_whois_server'], # 二级域名服务器
"reg_name": "", # 注册姓名
"reg_phone": "", # 注册电话
"reg_email": "", # 注册email
"org_name": "", # 注册公司名称
"creation_date": "", # 创建时间
"expiration_date": "", # 到期时间
"updated_date": "", # 更新时间
"insert_time": Static.get_now_time(), # 信息插入时间
"details": recv_info_from_client['whois_details'], # 细节
"name_server": "", # 域名服务器
"whois_hash": 0, # whois信息哈希值
}
domain_whois = get_deal_result(domain_whois, func_name, result_flag)
if not domain_whois:
return
# 第一次跑数据
if exist:
# 数据库中存在该记录
if not is_exist(domain_whois.get('domain_status')):
# 域名为未存在
Static.whois_db.delete_whois_info(domain_whois['domain'])
else:
Static.whois_db.update_whois_info(**domain_whois)
else:
# 数据库中不存在该记录
if is_exist(domain_whois.get('domain_status')):
# 域名存在
Static.whois_db.db_whois_insert(**domain_whois)
return domain_whois
def main():
print Static.get_now_time(), ' Start'
# api_main.open_search_api() # 打开api服务器
Static.whois_db.connect() # 数据库连接
WorkDispatch().open_dispatch() # 开启任务分发
WhoisQueueMonitor().open_monitor() # 开启whois队列监控
def open_search_api():
print Static.get_now_time(), '打开查询api 端口:', Static.WEB_SERVER_PORT
app.run()