def index():
"""index page"""
_start = "<h1>Whois API Service v1.1 </h1>"
_start += "<h3>author - h-j-13(@`13)</h3>"
_start += "<h3>author - WUD(@wangjunx)</h3><br><b4>"
_start += "<h3>Harbin Institute of Technology at Weihai</h3>"
_start += "system clk" + str(Static.get_local_time())
return _start
def SET_WHOIS_RAW_INFO(whois_dict):
SQL = """REPLACE INTO `{table}` set """.format(
table='WHOIS_raw_' +
str(Static.get_table_num(whois_dict['domain'])))
SQL += """`raw_whois` = '{Value}' ,""".format(
Value=whois_dict['details'])
SQL += """`domain` = '{Value}' """.format(Value=whois_dict['domain'])
return SQL
def SET_DOMAIN(whois_dict):
SQL = """UPDATE `{table}` set """.format(
table='domain_' + str(Static.get_table_num(whois_dict['domain'])))
SQL += """`TLD` = '{Value}', """.format(Value=whois_dict['tld'])
SQL += """`top_whois_srv` = '{Value}' """.format(
Value=whois_dict['top_whois_server'])
SQL += """WHERE `domain` = '{Value}'""".format(
Value=whois_dict['domain'])
return SQL
def SET_WHOIS_INFO(whois_dict):
if whois_dict['phone_type'] is None:
whois_dict['phone_type'] = 0
SQL = """REPLACE INTO `{table}` set """.format(
table='WHOIS_' + str(Static.get_table_num(whois_dict['domain'])))
SQL += """`domain` = '{Value}', """.format(Value=whois_dict['domain'])
SQL += """`domain_status` = '{Value}', """.format(
Value=whois_dict['domain_status'])
SQL += """`registrar` = '{Value}', """.format(
Value=whois_dict['sponsoring_registrar'])
SQL += """`sec_whois_srv` = '{Value}', """.format(
Value=whois_dict['sec_whois_server'])
SQL += """`reg_name` = '{Value}', """.format(
Value=whois_dict['reg_name'])
SQL += """`reg_phone` = '{Value}', """.format(
Value=whois_dict['reg_phone'])
SQL += """`phone_country_code` = '{Value}', """.format(
Value=whois_dict['phone_country_code'])
SQL += """`phone_position_code` = '{Value}', """.format(
Value=whois_dict['phone_position_code'])
SQL += """`phone_number` = '{Value}', """.format(
Value=whois_dict['phone_number'])
SQL += """`phone_type` = '{Value}', """.format(
Value=whois_dict['phone_type'])
SQL += """`reg_email` = '{Value}', """.format(
Value=whois_dict['reg_email'])
SQL += """`org_name` = '{Value}', """.format(
Value=whois_dict['org_name'])
if not whois_dict['creation_date'] == '':
SQL += """`creation_date` = '{Value}', """.format(
Value=whois_dict['creation_date'])
if not whois_dict['expiration_date'] == '':
SQL += """`expiration_date` = '{Value}', """.format(
Value=whois_dict['expiration_date'])
if not whois_dict['updated_date'] == '':
SQL += """`updated_date` = '{Value}', """.format(
Value=whois_dict['updated_date'])
SQL += """`name_server` = '{Value}' """.format(
Value=whois_dict['name_server'])
return SQL
#!/usr/bin/env python
# encoding:utf-8
"""
获取whois服务器对应的提取函数
==================================
version : 1.0
author : @`13
time : 2017.1.19
"""
from Setting.static import Static
Static.static_value_init()
class Func(object):
"""获取whois服务器对应的提取函数"""
# Singleton
_instance = None
def __new__(cls, *args, **kw):
if not cls._instance:
cls._instance = super(Func, cls).__new__(cls, *args, **kw)
return cls._instance
def __init__(self):
"""数据初始化"""
self.server_func_dict = {}
WhoisServiceFuncFile = open(Static.WHOIS_FUNC_FILE, 'r')
for line in WhoisServiceFuncFile.readlines():
# encoding:utf-8
"""
whois原始数据处理
=========================
version : 1.0
author : @`13
time : 2017.2.8
"""
from whois_func import * # 提取函数
from Setting.static import Static # 静态变量,设置
from domain_status import get_status_value # 处理状态值函数
from WhoisConnect.whois_connect import GetWhoisInfo, WhoisConnectException # whois通信
Static.log_init()
log_func = Static.LOGGER
def get_result(domain_punycode, tld, whois_addr, func_name, data, flag):
"""
:param domain_punycode: punycode格式的域名
:param tld: 顶级域
:param whois_addr: whois服务器
:param func_name: 处理函数
:param data: 服务器返回数据
:param flag: 数据正确性标记位
:return: whois 信息字典
"""
# 返回结果初始化
domain_whois = {
author : @`13
time : 2017.04.09
time_v1.1 : 2017.04.24
"""
import Queue
import urllib2
import threading
import socket
from time import sleep
from Setting.static import Static # 静态变量,设置
from Database.db_opreation import DataBase # 数据库对象
from Database.SQL_generate import SQL_generate # SQL语句生成器
Static.init()
global TaskQueue, ResultQueue
TaskQueue = Queue.Queue() # 任务队列
ResultQueue = Queue.Queue() # 结果队列
def work():
"""工作函数:获取域名的状态码以及解析情况,存入数据库"""
global TaskQueue, ResultQueue
while not TaskQueue.empty():
domain = TaskQueue.get()
available, HTTPcode = GetURLStatusCode(domain)
r = [domain, available, HTTPcode]
# Test - >
print 'TEST->' + domain + ':',
print r
def main(domain_list):
"""
主流程函数,获取whois并更新到数据库中
:param Worktype: 工作方式 0 - 获取新增域名
1 - 重新获取所有域名数据
-1 - 获取失败域名
SQL - 特定获取域名的SQL语句
"""
Static.init()
log_main = Static.LOGGER
# 全局变量声明
global whois_result_dict
whois_result_dict = []
global DomainQueue, WhoisQueue
DomainQueue = Queue.Queue() # 任务域名队列
WhoisQueue = Queue.Queue() # whois结果队列
global Process_Num, Ban_Setting
Process_Num = Static.PROCESS_NUM # 获取进程数
Ban_Setting = False # 防Ban设置
# 初始化操作对象
DB = DataBase()
DB.db_connect()
# 填充域名队列
for domain in domain_list:
# print domain
DomainQueue.put(str(domain).strip())
# 开始多线程获取域名
thread_list = []
for i in range(Process_Num):
get_whois_thread = threading.Thread(target=GetWhois)
get_whois_thread.setDaemon(True)
get_whois_thread.start()
thread_list.append(get_whois_thread)
# 开始域名字典,更新数据库
sleep(Static.SOCKS_TIMEOUT) # 等待队列填充
update_whois_thread = threading.Thread(target=WriteWhoisInfo(DB))
update_whois_thread.setDaemon(True)
update_whois_thread.start()
thread_list.append(update_whois_thread)
# 挂起进程直到结束
for update_whois_thread in thread_list:
update_whois_thread.join()
log_main.error("结束")
# 清空队列
while not WhoisQueue.empty():
whois_result_dict.append(WhoisQueue.get())
while not DomainQueue.empty():
DomainQueue.get()
DB.db_commit()
DB.db_close()
return whois_result_dict
# if __name__ == '__main__':
# print "whois探测开始"
# domain_list = ['sina.com']
# result = main(domain_list)
# #
# print result
#!/usr/bin/env python
# encoding:utf-8
"""
whois服务器通信
=====================
version : 1.0
author : @`13
time : 2017.1.18
"""
import socks
from Setting.static import Static
Static().static_value_init() # 静态值初始化
TIMEOUT = Static.SOCKS_TIMEOUT # 超时设定
Proxy_Flag = Static.PROXY_SOCKS_FLAG # 使用代理标志
_proxy_socks = None
# 获取代理ip
if Proxy_Flag:
from proxy_socks import ProxySocks
_proxy_socks = ProxySocks() # socks代理 获取对象
class WhoisConnectException(Exception):
"""whois 信息通信错误"""
def __init__(self, value):
self.value = value
def Update(self, whois_dict, old_flag):
"""
将 whois数据字典 更新至 数据库 中
:param whois_dict:whois信息字典
:param old_flag:之前获取域名的whois_flag
"""
# 2017.12.23 不再向WHOIS表中添加不存在的WHOIS记录,而把Domain表中的flag 置为-29
# flag 29 -> -29
old_flag = int(old_flag)
self.commit_count += 1
# commit 设置
if self.commit_count >= self.commit_limit:
self.DB.db_commit()
self.commit_count = 0
if whois_dict['domain_status'].strip() == '29': # 只更新domain 表
whois_dict['flag'] = '-29'
self.DB.execute_no_return(SQL_generate.SET_DOMAIN_INFO(whois_dict))
elif int(whois_dict['flag']) < 0: # 当前获取WHOIS 失败 只更新domain 表
self.DB.execute_no_return(SQL_generate.SET_DOMAIN(whois_dict))
elif old_flag == 0 and int(whois_dict['flag']) > 0: # 新增的数据
self.DB.execute_no_return(SQL_generate.SET_DOMAIN_INFO(whois_dict))
self.DB.execute_no_return(SQL_generate.SET_WHOIS_INFO(whois_dict))
self.DB.execute_no_return(
SQL_generate.SET_WHOIS_RAW_INFO(whois_dict))
else: # 之前已有成功获取的数据
record_transform = False # whois 记录归档标志
table = 'WHOIS_' + str(Static.get_table_num(whois_dict['domain']))
get_whois_info_SQL = SQL_generate.GET_WHOIS_INFO(
whois_dict['domain'], 'updated_date', table)
origin_update_date = self.DB.execute(get_whois_info_SQL)
if origin_update_date is None or str(
old_flag) == '0': # 之前不存在的记录 : 单独插入更新whois
pass # 之前不存在的记录 : 单独插入更新whois
else:
try:
origin_update_date = origin_update_date[0][0]
except Exception as error:
update_log.error('提取whois更新时间->' +
str(whois_dict['domain']) +
'unexpected error happen!' + str(error))
if str(origin_update_date) != str(whois_dict['updated_date']) \
and old_flag > 0 \
and whois_dict['updated_date'] \
and origin_update_date:
update_log.error('whowas更新 : ' + str(whois_dict['domain']))
record_transform = True
if record_transform:
whowas_table = 'WHOWAS_' + str(
Static.get_table_num(whois_dict['domain']))
whois_table = 'WHOIS_' + str(
Static.get_table_num(whois_dict['domain']))
# whowas
whowas_transform_SQL = SQL_generate.WHOWAS_TRANSFORM(
whowas_table, whois_table, whois_dict['domain'])
self.DB.execute_no_return(whowas_transform_SQL)
set_whois_info_SQL = SQL_generate.SET_WHOIS_INFO(
whois_dict)
self.DB.execute_no_return(set_whois_info_SQL)
self.DB.db_commit()
def index():
_result = "<h1>WHOIS api Service v0.1</h1>"
_result += "<h3>author - h-j-13(@`13)</h5>"
_result += "<h3>Harbin Institute of Technology at Weihai</h5> <br></br><br></br>"
_result += ("<h5>system clock -> " + Static.get_local_time() + "</h5>")
return _result