def SET_WHOIS_RAW_INFO(whois_dict):
SQL = """REPLACE INTO `{table}` set """.format(
table='WHOIS_raw_' +
str(Static.get_table_num(whois_dict['domain'])))
SQL += """`raw_whois` = '{Value}' ,""".format(
Value=whois_dict['details'])
SQL += """`domain` = '{Value}' """.format(Value=whois_dict['domain'])
return SQL
def SET_DOMAIN(whois_dict):
SQL = """UPDATE `{table}` set """.format(
table='domain_' + str(Static.get_table_num(whois_dict['domain'])))
SQL += """`TLD` = '{Value}', """.format(Value=whois_dict['tld'])
SQL += """`top_whois_srv` = '{Value}' """.format(
Value=whois_dict['top_whois_server'])
SQL += """WHERE `domain` = '{Value}'""".format(
Value=whois_dict['domain'])
return SQL
def SET_WHOIS_INFO(whois_dict):
if whois_dict['phone_type'] is None:
whois_dict['phone_type'] = 0
SQL = """REPLACE INTO `{table}` set """.format(
table='WHOIS_' + str(Static.get_table_num(whois_dict['domain'])))
SQL += """`domain` = '{Value}', """.format(Value=whois_dict['domain'])
SQL += """`domain_status` = '{Value}', """.format(
Value=whois_dict['domain_status'])
SQL += """`registrar` = '{Value}', """.format(
Value=whois_dict['sponsoring_registrar'])
SQL += """`sec_whois_srv` = '{Value}', """.format(
Value=whois_dict['sec_whois_server'])
SQL += """`reg_name` = '{Value}', """.format(
Value=whois_dict['reg_name'])
SQL += """`reg_phone` = '{Value}', """.format(
Value=whois_dict['reg_phone'])
SQL += """`phone_country_code` = '{Value}', """.format(
Value=whois_dict['phone_country_code'])
SQL += """`phone_position_code` = '{Value}', """.format(
Value=whois_dict['phone_position_code'])
SQL += """`phone_number` = '{Value}', """.format(
Value=whois_dict['phone_number'])
SQL += """`phone_type` = '{Value}', """.format(
Value=whois_dict['phone_type'])
SQL += """`reg_email` = '{Value}', """.format(
Value=whois_dict['reg_email'])
SQL += """`org_name` = '{Value}', """.format(
Value=whois_dict['org_name'])
if not whois_dict['creation_date'] == '':
SQL += """`creation_date` = '{Value}', """.format(
Value=whois_dict['creation_date'])
if not whois_dict['expiration_date'] == '':
SQL += """`expiration_date` = '{Value}', """.format(
Value=whois_dict['expiration_date'])
if not whois_dict['updated_date'] == '':
SQL += """`updated_date` = '{Value}', """.format(
Value=whois_dict['updated_date'])
SQL += """`name_server` = '{Value}' """.format(
Value=whois_dict['name_server'])
return SQL
def Update(self, whois_dict, old_flag):
"""
将 whois数据字典 更新至 数据库 中
:param whois_dict:whois信息字典
:param old_flag:之前获取域名的whois_flag
"""
# 2017.12.23 不再向WHOIS表中添加不存在的WHOIS记录,而把Domain表中的flag 置为-29
# flag 29 -> -29
old_flag = int(old_flag)
self.commit_count += 1
# commit 设置
if self.commit_count >= self.commit_limit:
self.DB.db_commit()
self.commit_count = 0
if whois_dict['domain_status'].strip() == '29': # 只更新domain 表
whois_dict['flag'] = '-29'
self.DB.execute_no_return(SQL_generate.SET_DOMAIN_INFO(whois_dict))
elif int(whois_dict['flag']) < 0: # 当前获取WHOIS 失败 只更新domain 表
self.DB.execute_no_return(SQL_generate.SET_DOMAIN(whois_dict))
elif old_flag == 0 and int(whois_dict['flag']) > 0: # 新增的数据
self.DB.execute_no_return(SQL_generate.SET_DOMAIN_INFO(whois_dict))
self.DB.execute_no_return(SQL_generate.SET_WHOIS_INFO(whois_dict))
self.DB.execute_no_return(
SQL_generate.SET_WHOIS_RAW_INFO(whois_dict))
else: # 之前已有成功获取的数据
record_transform = False # whois 记录归档标志
table = 'WHOIS_' + str(Static.get_table_num(whois_dict['domain']))
get_whois_info_SQL = SQL_generate.GET_WHOIS_INFO(
whois_dict['domain'], 'updated_date', table)
origin_update_date = self.DB.execute(get_whois_info_SQL)
if origin_update_date is None or str(
old_flag) == '0': # 之前不存在的记录 : 单独插入更新whois
pass # 之前不存在的记录 : 单独插入更新whois
else:
try:
origin_update_date = origin_update_date[0][0]
except Exception as error:
update_log.error('提取whois更新时间->' +
str(whois_dict['domain']) +
'unexpected error happen!' + str(error))
if str(origin_update_date) != str(whois_dict['updated_date']) \
and old_flag > 0 \
and whois_dict['updated_date'] \
and origin_update_date:
update_log.error('whowas更新 : ' + str(whois_dict['domain']))
record_transform = True
if record_transform:
whowas_table = 'WHOWAS_' + str(
Static.get_table_num(whois_dict['domain']))
whois_table = 'WHOIS_' + str(
Static.get_table_num(whois_dict['domain']))
# whowas
whowas_transform_SQL = SQL_generate.WHOWAS_TRANSFORM(
whowas_table, whois_table, whois_dict['domain'])
self.DB.execute_no_return(whowas_transform_SQL)
set_whois_info_SQL = SQL_generate.SET_WHOIS_INFO(
whois_dict)
self.DB.execute_no_return(set_whois_info_SQL)
self.DB.db_commit()
