def SET_WHOIS_RAW_INFO(whois_dict): SQL = """REPLACE INTO `{table}` set """.format( table='WHOIS_raw_' + str(Static.get_table_num(whois_dict['domain']))) SQL += """`raw_whois` = '{Value}' ,""".format( Value=whois_dict['details']) SQL += """`domain` = '{Value}' """.format(Value=whois_dict['domain']) return SQL
def SET_DOMAIN(whois_dict): SQL = """UPDATE `{table}` set """.format( table='domain_' + str(Static.get_table_num(whois_dict['domain']))) SQL += """`TLD` = '{Value}', """.format(Value=whois_dict['tld']) SQL += """`top_whois_srv` = '{Value}' """.format( Value=whois_dict['top_whois_server']) SQL += """WHERE `domain` = '{Value}'""".format( Value=whois_dict['domain']) return SQL
def SET_WHOIS_INFO(whois_dict): if whois_dict['phone_type'] is None: whois_dict['phone_type'] = 0 SQL = """REPLACE INTO `{table}` set """.format( table='WHOIS_' + str(Static.get_table_num(whois_dict['domain']))) SQL += """`domain` = '{Value}', """.format(Value=whois_dict['domain']) SQL += """`domain_status` = '{Value}', """.format( Value=whois_dict['domain_status']) SQL += """`registrar` = '{Value}', """.format( Value=whois_dict['sponsoring_registrar']) SQL += """`sec_whois_srv` = '{Value}', """.format( Value=whois_dict['sec_whois_server']) SQL += """`reg_name` = '{Value}', """.format( Value=whois_dict['reg_name']) SQL += """`reg_phone` = '{Value}', """.format( Value=whois_dict['reg_phone']) SQL += """`phone_country_code` = '{Value}', """.format( Value=whois_dict['phone_country_code']) SQL += """`phone_position_code` = '{Value}', """.format( Value=whois_dict['phone_position_code']) SQL += """`phone_number` = '{Value}', """.format( Value=whois_dict['phone_number']) SQL += """`phone_type` = '{Value}', """.format( Value=whois_dict['phone_type']) SQL += """`reg_email` = '{Value}', """.format( Value=whois_dict['reg_email']) SQL += """`org_name` = '{Value}', """.format( Value=whois_dict['org_name']) if not whois_dict['creation_date'] == '': SQL += """`creation_date` = '{Value}', """.format( Value=whois_dict['creation_date']) if not whois_dict['expiration_date'] == '': SQL += """`expiration_date` = '{Value}', """.format( Value=whois_dict['expiration_date']) if not whois_dict['updated_date'] == '': SQL += """`updated_date` = '{Value}', """.format( Value=whois_dict['updated_date']) SQL += """`name_server` = '{Value}' """.format( Value=whois_dict['name_server']) return SQL
def Update(self, whois_dict, old_flag): """ 将 whois数据字典 更新至 数据库 中 :param whois_dict:whois信息字典 :param old_flag:之前获取域名的whois_flag """ # 2017.12.23 不再向WHOIS表中添加不存在的WHOIS记录,而把Domain表中的flag 置为-29 # flag 29 -> -29 old_flag = int(old_flag) self.commit_count += 1 # commit 设置 if self.commit_count >= self.commit_limit: self.DB.db_commit() self.commit_count = 0 if whois_dict['domain_status'].strip() == '29': # 只更新domain 表 whois_dict['flag'] = '-29' self.DB.execute_no_return(SQL_generate.SET_DOMAIN_INFO(whois_dict)) elif int(whois_dict['flag']) < 0: # 当前获取WHOIS 失败 只更新domain 表 self.DB.execute_no_return(SQL_generate.SET_DOMAIN(whois_dict)) elif old_flag == 0 and int(whois_dict['flag']) > 0: # 新增的数据 self.DB.execute_no_return(SQL_generate.SET_DOMAIN_INFO(whois_dict)) self.DB.execute_no_return(SQL_generate.SET_WHOIS_INFO(whois_dict)) self.DB.execute_no_return( SQL_generate.SET_WHOIS_RAW_INFO(whois_dict)) else: # 之前已有成功获取的数据 record_transform = False # whois 记录归档标志 table = 'WHOIS_' + str(Static.get_table_num(whois_dict['domain'])) get_whois_info_SQL = SQL_generate.GET_WHOIS_INFO( whois_dict['domain'], 'updated_date', table) origin_update_date = self.DB.execute(get_whois_info_SQL) if origin_update_date is None or str( old_flag) == '0': # 之前不存在的记录 : 单独插入更新whois pass # 之前不存在的记录 : 单独插入更新whois else: try: origin_update_date = origin_update_date[0][0] except Exception as error: update_log.error('提取whois更新时间->' + str(whois_dict['domain']) + 'unexpected error happen!' + str(error)) if str(origin_update_date) != str(whois_dict['updated_date']) \ and old_flag > 0 \ and whois_dict['updated_date'] \ and origin_update_date: update_log.error('whowas更新 : ' + str(whois_dict['domain'])) record_transform = True if record_transform: whowas_table = 'WHOWAS_' + str( Static.get_table_num(whois_dict['domain'])) whois_table = 'WHOIS_' + str( Static.get_table_num(whois_dict['domain'])) # whowas whowas_transform_SQL = SQL_generate.WHOWAS_TRANSFORM( whowas_table, whois_table, whois_dict['domain']) self.DB.execute_no_return(whowas_transform_SQL) set_whois_info_SQL = SQL_generate.SET_WHOIS_INFO( whois_dict) self.DB.execute_no_return(set_whois_info_SQL) self.DB.db_commit()