def save_whois(domain, whois_obj):
logging.info("Saving whois(domain: '%s') to file..." % domain)
original_domain_str = domain
#: 反向拆解域名，构造 whois 文件存放目录
domain = domain.split('.')
domain.reverse()
try:
path = os.path.join(WHOIS_DIR, *domain)
if not os.path.exists(path):
os.makedirs(path)
# 文件形式保存 whois 原始信息
absolute_path = os.path.join(path, WHOIS_FILENAME)
with open(absolute_path, 'w') as whois_file:
whois_file.write(whois_obj.text)
except Exception as e:
logging.error("Saving to file Exception: %s" % e)
# 然后以 Json 形式保存到数据库
try:
with DB_CONN.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `whois` (`domain`, `whois`, `last_modified`) VALUES (%s, %s, %s)"
cursor.execute(sql, (original_domain_str,
json.dumps(json.loads(str(whois_obj)),
ensure_ascii=False,
separators=(',', ':')),
time.strftime('%Y-%m-%d %H:%M:%S')))
# connection is not autocommit by default. So you must commit to save your changes.
DB_CONN.commit()
except AttributeError as ae:
logging.error("DB connection timeout. Exception: %s" % ae.args)
DB_CONN.connect()
except Exception as e:
logging.error("Save into SQL error. Exception: %s" % e.args)
try:
with DB_CONN.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `domain` FROM `whois` WHERE `domain`=%s"
cursor.execute(sql, (original_domain_str, ))
result = cursor.fetchone()
logging.debug("Saved whois json into mysql: %s" % result)
except AttributeError as ae:
logging.error("DB connection timeout. Exception: %s" % ae.args)
DB_CONN.connect()
except Exception as e:
logging.error("Query SQL error. Exception: %s" % e.args)
def save_whois(domain, whois_obj):
logging.info("Saving whois(domain: '%s') to file..." % domain)
original_domain_str = domain
#: 反向拆解域名，构造 whois 文件存放目录
domain = domain.split('.')
domain.reverse()
try:
path = os.path.join(WHOIS_DIR, *domain)
if not os.path.exists(path):
os.makedirs(path)
# 文件形式保存 whois 原始信息
absolute_path = os.path.join(path, WHOIS_FILENAME)
with open(absolute_path, 'w') as whois_file:
whois_file.write(whois_obj.text)
except Exception as e:
logging.error("Saving to file Exception: %s" % e)
# 然后以 Json 形式保存到数据库
try:
with DB_CONN.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `whois` (`domain`, `whois`, `last_modified`) VALUES (%s, %s, %s)"
cursor.execute(sql, (original_domain_str,
json.dumps(json.loads(str(whois_obj)), ensure_ascii=False, separators=(',', ':')),
time.strftime('%Y-%m-%d %H:%M:%S')))
# connection is not autocommit by default. So you must commit to save your changes.
DB_CONN.commit()
except AttributeError as ae:
logging.error("DB connection timeout. Exception: %s" % ae.args)
DB_CONN.connect()
except Exception as e:
logging.error("Save into SQL error. Exception: %s" % e.args)
try:
with DB_CONN.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `domain` FROM `whois` WHERE `domain`=%s"
cursor.execute(sql, (original_domain_str,))
result = cursor.fetchone()
logging.debug("Saved whois json into mysql: %s" % result)
except AttributeError as ae:
logging.error("DB connection timeout. Exception: %s" % ae.args)
DB_CONN.connect()
except Exception as e:
logging.error("Query SQL error. Exception: %s" % e.args)
if not self.isAlive():
self.setDaemon(True)
self.start()
try:
fail_domain = fail_quene.get()
logging.info("Saving fail querying domain '%s' to fail.txt." % fail_domain)
with open('data/fail.txt', 'a') as fail_file:
fail_file.write("%s%s" % (fail_domain, '\n'))
except fail_quene.empty():
continue
fail_quene.task_done()
def setup_logging():
coloredlogs.DEFAULT_LOG_FORMAT = '[%(levelname)-8s %(filename)s:%(lineno)d] %(message)s'
coloredlogs.install(level=LOG_LEVEL)
# logging.basicConfig(level=LOG_LEVEL, format='[%(levelname)s %(filename)s:%(lineno)d] %(message)s')
if __name__ == '__main__':
EXIT_FLAG.clear()
setup_logging()
start = time.time()
work_manager = WorkManager()
domains.join()
EXIT_FLAG.set()
# work_manager.wait_all_complete()
DB_CONN.close()
logging.debug("Finish whois querying, time cost: %ss." % (time.time() - start))
