Exemplo n.º 1
0
def save_whois(domain, whois_obj):
    logging.info("Saving whois(domain: '%s') to file..." % domain)
    original_domain_str = domain
    #: 反向拆解域名,构造 whois 文件存放目录
    domain = domain.split('.')
    domain.reverse()
    try:
        path = os.path.join(WHOIS_DIR, *domain)
        if not os.path.exists(path):
            os.makedirs(path)

        # 文件形式保存 whois 原始信息
        absolute_path = os.path.join(path, WHOIS_FILENAME)
        with open(absolute_path, 'w') as whois_file:
            whois_file.write(whois_obj.text)
    except Exception as e:
        logging.error("Saving to file Exception: %s" % e)

    # 然后以 Json 形式保存到数据库
    try:
        with DB_CONN.cursor() as cursor:
            # Create a new record
            sql = "INSERT INTO `whois` (`domain`, `whois`, `last_modified`) VALUES (%s, %s, %s)"
            cursor.execute(sql, (original_domain_str,
                                 json.dumps(json.loads(str(whois_obj)),
                                            ensure_ascii=False,
                                            separators=(',', ':')),
                                 time.strftime('%Y-%m-%d %H:%M:%S')))
        # connection is not autocommit by default. So you must commit to save your changes.
        DB_CONN.commit()
    except AttributeError as ae:
        logging.error("DB connection timeout. Exception: %s" % ae.args)
        DB_CONN.connect()
    except Exception as e:
        logging.error("Save into SQL error. Exception: %s" % e.args)

    try:
        with DB_CONN.cursor() as cursor:
            # Read a single record
            sql = "SELECT `id`, `domain` FROM `whois` WHERE `domain`=%s"
            cursor.execute(sql, (original_domain_str, ))
            result = cursor.fetchone()
            logging.debug("Saved whois json into mysql: %s" % result)
    except AttributeError as ae:
        logging.error("DB connection timeout. Exception: %s" % ae.args)
        DB_CONN.connect()
    except Exception as e:
        logging.error("Query SQL error. Exception: %s" % e.args)
Exemplo n.º 2
0
def save_whois(domain, whois_obj):
    logging.info("Saving whois(domain: '%s') to file..." % domain)
    original_domain_str = domain
    #: 反向拆解域名,构造 whois 文件存放目录
    domain = domain.split('.')
    domain.reverse()
    try:
        path = os.path.join(WHOIS_DIR, *domain)
        if not os.path.exists(path):
            os.makedirs(path)

        # 文件形式保存 whois 原始信息
        absolute_path = os.path.join(path, WHOIS_FILENAME)
        with open(absolute_path, 'w') as whois_file:
            whois_file.write(whois_obj.text)
    except Exception as e:
        logging.error("Saving to file Exception: %s" % e)

    # 然后以 Json 形式保存到数据库
    try:
        with DB_CONN.cursor() as cursor:
            # Create a new record
            sql = "INSERT INTO `whois` (`domain`, `whois`, `last_modified`) VALUES (%s, %s, %s)"
            cursor.execute(sql, (original_domain_str,
                                 json.dumps(json.loads(str(whois_obj)), ensure_ascii=False, separators=(',', ':')),
                                 time.strftime('%Y-%m-%d %H:%M:%S')))
        # connection is not autocommit by default. So you must commit to save your changes.
        DB_CONN.commit()
    except AttributeError as ae:
        logging.error("DB connection timeout. Exception: %s" % ae.args)
        DB_CONN.connect()
    except Exception as e:
        logging.error("Save into SQL error. Exception: %s" % e.args)

    try:
        with DB_CONN.cursor() as cursor:
            # Read a single record
            sql = "SELECT `id`, `domain` FROM `whois` WHERE `domain`=%s"
            cursor.execute(sql, (original_domain_str,))
            result = cursor.fetchone()
            logging.debug("Saved whois json into mysql: %s" % result)
    except AttributeError as ae:
        logging.error("DB connection timeout. Exception: %s" % ae.args)
        DB_CONN.connect()
    except Exception as e:
        logging.error("Query SQL error. Exception: %s" % e.args)
Exemplo n.º 3
0
            if not self.isAlive():
                self.setDaemon(True)
                self.start()
            try:
                fail_domain = fail_quene.get()
                logging.info("Saving fail querying domain '%s' to fail.txt." % fail_domain)
                with open('data/fail.txt', 'a') as fail_file:
                    fail_file.write("%s%s" % (fail_domain, '\n'))
            except fail_quene.empty():
                continue
            fail_quene.task_done()


def setup_logging():
    coloredlogs.DEFAULT_LOG_FORMAT = '[%(levelname)-8s %(filename)s:%(lineno)d] %(message)s'
    coloredlogs.install(level=LOG_LEVEL)
    # logging.basicConfig(level=LOG_LEVEL, format='[%(levelname)s %(filename)s:%(lineno)d] %(message)s')

if __name__ == '__main__':
    EXIT_FLAG.clear()

    setup_logging()
    start = time.time()
    work_manager = WorkManager()
    domains.join()
    EXIT_FLAG.set()

    # work_manager.wait_all_complete()
    DB_CONN.close()
    logging.debug("Finish whois querying, time cost: %ss." % (time.time() - start))