Python Crawler.get_proxiesの例

コード例 #1

class Getter(object):
    def __init__(self):
        self.db = RedisClient()
        self.crawl = Crawler()

    def is_over_threshold(self):
        if self.db.count() >= POOL_UPPER_THRESHOLD:
            return True
        else:
            return False

    def run(self):
        if not self.is_over_threshold():
            func_names = self.crawl.__FUNC__
            for func_name in func_names:
                proxies = self.crawl.get_proxies(func_name)
                for proxy in proxies:
                    self.db.add(proxy)

コード例 #2

ファイル: getter_.py プロジェクト: cshk/proxy_pool

class Getter(object):
    def __init__(self):
        self.redis = RedisClient()
        self.crawler = Crawler()

    def is_over_threshold(self):
        if self.redis.count() >= POOL_UPPER_THRESHOLD:
            return True
        else:
            return False

    def run(self):
        print('获取器开始执行...')
        if not self.is_over_threshold():
            for callback_label in range(self.crawler.__CrawlFuncCount__):
                callback = self.crawler.__CrawlFunc__[callback_label]
                proxies = self.crawler.get_proxies(callback)
                for proxy in proxies:
                    self.redis.add(proxy)


# r = Getter()
# r.run()