def __init__(self, database=None, url_prefix=None, fetcher=None, checker=None): if not database: self.database = RedisWrapper("127.0.0.1", 6379, 0) else: self.database = RedisWrapper(database.host, database.port, database.db, database.password) self._origin_prefix = 'origin_proxy' self._useful_prefix = 'useful_proxy' self._hundred_prefix = 'hundred_proxy' self._current_prefix = 'current_proxy' if not url_prefix: self._url_prefix = "default" else: self._url_prefix = url_prefix if not fetcher: # validater self._fetcher = Fetcher() else: # refresher self._fetcher = fetcher self._fetcher.backup_provider() log.info("REFRESH FETCHER BACKUP PROVIDER {0}".format( str(self._fetcher))) if not checker: self._checker = Checker() else: self._checker = checker self.log = log
class ProxyPipe(object): def __init__(self, database=None, fetcher=None, url_prefix=None): if not fetcher: self._fetcher = Fetcher() else: self._fetcher = fetcher self._database = database self._url_prefix = url_prefix def set_fetcher(self, provider_list): self._fetcher.set_provider(provider_list) return self def start(self): proc1 = Process(target=validater_run, args=( self._url_prefix, self._database, )) proc2 = Process(target=refresher_run, args=( self._url_prefix, self._fetcher, self._database, )) proc_list = [proc1, proc2] for proc in proc_list: proc.start() time.sleep(1) for proc in proc_list: proc.join()
def __init__(self, database=None, fetcher=None, url_prefix=None): if not fetcher: self._fetcher = Fetcher() else: self._fetcher = fetcher self._database = database self._url_prefix = url_prefix
def __init__(self, database=None, url_prefix=None, fetcher=None, checker=None): if not database: self.database = RedisWrapper("127.0.0.1", 6379, 0) else: self.database = RedisWrapper(database.host, database.port, database.db) self._origin_prefix = 'origin_proxy' self._useful_prefix = 'useful_proxy' if not url_prefix: self._url_prefix = "default" else: self._url_prefix = url_prefix if not fetcher: self._fetcher = Fetcher() else: self._fetcher = fetcher if not checker: self._checker = Checker() else: self._checker = checker self.log = log
def main_run(): redis = RedisConfig("127.0.0.1", 21009) p1 = ProxyPipe(url_prefix="https://www.baidu.com", fetcher=Fetcher(use_default=False), database=redis, checker=CheckerBaidu()).set_fetcher( [KuaiProvider()]).add_fetcher([XiciProvider()]) p1.start()
def __init__(self, database=None, fetcher=None, url_prefix=None, checker=None, validater_thread_num=30, refresher_thread_num=30): if not fetcher: self._fetcher = Fetcher() else: self._fetcher = fetcher if not checker: self._checker = Checker() else: self._checker = checker self._database = database self._url_prefix = url_prefix self.validater_thread_num = validater_thread_num self.refresher_thread_num = refresher_thread_num
def __init__(self, url_prefix=None, fetcher=None): self.database = RedisWrapper("127.0.0.1", 6379) self._origin_prefix = 'origin_proxy' self._useful_prefix = 'useful_proxy' if not url_prefix: self._url_prefix = "default" else: self._url_prefix = url_prefix if not fetcher: self._fetcher = Fetcher() else: self._fetcher = fetcher self.log = log
class Manager(object): def __init__(self, database=None, url_prefix=None, fetcher=None, checker=None): if not database: self.database = RedisWrapper("127.0.0.1", 6379, 0) else: self.database = RedisWrapper(database.host, database.port, database.db, database.password) self._origin_prefix = 'origin_proxy' self._useful_prefix = 'useful_proxy' self._hundred_prefix = 'hundred_proxy' self._current_prefix = 'current_proxy' if not url_prefix: self._url_prefix = "default" else: self._url_prefix = url_prefix if not fetcher: # validater self._fetcher = Fetcher() else: # refresher self._fetcher = fetcher self._fetcher.backup_provider() log.info("REFRESH FETCHER BACKUP PROVIDER {0}".format( str(self._fetcher))) if not checker: self._checker = Checker() else: self._checker = checker self.log = log def get_netloc(self): if self._url_prefix == "default": return "default" return urlparse(self._url_prefix).netloc def generate_name(self, prefix): return ":".join(["spoon", self.get_netloc(), prefix]) def refresh_condition(self): all_proxy_score = [[ k.decode('utf-8'), int(v.decode('utf-8')) ] for (k, v) in self.get_all_kv_from( self.generate_name(self._useful_prefix)).items()] all_length = len(all_proxy_score) count_length = len([0 for (k, v) in all_proxy_score if v >= 95]) if all_length <= 100: return True if count_length / all_length >= 0.2: return True else: return False def refresh(self): log.info("REFRESH START WITH {0} TARGET {1}".format( str(self._fetcher), self.get_netloc())) if not self.refresh_condition(): log.info("REFRESH DID NOT MEET CONDITION. TARGET{0}".format( self.get_netloc())) return if len(self._fetcher) < 6: self._fetcher.restore_provider() log.info( "REFRESH FETCHER FAILED: NO ENOUGH PROVIDER, RESTORE PROVIDERS TO {0} for TARGET {1}" .format(str(self._fetcher), self.get_netloc())) proxy_set = set() provider_to_be_removed_index = [] for index in range(len(self._fetcher)): provider = self._fetcher.get_provider(index) try: for proxy in provider.getter(): if proxy.strip(): self.log.info( "REFRESH FETCHER: TARGET {0} PROVIDER {1} PROXY {2}" .format(self.get_netloc(), provider.__class__.__name__, proxy.strip())) proxy_set.add(proxy.strip()) except Exception as e: provider_to_be_removed_index.append(index) log.error( "REFRESH FETCHER FAILED: PROVIDER {0} WILL BE REMOVED ERROR {1}" .format(provider.__class__.__name__, e)) for proxy in proxy_set: self.database.set_value("spoon:proxy_stale", proxy, time.time()) self.database.put(self.generate_name(self._origin_prefix), proxy) log.info("REFRESH FETCHER DELETE {0}. TARGET {1}".format( provider_to_be_removed_index, self.get_netloc())) self._fetcher.remove_provider(provider_to_be_removed_index) def get(self): return self.database.get(self.generate_name(self._useful_prefix)) def set_value(self, key, value): return self.database.set_value(self.generate_name(self._useful_prefix), key, value) def delete(self, proxy): self.database.delete(self.generate_name(self._useful_prefix), proxy) def get_all(self): return self.database.get_all(self.generate_name(self._useful_prefix)) def get_status(self): total_origin_proxy = self.database.get_status( self.generate_name(self._origin_prefix)) total_useful_queue = self.database.get_status( self.generate_name(self._useful_prefix)) return { 'origin_proxy': total_origin_proxy, 'useful_proxy': total_useful_queue } # For spoon_web def get_keys(self): return [key.decode("utf-8") for key in self.database.get_keys()] def get_from(self, target): return self.database.get(target) def get_all_from(self, target): return self.database.get_all(target) def get_all_kv_from(self, target): return self.database.get_all_kv(target) def get_range_from(self, target): return self.database.zrange(target, 0, -1) def scan_kv_from(self, target, cursor): return self.database.scan_kv(target, cursor)
def main_run(): redis = RedisConfig("127.0.0.1", 21009) p1 = ProxyPipe(url_prefix="https://www.google.com", fetcher=Fetcher(use_default=False), database=redis).set_fetcher([UsProvider()]) p1.start()
def main_run(): p1 = ProxyPipe(url_prefix="https://www.google.com", fetcher=Fetcher(use_default=False)).set_fetcher( [UsProvider()]) p1.start()
def __init__(self, fetcher=None, url_prefix=None): if not fetcher: self._fetcher = Fetcher() self._url_prefix = url_prefix