def _validate_proxy(self):
origin_proxy = self.database.pop(self.generate_name(self._origin_prefix))
exist_proxy = self.database.get_all(self.generate_name(self._origin_prefix))
while origin_proxy:
if validate(self._url_prefix, origin_proxy) and (origin_proxy not in exist_proxy):
self.database.put(self.generate_name(self._useful_prefix), origin_proxy)
origin_proxy = self.database.pop(self.generate_name(self._origin_prefix))
def _validate_proxy(self):
while True:
for each_proxy in self.database.get_all(
self.generate_name(self._useful_prefix)):
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
if validate(self._url_prefix, each_proxy):
self.database.inckey(
self.generate_name(self._useful_prefix), each_proxy, 1)
else:
self.database.inckey(
self.generate_name(self._useful_prefix), each_proxy,
-1)
value = self.database.getvalue(
self.generate_name(self._useful_prefix), each_proxy)
if value and int(value) < -2:
self.database.delete(
self.generate_name(self._useful_prefix), each_proxy)
time.sleep(2)
def _validate_proxy(self, each_proxy):
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
value = self.database.getvalue(self.generate_name(self._useful_prefix),
each_proxy)
if int(value) < -2:
self.database.delete(self.generate_name(self._useful_prefix),
each_proxy)
if validate(self._url_prefix, each_proxy):
if not int(value) >= 100:
self.database.inckey(self.generate_name(self._useful_prefix),
each_proxy, 1)
else:
self.database.set_value(
self.generate_name(self._useful_prefix), each_proxy, 100)
else:
if int(value) > 0:
self.database.set_value(
self.generate_name(self._useful_prefix), each_proxy,
int(int(value) / 2))
self.database.inckey(self.generate_name(self._useful_prefix),
each_proxy, -1)
def _validate_proxy(self, each_proxy):
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
value = int(
self.database.getvalue(self.generate_name(self._useful_prefix),
each_proxy))
if value < 0:
self.database.delete(self.generate_name(self._useful_prefix),
each_proxy)
else:
if validate(self._url_prefix, each_proxy, self._checker):
self.database.zadd(self.generate_name(self._current_prefix),
each_proxy, int(-1 * time.time()))
self.database.zremrangebyrank(
self.generate_name(self._current_prefix), 100, 10000)
if not value >= 100:
if value == 99:
self.database.set_value(
self.generate_name(self._hundred_prefix),
each_proxy, time.time())
self.database.inckey(
self.generate_name(self._useful_prefix), each_proxy, 1)
else:
self.database.set_value(
self.generate_name(self._hundred_prefix), each_proxy,
time.time())
self.database.set_value(
self.generate_name(self._useful_prefix), each_proxy,
100)
else:
self.database.zrem(self.generate_name(self._current_prefix),
each_proxy)
if value > 0:
self.database.set_value(
self.generate_name(self._useful_prefix), each_proxy,
value // 2)
self.database.inckey(self.generate_name(self._useful_prefix),
each_proxy, -1)