def save_proxy(self, item: dict):
s = ProxySerializer(item)
if s.is_valid(raise_e=True):
save_res = self.cli.hmset_dict(s.key(), s.data())
logger.debug('Saving checked proxy: %s '
'HMSET result: %s' % (item, save_res))
async def get(self, *args, **kwargs):
"""
get proxies
"""
count = int(self.get_param('count', 1))
scheme = self.get_param('scheme')
if scheme:
scheme = scheme.lower()
anonymity = self.get_param('anonymity')
spec = dict(count=count, scheme=scheme, anonymity=anonymity)
_items = await proxy_srv.query(spec)
items = []
for i in _items:
s = ProxySerializer(i)
items.append(s.to_representation())
data = {
"count": len(items),
"detail": items,
}
# sort_by_speed = self.get_param('sort_by_speed', 0)
self.do_success(data)
async def new_proxy(self, item):
s = ProxySerializer(item)
if s.is_valid(raise_e=True):
save_res = await self.cli.hmset_dict(s.key, s.validated_data)
logger.debug('Saving checked proxy: %s '
'HMSET result: %s' % (s.validated_data, save_res))
return s.key
async def new_proxy(self, item):
s = ProxySerializer(item)
if s.is_valid(raise_e=True):
save_res = await self.cli.hmset_dict(s.key, item)
logger.info(
'Got proxy: %s hmset result: %s' % (item, save_res)
)
def add_failure(self, key):
item = self.hgetall_dict(key)
s = ProxySerializer(item)
fail_times = int(item.get('fail_times', 0))
current_fail_times = fail_times + 1
if current_fail_times >= FAIL_TIMES_MAXIMUM:
self.delete(key, 'Failed %s times' % fail_times)
else:
if s.is_valid():
self.cli.hincrby(key, 'fail_times', 1)
logger.debug('Key: %s Fail times: %s' %
(key, current_fail_times))
async def add_failure(self, key):
item = await self.cli.hgetall(key)
fail_times = int(item.get('fail_times', 0))
s = ProxySerializer(item)
current_fail_times = fail_times + 1
if current_fail_times >= FAIL_TIMES_MAXIMUM:
await self.cli.delete(key)
logger.info('Remove key: %s' % key)
else:
if s.is_valid():
await self.cli.hincrby(key, 'fail_times', 1)
logger.debug('Key: %s Fail times: %s' %
(key, current_fail_times))