def build_check_recipient(self, ip, port, scheme,
user=None, password=None):
"""
1. build a request for availability checking
2. drop it if already existed
:return: Request
"""
if self.complete_condition():
raise exceptions.CloseSpider('Enough items')
spec = dict(ip=ip, port=port, scheme=scheme)
if self.already_exists(spec):
self.logger.debug('Dropped duplicated: %s' % spec.values())
return {} # drop it
proxy_url = utils.build_proxy_url(ip, port, scheme, user, password)
need_auth = bool(user and password)
item = Proxy(
ip=ip,
scheme=scheme,
port=port,
need_auth=need_auth,
url=proxy_url,
)
if need_auth:
item['user'], item['password'] = user, password
return self.build_check_request(item)
def validated_data(self):
assert hasattr(self,
'_is_valid'), ('You must call `.is_valid` at first')
assert self._is_valid, 'Data is not valid.'
result = dict(self._item)
result['scheme'] = result['scheme'].lower()
result.setdefault('url', build_proxy_url(**result))
result.setdefault('last_check', 0)
result.setdefault('fail_times', 0)
result.setdefault('need_auth', 0)
result.setdefault('anonymity', 'transparent')
print(result)
return result