def is_valid(self, raise_e=False):
"""
:param raise_e: whether raise exception
:rtype: bool
"""
result = functions.valid_format(self._item)
self._is_valid = result
if not result and raise_e:
raise AssertionError('Format error %s' % self._item)
return result
def start_requests(self):
keys = self.srv.get_all_keys()
for key in keys:
data = self.srv.hgetall_dict(key)
last_check = data.get('last_check', 0)
if not valid_format(data):
self.srv.delete(key, 'Error format %s' % data)
continue
if exceed_check_period(last_check):
item = Proxy(**data)
yield self.build_check_request(item)
def build_check_recipient(self,
ip,
port,
scheme,
user=None,
password=None):
"""
1. build a request for availability checking
2. drop it if already existed
:return: Request
"""
if self.complete_condition():
raise exceptions.CloseSpider('Enough items')
spec = dict(ip=ip, port=port, scheme=scheme)
if not valid_format(spec):
self.logger.debug('Got wrong format (%s, %s). Clear it.' %
(ip, port))
return {}
if self.already_exists(spec):
self.logger.debug('Got duplicated %s. Clear it.' % spec.values())
return {} # drop it
proxy_url = utils.build_proxy_url(ip, port, scheme, user, password)
need_auth = bool(user and password)
item = Proxy(
ip=ip,
scheme=scheme,
port=port,
need_auth=need_auth,
url=proxy_url,
)
if need_auth:
item['user'], item['password'] = user, password
self.logger.debug('Got unchecked %s' % item)
return self.build_check_request(item)
def _validate_format(self):
fmt_ok = functions.valid_format(self._item)
assert fmt_ok, 'Format error %s' % self._item