def pop(cls, db: StrictRedis, names):
"""
从指定爬虫中弹出一条最合适的URL
Returns: (url, name)
当所有队列为空时，url == name == None
"""
# 随机挑选爬虫
names = copy.deepcopy(names)
random.shuffle(names)
sha = utils.RedisScripts.sha1('url_pop')
url, name, address = db.evalsha(sha, 1, time.time(), *names)
if url and name:
return [url.decode(), name.decode(), address.decode() if address else None]
else:
return [None, None, None]
def eval(self, redis: StrictRedis, *keys, **kwargs):
"""Tries to call ``EVALSHA`` with the `hash` and then, if it fails, calls
regular ``EVAL`` with.
"""
args = kwargs.pop('args', ())
# make sure number of keys passed matches keys required
if len(keys) != self._num_keys:
raise RedisScriptError(
f"Key length mismatch. Expected {self._num_keys}. "
f"Got {len(keys)} in {self._name}")
if kwargs:
raise TypeError(
f"Unexpected keyword arguments {kwargs.keys()} in {self._name}."
)
try:
return redis.evalsha(self._script_digest, self._num_keys,
*keys + args)
except NoScriptError:
return redis.eval(self._script_encoded, self._num_keys,
*keys + args)
