Exemplo n.º 1
0
    def pop(cls, db: StrictRedis, names):
        """
        从指定爬虫中弹出一条最合适的URL
        Returns: (url, name)
            当所有队列为空时,url == name == None
        """
        # 随机挑选爬虫
        names = copy.deepcopy(names)
        random.shuffle(names)

        sha = utils.RedisScripts.sha1('url_pop')
        url, name, address = db.evalsha(sha, 1, time.time(), *names)
        if url and name:
            return [url.decode(), name.decode(), address.decode() if address else None]
        else:
            return [None, None, None]
Exemplo n.º 2
0
    def eval(self, redis: StrictRedis, *keys, **kwargs):
        """Tries to call ``EVALSHA`` with the `hash` and then, if it fails, calls
        regular ``EVAL`` with.
        """
        args = kwargs.pop('args', ())
        # make sure number of keys passed matches keys required
        if len(keys) != self._num_keys:
            raise RedisScriptError(
                f"Key length mismatch. Expected {self._num_keys}. "
                f"Got {len(keys)} in {self._name}")

        if kwargs:
            raise TypeError(
                f"Unexpected keyword arguments {kwargs.keys()} in {self._name}."
            )
        try:
            return redis.evalsha(self._script_digest, self._num_keys,
                                 *keys + args)
        except NoScriptError:
            return redis.eval(self._script_encoded, self._num_keys,
                              *keys + args)