Ejemplo n.º 1
0
 def _load_backend_decorator(inpt):
     _backend = known_backends.get(inpt, inpt)
     try:
         backend = util.load_obj_from_path(
             _backend,
             {'key': '%s_backend' % backend_type, backend_type: _backend})
     except Exception as err:
         log.error(
             "Could not load %s backend. err: %s" % (backend_type, err),
             extra={'%s_backend' % backend_type: _backend,
                    'err_kls': type(err)})
         raise
     return backend
Ejemplo n.º 2
0
def _get_parent_parsers(objects):
    """Internal function to call m.build_arg_parser() for each m in objects"""
    for m in set(objects):
        if not isinstance(m, argparse.ArgumentParser):
            p = m.build_arg_parser()
            if not isinstance(p, argparse.ArgumentParser):
                msg = ("Failed to initialize Stolos because the initializer"
                       " expected an instance of argparse.ArgumentParser but"
                       " received something else")
                log.error(msg, extra=dict(unrecognized_object=p))
                raise TypeError("%s.  Received: %s" % (msg, type(p)))
        else:
            p = m
        yield p
Ejemplo n.º 3
0
def _get_parent_parsers(objects):
    """Internal function to call m.build_arg_parser() for each m in objects"""
    for m in set(objects):
        if not isinstance(m, argparse.ArgumentParser):
            p = m.build_arg_parser()
            if not isinstance(p, argparse.ArgumentParser):
                msg = (
                    "Failed to initialize Stolos because the initializer"
                    " expected an instance of argparse.ArgumentParser but"
                    " received something else")
                log.error(msg, extra=dict(unrecognized_object=p))
                raise TypeError("%s.  Received: %s" % (msg, type(p)))
        else:
            p = m
        yield p
Ejemplo n.º 4
0
    def _extend_lock_in_background(self):
        """
        background signal (not a thread) that keeps lock alive
        """
        while True:
            try:
                s = time.time()
                threads = []
                results = []
                for path in list(self.LOCKS):
                    try:
                        client_id = self.LOCKS[path]
                    except KeyError:
                        continue  # race condition optimization
                    t = threading.Thread(
                        name="%s Extend %s %s" % (
                            self.__class__.__name__, path, client_id),
                        target=self._extend, args=(path, client_id, results))
                    t.daemon = True
                    t.start()
                    threads.append(t)
                for t in threads:
                    t.join()
                if len(threads) != len(results):
                    raise Exception("Failed to extend lock")
                for x in results:
                    if x[2] != 1:
                        raise Exception(
                            "Failed to extend lock for path: %s. redis_msg: %s"
                            % (x[0], x[2]))

                # adjust sleep time based on min expireat
                delta = time.time() - s
                sleep_time = \
                    self._lock_timeout - self._max_network_delay - delta
                assert self._lock_timeout > sleep_time > 0, (
                    'took too long to extend the locks.'
                    ' increase lock_timeout or reduce number of concurrent'
                    ' locks you have or improve network latency')
                time.sleep(sleep_time)
            except Exception as err:
                # kill parent process
                log.error(
                    'Redis Queue Backend asking Stolos to exit(1)'
                    ' because of err: %s' % err)
                os.kill(os.getpid(), BaseStolosRedis._SIGNAL)
                raise
Ejemplo n.º 5
0
    def _extend_lock_in_background(self):
        """
        background signal (not a thread) that keeps lock alive
        """
        while True:
            try:
                s = time.time()
                threads = []
                results = []
                for path in list(self.LOCKS):
                    try:
                        client_id = self.LOCKS[path]
                    except KeyError:
                        continue  # race condition optimization
                    t = threading.Thread(
                        name="%s Extend %s %s" % (
                            self.__class__.__name__, path, client_id),
                        target=self._extend, args=(path, client_id, results))
                    t.daemon = True
                    t.start()
                    threads.append(t)
                for t in threads:
                    t.join()
                if len(threads) != len(results):
                    raise Exception("Failed to extend lock")
                for x in results:
                    if x[2] != 1:
                        raise Exception(
                            "Failed to extend lock for path: %s. redis_msg: %s"
                            % (x[0], x[2]))

                # adjust sleep time based on min expireat
                delta = time.time() - s
                sleep_time = \
                    self._lock_timeout - self._max_network_delay - delta
                assert self._lock_timeout > sleep_time > 0, (
                    'took too long to extend the locks.'
                    ' increase lock_timeout or reduce number of concurrent'
                    ' locks you have or improve network latency')
                time.sleep(sleep_time)
            except Exception as err:
                # kill parent process
                log.error(
                    'Redis Queue Backend asking Stolos to exit(1)'
                    ' because of err: %s' % err)
                os.kill(os.getpid(), BaseStolosRedis._SIGNAL)
                raise
Ejemplo n.º 6
0
def validate_job_id(app_name, job_id, q, timeout):
    """Return True if valid job_id.
    If invalid, do whatever cleanup for this job is necessary and return False.
      --> necessary cleanup may include removing this job_id from queue
    """
    if job_id is None:
        log.info('No jobs found in %d seconds...' % timeout,
                 extra=dict(app_name=app_name))
        return False
    try:
        dt.parse_job_id(app_name, job_id)
    except exceptions.InvalidJobId as err:
        log.error(
            ("Stolos found an invalid job_id.  Removing it from queue"
             " and marking that job_id as failed.  Error details: %s") % err,
            extra=dict(app_name=app_name, job_id=job_id))
        q.consume()
        qb._set_state_unsafe(app_name, job_id, failed=True)
        return False
    return True
Ejemplo n.º 7
0
def validate_job_id(app_name, job_id, q, timeout):
    """Return True if valid job_id.
    If invalid, do whatever cleanup for this job is necessary and return False.
      --> necessary cleanup may include removing this job_id from queue
    """
    if job_id is None:
        log.info('No jobs found in %d seconds...' % timeout, extra=dict(
            app_name=app_name))
        return False
    try:
        dt.parse_job_id(app_name, job_id)
    except exceptions.InvalidJobId as err:
        log.error((
            "Stolos found an invalid job_id.  Removing it from queue"
            " and marking that job_id as failed.  Error details: %s") % err,
            extra=dict(app_name=app_name, job_id=job_id))
        q.consume()
        qb._set_state_unsafe(
            app_name, job_id, failed=True)
        return False
    return True
Ejemplo n.º 8
0
    def release(self):
        """
        Release a lock at the Lock's path.
        Return True if success.  Raise UserWarning otherwise, possibly due to:
            - did not release a lock
            - lock already released
            - lock does not exist (perhaps it was never acquired)
        """
        if self.LOCKS.get(self._path) != self._client_id:
            raise UserWarning("You must acquire lock before releasing it")
        self.LOCKS.pop(self._path)

        try:
            rv = raw_client().evalsha(self._SHAS['l_unlock'],
                                      len(self.SCRIPTS['l_unlock']['keys']),
                                      self._path, self._client_id)
            assert rv == 1
        except AssertionError:
            raise UserWarning("Lock did not exist on Redis server")
        except Exception as err:
            msg = "Could not release lock.  Got error: %s" % err
            log.error(msg, extra=dict(lock_path=self._path))
            raise UserWarning(msg)
Ejemplo n.º 9
0
    def release(self):
        """
        Release a lock at the Lock's path.
        Return True if success.  Raise UserWarning otherwise, possibly due to:
            - did not release a lock
            - lock already released
            - lock does not exist (perhaps it was never acquired)
        """
        if self.LOCKS.get(self._path) != self._client_id:
            raise UserWarning("You must acquire lock before releasing it")
        self.LOCKS.pop(self._path)

        try:
            rv = raw_client().evalsha(
                self._SHAS['l_unlock'],
                len(self.SCRIPTS['l_unlock']['keys']),
                self._path, self._client_id)
            assert rv == 1
        except AssertionError:
            raise UserWarning("Lock did not exist on Redis server")
        except Exception as err:
            msg = "Could not release lock.  Got error: %s" % err
            log.error(msg, extra=dict(lock_path=self._path))
            raise UserWarning(msg)
Ejemplo n.º 10
0
def _log_raise(msg, extra, exception_kls):
    log.error(msg, extra=extra)
    raise exception_kls(msg)