def _load_backend_decorator(inpt):
_backend = known_backends.get(inpt, inpt)
try:
backend = util.load_obj_from_path(
_backend,
{'key': '%s_backend' % backend_type, backend_type: _backend})
except Exception as err:
log.error(
"Could not load %s backend. err: %s" % (backend_type, err),
extra={'%s_backend' % backend_type: _backend,
'err_kls': type(err)})
raise
return backend
def _get_parent_parsers(objects):
"""Internal function to call m.build_arg_parser() for each m in objects"""
for m in set(objects):
if not isinstance(m, argparse.ArgumentParser):
p = m.build_arg_parser()
if not isinstance(p, argparse.ArgumentParser):
msg = ("Failed to initialize Stolos because the initializer"
" expected an instance of argparse.ArgumentParser but"
" received something else")
log.error(msg, extra=dict(unrecognized_object=p))
raise TypeError("%s. Received: %s" % (msg, type(p)))
else:
p = m
yield p
def _get_parent_parsers(objects):
"""Internal function to call m.build_arg_parser() for each m in objects"""
for m in set(objects):
if not isinstance(m, argparse.ArgumentParser):
p = m.build_arg_parser()
if not isinstance(p, argparse.ArgumentParser):
msg = (
"Failed to initialize Stolos because the initializer"
" expected an instance of argparse.ArgumentParser but"
" received something else")
log.error(msg, extra=dict(unrecognized_object=p))
raise TypeError("%s. Received: %s" % (msg, type(p)))
else:
p = m
yield p
def _extend_lock_in_background(self):
"""
background signal (not a thread) that keeps lock alive
"""
while True:
try:
s = time.time()
threads = []
results = []
for path in list(self.LOCKS):
try:
client_id = self.LOCKS[path]
except KeyError:
continue # race condition optimization
t = threading.Thread(
name="%s Extend %s %s" % (
self.__class__.__name__, path, client_id),
target=self._extend, args=(path, client_id, results))
t.daemon = True
t.start()
threads.append(t)
for t in threads:
t.join()
if len(threads) != len(results):
raise Exception("Failed to extend lock")
for x in results:
if x[2] != 1:
raise Exception(
"Failed to extend lock for path: %s. redis_msg: %s"
% (x[0], x[2]))
# adjust sleep time based on min expireat
delta = time.time() - s
sleep_time = \
self._lock_timeout - self._max_network_delay - delta
assert self._lock_timeout > sleep_time > 0, (
'took too long to extend the locks.'
' increase lock_timeout or reduce number of concurrent'
' locks you have or improve network latency')
time.sleep(sleep_time)
except Exception as err:
# kill parent process
log.error(
'Redis Queue Backend asking Stolos to exit(1)'
' because of err: %s' % err)
os.kill(os.getpid(), BaseStolosRedis._SIGNAL)
raise
def _extend_lock_in_background(self):
"""
background signal (not a thread) that keeps lock alive
"""
while True:
try:
s = time.time()
threads = []
results = []
for path in list(self.LOCKS):
try:
client_id = self.LOCKS[path]
except KeyError:
continue # race condition optimization
t = threading.Thread(
name="%s Extend %s %s" % (
self.__class__.__name__, path, client_id),
target=self._extend, args=(path, client_id, results))
t.daemon = True
t.start()
threads.append(t)
for t in threads:
t.join()
if len(threads) != len(results):
raise Exception("Failed to extend lock")
for x in results:
if x[2] != 1:
raise Exception(
"Failed to extend lock for path: %s. redis_msg: %s"
% (x[0], x[2]))
# adjust sleep time based on min expireat
delta = time.time() - s
sleep_time = \
self._lock_timeout - self._max_network_delay - delta
assert self._lock_timeout > sleep_time > 0, (
'took too long to extend the locks.'
' increase lock_timeout or reduce number of concurrent'
' locks you have or improve network latency')
time.sleep(sleep_time)
except Exception as err:
# kill parent process
log.error(
'Redis Queue Backend asking Stolos to exit(1)'
' because of err: %s' % err)
os.kill(os.getpid(), BaseStolosRedis._SIGNAL)
raise
def validate_job_id(app_name, job_id, q, timeout):
"""Return True if valid job_id.
If invalid, do whatever cleanup for this job is necessary and return False.
--> necessary cleanup may include removing this job_id from queue
"""
if job_id is None:
log.info('No jobs found in %d seconds...' % timeout,
extra=dict(app_name=app_name))
return False
try:
dt.parse_job_id(app_name, job_id)
except exceptions.InvalidJobId as err:
log.error(
("Stolos found an invalid job_id. Removing it from queue"
" and marking that job_id as failed. Error details: %s") % err,
extra=dict(app_name=app_name, job_id=job_id))
q.consume()
qb._set_state_unsafe(app_name, job_id, failed=True)
return False
return True
def validate_job_id(app_name, job_id, q, timeout):
"""Return True if valid job_id.
If invalid, do whatever cleanup for this job is necessary and return False.
--> necessary cleanup may include removing this job_id from queue
"""
if job_id is None:
log.info('No jobs found in %d seconds...' % timeout, extra=dict(
app_name=app_name))
return False
try:
dt.parse_job_id(app_name, job_id)
except exceptions.InvalidJobId as err:
log.error((
"Stolos found an invalid job_id. Removing it from queue"
" and marking that job_id as failed. Error details: %s") % err,
extra=dict(app_name=app_name, job_id=job_id))
q.consume()
qb._set_state_unsafe(
app_name, job_id, failed=True)
return False
return True
def release(self):
"""
Release a lock at the Lock's path.
Return True if success. Raise UserWarning otherwise, possibly due to:
- did not release a lock
- lock already released
- lock does not exist (perhaps it was never acquired)
"""
if self.LOCKS.get(self._path) != self._client_id:
raise UserWarning("You must acquire lock before releasing it")
self.LOCKS.pop(self._path)
try:
rv = raw_client().evalsha(self._SHAS['l_unlock'],
len(self.SCRIPTS['l_unlock']['keys']),
self._path, self._client_id)
assert rv == 1
except AssertionError:
raise UserWarning("Lock did not exist on Redis server")
except Exception as err:
msg = "Could not release lock. Got error: %s" % err
log.error(msg, extra=dict(lock_path=self._path))
raise UserWarning(msg)
def release(self):
"""
Release a lock at the Lock's path.
Return True if success. Raise UserWarning otherwise, possibly due to:
- did not release a lock
- lock already released
- lock does not exist (perhaps it was never acquired)
"""
if self.LOCKS.get(self._path) != self._client_id:
raise UserWarning("You must acquire lock before releasing it")
self.LOCKS.pop(self._path)
try:
rv = raw_client().evalsha(
self._SHAS['l_unlock'],
len(self.SCRIPTS['l_unlock']['keys']),
self._path, self._client_id)
assert rv == 1
except AssertionError:
raise UserWarning("Lock did not exist on Redis server")
except Exception as err:
msg = "Could not release lock. Got error: %s" % err
log.error(msg, extra=dict(lock_path=self._path))
raise UserWarning(msg)
def _log_raise(msg, extra, exception_kls):
log.error(msg, extra=extra)
raise exception_kls(msg)