def ping_every5s(self):
"""Method called every 5 second."""
if not self.redis_started:
return
self.fill_watchers()
for watcher in self.watchers:
if not watcher.startswith("step."):
continue
key = watcher.replace('step.', 'latest.', 1)
latest = self.exception_safe_redis_get(key)
if latest is None or \
((get_utc_unix_timestamp() - int(latest)) > 300):
if self.is_watcher_active(watcher):
log.info("stopping watcher %s (inactivity)" % watcher)
self.call("stop", name=watcher)
def __run_in_daemon_mode(self,
redis_host,
redis_port,
redis_queue,
redis_unix_socket_path=None):
self.info("Start the daemon mode with redis_queue=%s", redis_queue)
if redis_unix_socket_path:
r = redis.StrictRedis(unix_socket_path=redis_unix_socket_path)
else:
r = redis.StrictRedis(host=redis_host, port=redis_port)
redis_connection_exceptions_counter = 0
while not self.stop_flag:
self.__ping_if_needed()
try:
msg = r.brpop(redis_queue, 1)
except KeyboardInterrupt:
self.stop_flag = True
continue
except redis.exceptions.ConnectionError:
redis_connection_exceptions_counter = \
redis_connection_exceptions_counter + 1
if redis_connection_exceptions_counter >= 10:
self.critical("Can't connect to redis after 10s => exit")
self.stop_flag = True
continue
else:
self.debug("Can't connect to redis => "
"I will try again after 1s sleep")
time.sleep(1)
continue
redis_connection_exceptions_counter = 0
if msg is None:
# brpop timeout
continue
try:
decoded_msg = json.loads(msg[1].decode('utf8'))
filepath = os.path.join(decoded_msg['directory'],
decoded_msg['filename'])
except Exception:
self.warning("wrong message type popped on bus "
"=> stopping to force a restart")
self.stop_flag = True
continue
if not os.path.exists(filepath):
self.debug(
"filepath: %s does not exist anymore "
"=> ignoring event", filepath)
continue
xaf = xattrfile.XattrFile(filepath)
counter = "counter.%s.%s" % (self.plugin_name, self.step_name)
latest = "latest.%s.%s" % (self.plugin_name, self.step_name)
try:
r.incr(counter)
except Exception:
self.warning("can't update redis counter: %s" % counter)
try:
r.set(latest, str(get_utc_unix_timestamp()))
except Exception:
self.warning("can't set latest timestamp: %s" % latest)
self._process(xaf)
self.debug("Stop to brpop queue %s" % redis_queue)
def test_get_utc_unix_timestamp(self):
x = get_utc_unix_timestamp()
self.assertTrue(x > 1513000000)
def test_get_utc_unix_timestamp(self):
x = get_utc_unix_timestamp()
self.assertTrue(x > 1513000000)