def setUp(self):
self._halt_event = Event()
self._redis_queue = gevent.queue.Queue()
self._redis_sink = OperationalStatsRedisSink(self._halt_event, self._redis_queue, _node_name)
self._redis_sink.link_exception(_unhandled_greenlet_exception)
self._redis_sink.start()
self._redis_connection = create_redis_connection()
def setUp(self):
self._halt_event = Event()
self._redis_queue = gevent.queue.Queue()
self._redis_sink = OperationalStatsRedisSink(self._halt_event,
self._redis_queue,
_node_name)
self._redis_sink.link_exception(_unhandled_greenlet_exception)
self._redis_sink.start()
self._redis_connection = create_redis_connection()
def _run(self):
self._redis_connection = create_redis_connection()
self._log.debug("start halt_event loop")
while not self._halt_event.is_set():
try:
key, entry = self._redis_queue.get(block=True, timeout=1.0)
except gevent.queue.Empty:
continue
self.store(key, entry)
self._log.debug("end halt_event loop")
def _process_one_node(node_name,
node_id,
timestamp_cutoff,
dedupe_set,
collection_ops_accounting_rows,
new_dedupes,
node_keys_processed):
log = logging.getLogger("_process_one_node")
redis_connection = create_redis_connection(host=node_name)
search_key = compute_search_key(node_name)
keys = redis_connection.keys(search_key)
log.debug("found {0} keys from {1}".format(len(keys), search_key))
value_dict = dict()
for key_bytes in keys:
key = key_bytes.decode("utf-8")
node_name, timestamp, partial_key = parse_key(key)
if timestamp > timestamp_cutoff:
log.debug("ignoring recent key {0}".format(key))
continue
node_keys_processed.append(key)
if key in dedupe_set:
log.debug("ignoring duplicate key {0}".format(key))
continue
log.info("node = {0}, key = {1}".format(node_name, key))
hash_dict = redis_connection.hgetall(key)
for collection_id_bytes, count_bytes in hash_dict.items():
collection_id = int(collection_id_bytes)
count = int(count_bytes)
log.info(" collection_id = {0}, count = {1}".format(
collection_id, count))
value_key = (timestamp, collection_id, )
if not value_key in value_dict:
value_dict[value_key] = \
_collection_ops_accounting_row(node_id,
collection_id,
timestamp)
value_dict[value_key][partial_key] += count
new_dedupes.append((node_id, key, ))
collection_ops_accounting_rows.extend(value_dict.values())
def _remove_processed_keys(node_name, keys_processed):
log = logging.getLogger("_remove_processed_keys")
redis_connection = create_redis_connection(host=node_name)
for key in keys_processed:
log.info("removing {0}".format(key))
redis_connection.delete(key)
