def _process_one_node(node_name,
node_id,
timestamp_cutoff,
dedupe_set,
collection_ops_accounting_rows,
new_dedupes,
node_keys_processed):
log = logging.getLogger("_process_one_node")
redis_connection = create_redis_connection(host=node_name)
search_key = compute_search_key(node_name)
keys = redis_connection.keys(search_key)
log.debug("found {0} keys from {1}".format(len(keys), search_key))
value_dict = dict()
for key_bytes in keys:
key = key_bytes.decode("utf-8")
node_name, timestamp, partial_key = parse_key(key)
if timestamp > timestamp_cutoff:
log.debug("ignoring recent key {0}".format(key))
continue
node_keys_processed.append(key)
if key in dedupe_set:
log.debug("ignoring duplicate key {0}".format(key))
continue
log.info("node = {0}, key = {1}".format(node_name, key))
hash_dict = redis_connection.hgetall(key)
for collection_id_bytes, count_bytes in hash_dict.items():
collection_id = int(collection_id_bytes)
count = int(count_bytes)
log.info(" collection_id = {0}, count = {1}".format(
collection_id, count))
value_key = (timestamp, collection_id, )
if not value_key in value_dict:
value_dict[value_key] = \
_collection_ops_accounting_row(node_id,
collection_id,
timestamp)
value_dict[value_key][partial_key] += count
new_dedupes.append((node_id, key, ))
collection_ops_accounting_rows.extend(value_dict.values())
def tearDown(self):
log = logging.getLogger("teardown")
log.info("1")
if hasattr(self, "_halt_event"):
self._halt_event.set()
log.info("2")
if hasattr(self, "redis_sink"):
self._redis_sink.join()
log.info("3")
if hasattr(self, "_redis_queue"):
delattr(self, "_redis_queue")
log.info("4")
if hasattr(self, "_halt_event"):
delattr(self, "_halt_event")
log.info("5")
if hasattr(self, "redis_sink"):
delattr(self, "redis_sink")
log.info("6")
for key in self._redis_connection.keys(compute_search_key(_node_name)):
self._redis_connection.delete(key)
if hasattr(self, "_redis_connection"):
delattr(self, "_redis_connection")
log.info("7")
def tearDown(self):
log = logging.getLogger("teardown")
log.info("1")
if hasattr(self, "_halt_event"):
self._halt_event.set()
log.info("2")
if hasattr(self, "redis_sink"):
self._redis_sink.join()
log.info("3")
if hasattr(self, "_redis_queue"):
delattr(self, "_redis_queue")
log.info("4")
if hasattr(self, "_halt_event"):
delattr(self, "_halt_event")
log.info("5")
if hasattr(self, "redis_sink"):
delattr(self, "redis_sink")
log.info("6")
for key in self._redis_connection.keys(compute_search_key(_node_name)):
self._redis_connection.delete(key)
if hasattr(self, "_redis_connection"):
delattr(self, "_redis_connection")
log.info("7")
