def _handle_finish_conjoined_archive(self, message, _data):
log = logging.getLogger("_handle_finish_conjoined_archive")
log.info("request {0}: {1} {2} {3} {4}".format(
message["user-request-id"],
message["collection-id"],
message["key"],
message["unified-id"],
message["timestamp-repr"],
))
timestamp = parse_timestamp_repr(message["timestamp-repr"])
if "handoff-node-name" not in message or \
message["handoff-node-name"] is None:
handoff_node_id = None
else:
handoff_node_id = self._node_id_dict[message["handoff-node-name"]]
self._writer.finish_conjoined_archive(
message["collection-id"],
message["key"],
message["unified-id"],
timestamp,
handoff_node_id)
reply = {
"message-type" : "finish-conjoined-archive-reply",
"client-tag" : message["client-tag"],
"client-address" : message["client-address"],
"user-request-id" : message["user-request-id"],
"message-id" : message["message-id"],
"result" : "success",
"error-message" : None,
}
self._reply_pusher.send(reply)
def _handle_finish_conjoined_archive(self, message, _data):
log = logging.getLogger("_handle_finish_conjoined_archive")
log.info("request {0}: {1} {2} {3} {4}".format(
message["user-request-id"],
message["collection-id"],
message["key"],
message["unified-id"],
message["timestamp-repr"],
))
timestamp = parse_timestamp_repr(message["timestamp-repr"])
if "handoff-node-name" not in message or \
message["handoff-node-name"] is None:
handoff_node_id = None
else:
handoff_node_id = self._node_id_dict[message["handoff-node-name"]]
self._writer.finish_conjoined_archive(message["collection-id"],
message["key"],
message["unified-id"], timestamp,
handoff_node_id)
reply = {
"message-type": "finish-conjoined-archive-reply",
"client-tag": message["client-tag"],
"client-address": message["client-address"],
"user-request-id": message["user-request-id"],
"message-id": message["message-id"],
"result": "success",
"error-message": None,
}
self._reply_pusher.send(reply)
def _handle_web_server_start(state, message, _data):
log = logging.getLogger("_handle_web_server_start")
log.info("%s %s %s" % (message["unified-id"], message["timestamp-repr"], message["source-node-name"]))
source_node_id = state["node-id-dict"][message["source-node-name"]]
timestamp = parse_timestamp_repr(message["timestamp-repr"])
state["writer"].cancel_active_archives_from_node(source_node_id, timestamp)
def _handle_destroy_key(self, message, _data):
log = logging.getLogger("_handle_destroy_key")
log.info("request {0}: {1} {2} {3} {4} {5}".format(
message["user-request-id"], message["collection-id"],
message["key"], message["unified-id-to-delete"],
message["unified-id"], message["segment-num"]))
timestamp = parse_timestamp_repr(message["timestamp-repr"])
source_node_id = self._node_id_dict[message["source-node-name"]]
if message["handoff-node-name"] is None:
handoff_node_id = None
else:
handoff_node_id = self._node_id_dict[message["handoff-node-name"]]
self._writer.set_tombstone(message["collection-id"], message["key"],
message["unified-id-to-delete"],
message["unified-id"], timestamp,
message["segment-num"], source_node_id,
handoff_node_id, message["user-request-id"])
reply = {
"message-type": "destroy-key-reply",
"client-tag": message["client-tag"],
"client-address": message["client-address"],
"user-request-id": message["user-request-id"],
"message-id": message["message-id"],
"unified-id": message["unified-id"],
"result": "success",
"error-message": None,
}
self._reply_pusher.send(reply)
def _handle_web_writer_start(self, message, _data):
log = logging.getLogger("_handle_web_writer_start")
log.info("{0} {1} {2}".format(message["unified_id"],
message["timestamp_repr"],
message["source_node_name"]))
source_node_id = self._node_id_dict[message["source_node_name"]]
timestamp = parse_timestamp_repr(message["timestamp_repr"])
self._writer.cancel_active_archives_from_node(source_node_id,
timestamp)
def _handle_web_writer_start(self, message, _data):
log = logging.getLogger("_handle_web_writer_start")
log.info("{0} {1} {2}".format(message["unified_id"],
message["timestamp_repr"],
message["source_node_name"]))
source_node_id = self._node_id_dict[message["source_node_name"]]
timestamp = parse_timestamp_repr(message["timestamp_repr"])
self._writer.cancel_active_archives_from_node(
source_node_id, timestamp
)
def _handle_space_accounting_detail(state, message, _data):
log = logging.getLogger("_handle_space_accounting_detail")
message_datetime = parse_timestamp_repr(message["timestamp-repr"])
message_hour = floor_hour(message_datetime)
log.info("hour = %s collection_id = %s, event = %s, value = %s" % (
message_hour, message["collection-id"], message["event"], message["value"]
))
hour_entry = state["data"].setdefault(message_hour, dict())
collection_entry = hour_entry.setdefault(message["collection-id"], dict())
collection_entry[message["event"]] = \
collection_entry.setdefault(message["event"], 0) + message["value"]
def _handle_finish_conjoined_archive(state, message, _data):
log = logging.getLogger("_handle_finish_conjoined_archive")
log.info(
"%r %r %s %s" % (message["collection-id"], message["key"], message["unified-id"], message["timestamp-repr"])
)
timestamp = parse_timestamp_repr(message["timestamp-repr"])
state["writer"].finish_conjoined_archive(message["collection-id"], message["key"], message["unified-id"], timestamp)
reply = {
"message-type": "finish-conjoined-archive-reply",
"client-tag": message["client-tag"],
"message-id": message["message-id"],
"result": "success",
"error-message": None,
}
state["resilient-server"].send_reply(reply)
def _finish_new_segment(
self,
collection_id,
unified_id,
timestamp_repr,
conjoined_part,
segment_num,
file_size,
file_adler32,
file_hash,
meta_dict,
):
"""
finalize storing one segment of data for a file
"""
segment_key = (unified_id, conjoined_part, segment_num, )
self._log.info("finish_new_segment %s %s" % (
unified_id,
segment_num,
))
segment_entry = self._active_segments.pop(segment_key)
timestamp = parse_timestamp_repr(timestamp_repr)
meta_rows = list()
for meta_key, meta_value in meta_dict.items():
meta_row = meta_row_template(
collection_id=collection_id,
segment_id=segment_entry["segment-id"],
meta_key=meta_key,
meta_value=meta_value,
timestamp=timestamp
)
meta_rows.append(meta_row)
_finalize_segment_row(
self._connection,
segment_entry["segment-id"],
file_size,
file_adler32,
file_hash,
meta_rows
)
def start_new_segment(
self,
collection_id,
key,
unified_id,
timestamp_repr,
conjoined_part,
segment_num,
source_node_id,
handoff_node_id,
user_request_id
):
"""
Initiate storing a segment of data for a file
"""
segment_key = (unified_id, conjoined_part, segment_num, )
self._log.info("request {0}: " \
"start_new_segment {1} {2} {3} {4} {5} {6} {7}".format(
user_request_id,
collection_id,
key,
unified_id,
timestamp_repr,
conjoined_part,
segment_num,
source_node_id,))
if segment_key in self._active_segments:
raise ValueError("duplicate segment %s" % (segment_key, ))
timestamp = parse_timestamp_repr(timestamp_repr)
self._active_segments[segment_key] = {
"segment-id" : _insert_new_segment_row(self._connection,
collection_id,
unified_id,
key,
timestamp,
conjoined_part,
segment_num,
source_node_id,
handoff_node_id),
}
def _handle_destroy_key(self, message, _data):
log = logging.getLogger("_handle_destroy_key")
log.info("request {0}: {1} {2} {3} {4} {5}".format(
message["user-request-id"],
message["collection-id"],
message["key"],
message["unified-id-to-delete"],
message["unified-id"],
message["segment-num"]
))
timestamp = parse_timestamp_repr(message["timestamp-repr"])
source_node_id = self._node_id_dict[message["source-node-name"]]
if message["handoff-node-name"] is None:
handoff_node_id = None
else:
handoff_node_id = self._node_id_dict[message["handoff-node-name"]]
self._writer.set_tombstone(
message["collection-id"],
message["key"],
message["unified-id-to-delete"],
message["unified-id"],
timestamp,
message["segment-num"],
source_node_id,
handoff_node_id,
message["user-request-id"]
)
reply = {
"message-type" : "destroy-key-reply",
"client-tag" : message["client-tag"],
"client-address" : message["client-address"],
"user-request-id" : message["user-request-id"],
"message-id" : message["message-id"],
"unified-id" : message["unified-id"],
"result" : "success",
"error-message" : None,
}
self._reply_pusher.send(reply)
def _handle_destroy_key(state, message, _data):
log = logging.getLogger("_handle_destroy_key")
log.info(
"%s %s %s %s %s"
% (
message["collection-id"],
message["key"],
message["unified-id-to-delete"],
message["unified-id"],
message["segment-num"],
)
)
timestamp = parse_timestamp_repr(message["timestamp-repr"])
source_node_id = state["node-id-dict"][message["source-node-name"]]
if message["handoff-node-name"] is None:
handoff_node_id = None
else:
handoff_node_id = state["node-id-dict"][message["handoff-node-name"]]
state["writer"].set_tombstone(
message["collection-id"],
message["key"],
message["unified-id-to-delete"],
message["unified-id"],
timestamp,
message["segment-num"],
source_node_id,
handoff_node_id,
)
reply = {
"message-type": "destroy-key-reply",
"client-tag": message["client-tag"],
"message-id": message["message-id"],
"result": "success",
"error-message": None,
}
state["resilient-server"].send_reply(reply)
def _handle_consistency_check_reply(state, message, _data):
log = logging.getLogger("_handle_consistency_check_reply")
timestamp = parse_timestamp_repr(message["timestamp-repr"])
state_key = (message["collection-id"], timestamp, )
try:
request_state = state["active-requests"][state_key]
except KeyError:
log.warn("Unknown state_key %s from %s" % (
state_key, message["node-name"]
))
return
if message["node-name"] in request_state.replies:
error_message = "duplicate reply from %s %s" % (
message["node-name"],
state_key,
)
log.error(error_message)
return
if message["result"] != "success":
log.error("%s (%s) %s from %s" % (
state_key,
message["result"],
message["error-message"],
message["node-name"],
))
reply_value = _error_reply
else:
reply_value = (message["count"], message["encoded-md5-digest"], )
request_state.replies[message["node-name"]] = reply_value
# not done yet, wait for more replies
if len(request_state.replies) < len(state["anti-entropy-clients"]):
return
# at this point we should have a reply from every node, so
# we don't want to preserve state anymore
del state["active-requests"][state_key]
database = AuditResultDatabase(state["central-database-connection"])
timestamp = create_timestamp()
# push the results into a dict to see how many unique entries there are
md5_digest_dict = dict()
md5_digest_dict[_error_reply] = list()
for node_name in request_state.replies.keys():
node_reply = request_state.replies[node_name]
if node_reply == _error_reply:
md5_digest_dict[_error_reply].append(node_name)
continue
_count, encoded_md5_digest = node_reply
if not encoded_md5_digest in md5_digest_dict:
md5_digest_dict[encoded_md5_digest] = list()
md5_digest_dict[encoded_md5_digest].append(node_name)
# if this audit was started by an anti-entropy-audit-request message,
# we want to send a reply
if request_state.client_tag is not None:
reply = {
"message-type" : "anti-entropy-audit-reply",
"client-tag" : request_state.client_tag,
"collection-id" : message["collection-id"],
"result" : None,
"error-message" : None,
}
else:
reply = None
error_reply_list = md5_digest_dict.pop(_error_reply)
if reply is not None:
reply["error-reply-nodes"] = error_reply_list
if len(md5_digest_dict) > 1:
log.error("found %s different hashes for (%s)" % (
len(md5_digest_dict),
message["collection-id"],
))
for index, value in enumerate(md5_digest_dict.values()):
log.info(str(value))
if reply is not None:
reply["mistmatch-nodes-%s" % (index+1, )] = value
# ok = no errors and all nodes have the same hash for every collection
if len(error_reply_list) == 0 and len(md5_digest_dict) == 1:
description = "collection %s compares ok" % (
message["collection-id"],
)
log.info(description)
state["event-push-client"].info(
"audit-ok", description, collection_id=message["collection-id"]
)
database.successful_audit(request_state.row_id, timestamp)
if reply is not None:
reply["result"] = "success"
state["resilient-server"].send_reply(reply)
return
# we have error(s), but the non-errors compare ok
if len(error_reply_list) > 0 and len(md5_digest_dict) == 1:
# if we come from anti-entropy-audit-request, don't retry
if reply is not None:
database.audit_error(request_state.row_id, timestamp)
database.close()
description = "There were error replies from %s nodes" % (
len(error_reply_list) ,
)
log.error(description)
state["event-push-client"].error(
"consistency-check-errors-replies",
description,
collection_id=message["collection-id"],
error_reply_nodes=error_reply_list
)
reply["result"] = "error"
reply["error-message"] = description
state["resilient-server"].send_reply(reply)
return
if request_state.retry_count >= max_retry_count:
description = "collection %s %s errors, too many retries" % (
message["collection-id"],
len(error_reply_list)
)
log.error(description)
state["event-push-client"].error(
"audit-errors",
description,
collection_id=message["collection-id"]
)
database.audit_error(request_state.row_id, timestamp)
# TODO: needto do something here
else:
description = "%s Error replies from %s nodes, will retry" % (
message["collection-id"],
len(error_reply_list)
)
log.warn(description)
state["event-push-client"].warn(
"audit-retry",
description,
collection_id=message["collection-id"]
)
state["retry-list"].append(
retry_entry_tuple(
retry_time=retry_time(),
collection_id=message["collection-id"],
row_id=request_state.row_id,
retry_count=request_state.retry_count,
)
)
database.wait_for_retry(request_state.row_id)
database.close()
return
# if we make it here, we have some form of mismatch, possibly mixed with
# errors
description = "%s error replies from %s nodes; hash mismatch(es) = %r" % (
message["collection-id"],
len(error_reply_list),
md5_digest_dict.values()
)
log.error(description)
state["event-push-client"].warn(
"audit-retry",
description,
collection_id=message["collection-id"]
)
# if we come from anti-entropy-audit-request, don't retry
if reply is not None:
database.audit_error(request_state.row_id, timestamp)
database.close()
reply["result"] = "audit-error"
reply["error-message"] = description
state["resilient-server"].send_reply(reply)
return
if request_state.retry_count >= max_retry_count:
log.error("%s too many retries" % (message["collection-id"], ))
database.audit_error(request_state.row_id, timestamp)
# TODO: need to do something here
else:
state["retry-list"].append(
retry_entry_tuple(
retry_time=retry_time(),
collection_id=message["collection-id"],
row_id=request_state.row_id,
retry_count=request_state.retry_count,
)
)
database.wait_for_retry(request_state.row_id)
database.close()
