def _handle_finish_conjoined_archive(self, message, _data): log = logging.getLogger("_handle_finish_conjoined_archive") log.info("request {0}: {1} {2} {3} {4}".format( message["user-request-id"], message["collection-id"], message["key"], message["unified-id"], message["timestamp-repr"], )) timestamp = parse_timestamp_repr(message["timestamp-repr"]) if "handoff-node-name" not in message or \ message["handoff-node-name"] is None: handoff_node_id = None else: handoff_node_id = self._node_id_dict[message["handoff-node-name"]] self._writer.finish_conjoined_archive( message["collection-id"], message["key"], message["unified-id"], timestamp, handoff_node_id) reply = { "message-type" : "finish-conjoined-archive-reply", "client-tag" : message["client-tag"], "client-address" : message["client-address"], "user-request-id" : message["user-request-id"], "message-id" : message["message-id"], "result" : "success", "error-message" : None, } self._reply_pusher.send(reply)
def _handle_finish_conjoined_archive(self, message, _data): log = logging.getLogger("_handle_finish_conjoined_archive") log.info("request {0}: {1} {2} {3} {4}".format( message["user-request-id"], message["collection-id"], message["key"], message["unified-id"], message["timestamp-repr"], )) timestamp = parse_timestamp_repr(message["timestamp-repr"]) if "handoff-node-name" not in message or \ message["handoff-node-name"] is None: handoff_node_id = None else: handoff_node_id = self._node_id_dict[message["handoff-node-name"]] self._writer.finish_conjoined_archive(message["collection-id"], message["key"], message["unified-id"], timestamp, handoff_node_id) reply = { "message-type": "finish-conjoined-archive-reply", "client-tag": message["client-tag"], "client-address": message["client-address"], "user-request-id": message["user-request-id"], "message-id": message["message-id"], "result": "success", "error-message": None, } self._reply_pusher.send(reply)
def _handle_web_server_start(state, message, _data): log = logging.getLogger("_handle_web_server_start") log.info("%s %s %s" % (message["unified-id"], message["timestamp-repr"], message["source-node-name"])) source_node_id = state["node-id-dict"][message["source-node-name"]] timestamp = parse_timestamp_repr(message["timestamp-repr"]) state["writer"].cancel_active_archives_from_node(source_node_id, timestamp)
def _handle_destroy_key(self, message, _data): log = logging.getLogger("_handle_destroy_key") log.info("request {0}: {1} {2} {3} {4} {5}".format( message["user-request-id"], message["collection-id"], message["key"], message["unified-id-to-delete"], message["unified-id"], message["segment-num"])) timestamp = parse_timestamp_repr(message["timestamp-repr"]) source_node_id = self._node_id_dict[message["source-node-name"]] if message["handoff-node-name"] is None: handoff_node_id = None else: handoff_node_id = self._node_id_dict[message["handoff-node-name"]] self._writer.set_tombstone(message["collection-id"], message["key"], message["unified-id-to-delete"], message["unified-id"], timestamp, message["segment-num"], source_node_id, handoff_node_id, message["user-request-id"]) reply = { "message-type": "destroy-key-reply", "client-tag": message["client-tag"], "client-address": message["client-address"], "user-request-id": message["user-request-id"], "message-id": message["message-id"], "unified-id": message["unified-id"], "result": "success", "error-message": None, } self._reply_pusher.send(reply)
def _handle_web_writer_start(self, message, _data): log = logging.getLogger("_handle_web_writer_start") log.info("{0} {1} {2}".format(message["unified_id"], message["timestamp_repr"], message["source_node_name"])) source_node_id = self._node_id_dict[message["source_node_name"]] timestamp = parse_timestamp_repr(message["timestamp_repr"]) self._writer.cancel_active_archives_from_node(source_node_id, timestamp)
def _handle_web_writer_start(self, message, _data): log = logging.getLogger("_handle_web_writer_start") log.info("{0} {1} {2}".format(message["unified_id"], message["timestamp_repr"], message["source_node_name"])) source_node_id = self._node_id_dict[message["source_node_name"]] timestamp = parse_timestamp_repr(message["timestamp_repr"]) self._writer.cancel_active_archives_from_node( source_node_id, timestamp )
def _handle_space_accounting_detail(state, message, _data): log = logging.getLogger("_handle_space_accounting_detail") message_datetime = parse_timestamp_repr(message["timestamp-repr"]) message_hour = floor_hour(message_datetime) log.info("hour = %s collection_id = %s, event = %s, value = %s" % ( message_hour, message["collection-id"], message["event"], message["value"] )) hour_entry = state["data"].setdefault(message_hour, dict()) collection_entry = hour_entry.setdefault(message["collection-id"], dict()) collection_entry[message["event"]] = \ collection_entry.setdefault(message["event"], 0) + message["value"]
def _handle_finish_conjoined_archive(state, message, _data): log = logging.getLogger("_handle_finish_conjoined_archive") log.info( "%r %r %s %s" % (message["collection-id"], message["key"], message["unified-id"], message["timestamp-repr"]) ) timestamp = parse_timestamp_repr(message["timestamp-repr"]) state["writer"].finish_conjoined_archive(message["collection-id"], message["key"], message["unified-id"], timestamp) reply = { "message-type": "finish-conjoined-archive-reply", "client-tag": message["client-tag"], "message-id": message["message-id"], "result": "success", "error-message": None, } state["resilient-server"].send_reply(reply)
def _finish_new_segment( self, collection_id, unified_id, timestamp_repr, conjoined_part, segment_num, file_size, file_adler32, file_hash, meta_dict, ): """ finalize storing one segment of data for a file """ segment_key = (unified_id, conjoined_part, segment_num, ) self._log.info("finish_new_segment %s %s" % ( unified_id, segment_num, )) segment_entry = self._active_segments.pop(segment_key) timestamp = parse_timestamp_repr(timestamp_repr) meta_rows = list() for meta_key, meta_value in meta_dict.items(): meta_row = meta_row_template( collection_id=collection_id, segment_id=segment_entry["segment-id"], meta_key=meta_key, meta_value=meta_value, timestamp=timestamp ) meta_rows.append(meta_row) _finalize_segment_row( self._connection, segment_entry["segment-id"], file_size, file_adler32, file_hash, meta_rows )
def start_new_segment( self, collection_id, key, unified_id, timestamp_repr, conjoined_part, segment_num, source_node_id, handoff_node_id, user_request_id ): """ Initiate storing a segment of data for a file """ segment_key = (unified_id, conjoined_part, segment_num, ) self._log.info("request {0}: " \ "start_new_segment {1} {2} {3} {4} {5} {6} {7}".format( user_request_id, collection_id, key, unified_id, timestamp_repr, conjoined_part, segment_num, source_node_id,)) if segment_key in self._active_segments: raise ValueError("duplicate segment %s" % (segment_key, )) timestamp = parse_timestamp_repr(timestamp_repr) self._active_segments[segment_key] = { "segment-id" : _insert_new_segment_row(self._connection, collection_id, unified_id, key, timestamp, conjoined_part, segment_num, source_node_id, handoff_node_id), }
def _handle_destroy_key(self, message, _data): log = logging.getLogger("_handle_destroy_key") log.info("request {0}: {1} {2} {3} {4} {5}".format( message["user-request-id"], message["collection-id"], message["key"], message["unified-id-to-delete"], message["unified-id"], message["segment-num"] )) timestamp = parse_timestamp_repr(message["timestamp-repr"]) source_node_id = self._node_id_dict[message["source-node-name"]] if message["handoff-node-name"] is None: handoff_node_id = None else: handoff_node_id = self._node_id_dict[message["handoff-node-name"]] self._writer.set_tombstone( message["collection-id"], message["key"], message["unified-id-to-delete"], message["unified-id"], timestamp, message["segment-num"], source_node_id, handoff_node_id, message["user-request-id"] ) reply = { "message-type" : "destroy-key-reply", "client-tag" : message["client-tag"], "client-address" : message["client-address"], "user-request-id" : message["user-request-id"], "message-id" : message["message-id"], "unified-id" : message["unified-id"], "result" : "success", "error-message" : None, } self._reply_pusher.send(reply)
def _handle_destroy_key(state, message, _data): log = logging.getLogger("_handle_destroy_key") log.info( "%s %s %s %s %s" % ( message["collection-id"], message["key"], message["unified-id-to-delete"], message["unified-id"], message["segment-num"], ) ) timestamp = parse_timestamp_repr(message["timestamp-repr"]) source_node_id = state["node-id-dict"][message["source-node-name"]] if message["handoff-node-name"] is None: handoff_node_id = None else: handoff_node_id = state["node-id-dict"][message["handoff-node-name"]] state["writer"].set_tombstone( message["collection-id"], message["key"], message["unified-id-to-delete"], message["unified-id"], timestamp, message["segment-num"], source_node_id, handoff_node_id, ) reply = { "message-type": "destroy-key-reply", "client-tag": message["client-tag"], "message-id": message["message-id"], "result": "success", "error-message": None, } state["resilient-server"].send_reply(reply)
def _handle_consistency_check_reply(state, message, _data): log = logging.getLogger("_handle_consistency_check_reply") timestamp = parse_timestamp_repr(message["timestamp-repr"]) state_key = (message["collection-id"], timestamp, ) try: request_state = state["active-requests"][state_key] except KeyError: log.warn("Unknown state_key %s from %s" % ( state_key, message["node-name"] )) return if message["node-name"] in request_state.replies: error_message = "duplicate reply from %s %s" % ( message["node-name"], state_key, ) log.error(error_message) return if message["result"] != "success": log.error("%s (%s) %s from %s" % ( state_key, message["result"], message["error-message"], message["node-name"], )) reply_value = _error_reply else: reply_value = (message["count"], message["encoded-md5-digest"], ) request_state.replies[message["node-name"]] = reply_value # not done yet, wait for more replies if len(request_state.replies) < len(state["anti-entropy-clients"]): return # at this point we should have a reply from every node, so # we don't want to preserve state anymore del state["active-requests"][state_key] database = AuditResultDatabase(state["central-database-connection"]) timestamp = create_timestamp() # push the results into a dict to see how many unique entries there are md5_digest_dict = dict() md5_digest_dict[_error_reply] = list() for node_name in request_state.replies.keys(): node_reply = request_state.replies[node_name] if node_reply == _error_reply: md5_digest_dict[_error_reply].append(node_name) continue _count, encoded_md5_digest = node_reply if not encoded_md5_digest in md5_digest_dict: md5_digest_dict[encoded_md5_digest] = list() md5_digest_dict[encoded_md5_digest].append(node_name) # if this audit was started by an anti-entropy-audit-request message, # we want to send a reply if request_state.client_tag is not None: reply = { "message-type" : "anti-entropy-audit-reply", "client-tag" : request_state.client_tag, "collection-id" : message["collection-id"], "result" : None, "error-message" : None, } else: reply = None error_reply_list = md5_digest_dict.pop(_error_reply) if reply is not None: reply["error-reply-nodes"] = error_reply_list if len(md5_digest_dict) > 1: log.error("found %s different hashes for (%s)" % ( len(md5_digest_dict), message["collection-id"], )) for index, value in enumerate(md5_digest_dict.values()): log.info(str(value)) if reply is not None: reply["mistmatch-nodes-%s" % (index+1, )] = value # ok = no errors and all nodes have the same hash for every collection if len(error_reply_list) == 0 and len(md5_digest_dict) == 1: description = "collection %s compares ok" % ( message["collection-id"], ) log.info(description) state["event-push-client"].info( "audit-ok", description, collection_id=message["collection-id"] ) database.successful_audit(request_state.row_id, timestamp) if reply is not None: reply["result"] = "success" state["resilient-server"].send_reply(reply) return # we have error(s), but the non-errors compare ok if len(error_reply_list) > 0 and len(md5_digest_dict) == 1: # if we come from anti-entropy-audit-request, don't retry if reply is not None: database.audit_error(request_state.row_id, timestamp) database.close() description = "There were error replies from %s nodes" % ( len(error_reply_list) , ) log.error(description) state["event-push-client"].error( "consistency-check-errors-replies", description, collection_id=message["collection-id"], error_reply_nodes=error_reply_list ) reply["result"] = "error" reply["error-message"] = description state["resilient-server"].send_reply(reply) return if request_state.retry_count >= max_retry_count: description = "collection %s %s errors, too many retries" % ( message["collection-id"], len(error_reply_list) ) log.error(description) state["event-push-client"].error( "audit-errors", description, collection_id=message["collection-id"] ) database.audit_error(request_state.row_id, timestamp) # TODO: needto do something here else: description = "%s Error replies from %s nodes, will retry" % ( message["collection-id"], len(error_reply_list) ) log.warn(description) state["event-push-client"].warn( "audit-retry", description, collection_id=message["collection-id"] ) state["retry-list"].append( retry_entry_tuple( retry_time=retry_time(), collection_id=message["collection-id"], row_id=request_state.row_id, retry_count=request_state.retry_count, ) ) database.wait_for_retry(request_state.row_id) database.close() return # if we make it here, we have some form of mismatch, possibly mixed with # errors description = "%s error replies from %s nodes; hash mismatch(es) = %r" % ( message["collection-id"], len(error_reply_list), md5_digest_dict.values() ) log.error(description) state["event-push-client"].warn( "audit-retry", description, collection_id=message["collection-id"] ) # if we come from anti-entropy-audit-request, don't retry if reply is not None: database.audit_error(request_state.row_id, timestamp) database.close() reply["result"] = "audit-error" reply["error-message"] = description state["resilient-server"].send_reply(reply) return if request_state.retry_count >= max_retry_count: log.error("%s too many retries" % (message["collection-id"], )) database.audit_error(request_state.row_id, timestamp) # TODO: need to do something here else: state["retry-list"].append( retry_entry_tuple( retry_time=retry_time(), collection_id=message["collection-id"], row_id=request_state.row_id, retry_count=request_state.retry_count, ) ) database.wait_for_retry(request_state.row_id) database.close()