def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
status = validate_block(block)
if status is True:
LOGGER.debug("block passes signature verification %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
LOGGER.debug("block signature is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
elif gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
status = validate_batch(batch)
if status is True:
LOGGER.debug("batch passes signature verification %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
def handle(self, connection_id, message_content):
message = GossipMessage()
message.ParseFromString(message_content)
if message.content_type == "BLOCK":
public_key = \
self._network.connection_id_to_public_key(connection_id)
block = Block()
block.ParseFromString(message.content)
header = BlockHeader()
header.ParseFromString(block.header)
if header.signer_public_key == public_key:
permitted = \
self._permission_verifier.check_network_consensus_role(
public_key)
if not permitted:
LOGGER.debug(
"Public key is not permitted to publish block, "
"remove connection: %s", connection_id)
self._gossip.unregister_peer(connection_id)
violation = AuthorizationViolation(
violation=RoleType.Value("NETWORK"))
return HandlerResult(HandlerStatus.RETURN_AND_CLOSE,
message_out=violation,
message_type=validator_pb2.Message.
AUTHORIZATION_VIOLATION)
# if allowed pass message
return HandlerResult(HandlerStatus.PASS)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == gossip_message.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
has_block = False
if self._completer.get_block(block.header_signature) is not None:
has_block = True
if not has_block and self._has_block(block.header_signature):
has_block = True
if has_block:
LOGGER.debug("Drop duplicate block: %s",
block.header_signature)
return HandlerResult(HandlerStatus.DROP)
if gossip_message.content_type == gossip_message.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
has_batch = False
if self._completer.get_batch(batch.header_signature) is not None:
has_batch = True
if not has_batch and self._has_batch(batch.header_signature):
has_batch = True
if has_batch:
LOGGER.debug("Drop duplicate batch: %s",
batch.header_signature)
return HandlerResult(HandlerStatus.DROP)
return HandlerResult(HandlerStatus.PASS)
def handle(self, connection_id, message_content):
exclude = [connection_id]
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.time_to_live == 0:
# Do not forward message if it has reached its time to live limit
return HandlerResult(status=HandlerStatus.PASS)
else:
# decrement time_to_live
time_to_live = gossip_message.time_to_live
gossip_message.time_to_live = time_to_live - 1
if gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
# If we already have this batch, don't forward it
if not self._completer.get_batch(batch.header_signature):
self._gossip.broadcast_batch(batch, exclude)
elif gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
# If we already have this block, don't forward it
if not self._completer.get_block(block.header_signature):
self._gossip.broadcast_block(block, exclude)
else:
LOGGER.info("received %s, not BATCH or BLOCK",
gossip_message.content_type)
return HandlerResult(status=HandlerStatus.PASS)
def test_signature_thread(self):
try:
self.verifier.start()
blocks = self._create_blocks(1, 1)
message = GossipMessage(content_type="Block",
content=blocks[0].SerializeToString())
content = message.SerializeToString()
msg = validator_pb2.Message(
message_type=validator_pb2.Message.GOSSIP_MESSAGE,
correlation_id=self._generate_id(),
content=content)
with self._in_condition:
self._in.put_nowait(msg)
self._in_condition.notify_all()
to = TimeOut(2)
while self._out.qsize() == 0:
if to.is_timed_out():
break
self.assertEqual(self._out.qsize(), 1)
finally:
with self._in_condition:
self.verifier.stop()
self._in_condition.notify_all()
def handle(self, identity, message_content):
ack = NetworkAcknowledgement()
ack.status = ack.OK
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
return HandlerResult(HandlerStatus.RETURN_AND_PASS,
message_out=ack,
message_type=validator_pb2.Message.GOSSIP_ACK)
def test_no_block(self):
"""
Test that if the block does not exist yet in the completer or the
chain controller, the gossip message is passed.
"""
block = Block(header_signature="Block1")
message = GossipMessage(content_type=GossipMessage.BLOCK,
content=block.SerializeToString())
handler_status = self.handler.handle("connection_id",
message.SerializeToString())
self.assertEqual(handler_status.status, HandlerStatus.PASS)
def test_publisher_has_block(self):
"""
Test that if the block does not exist yet in the completer or the
chain controller, the gossip message is passed.
"""
batch = Batch(header_signature="Batch1")
self.publisher.add_batch("Batch1")
message = GossipMessage(content_type=GossipMessage.BATCH,
content=batch.SerializeToString())
handler_status = self.handler.handle(
"connection_id", message.SerializeToString())
self.assertEqual(handler_status.status, HandlerStatus.DROP)
def handle(self, identity, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
self._gossip.broadcast_batch(batch)
elif gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
self._gossip.broadcast_block(block)
else:
LOGGER.info("received %s, not BATCH or BLOCK",
gossip_message.content_type)
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
if block.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated block: %s cache=%s",
block.header_signature[:8], len(self._seen_cache)
) #[sig[:8] for sig in self._seen_cache])
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_block(block):
LOGGER.debug("block signature is invalid: %s",
block.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
self._seen_cache[block.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
if batch.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated batch: %s",
batch.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_batch(batch):
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
self._seen_cache[batch.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCHES:
# check batches signature
LOGGER.debug("Check BATCHES signature!!!")
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.ENDPOINTS:
# check endpoints
LOGGER.debug("Check ENDPOINTS signature!!!")
return HandlerResult(status=HandlerStatus.PASS)
# should drop the message if it does not have a valid content_type
return HandlerResult(status=HandlerStatus.DROP)
def broadcast_consensus_message(self, message_envelope):
self.broadcast(
GossipMessage(
content_type=GossipMessage.CONSENSUS,
content=message_envelope.SerializeToString(),
time_to_live=self.get_time_to_live()),
validator_pb2.Message.GOSSIP_MESSAGE)
def gossip_message_preprocessor(message_content_bytes):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content_bytes)
tag = gossip_message.content_type
if tag == GossipMessage.BLOCK:
obj = Block()
obj.ParseFromString(gossip_message.content)
elif tag == GossipMessage.BATCH:
obj = Batch()
obj.ParseFromString(gossip_message.content)
content = obj, tag, gossip_message.time_to_live
return PreprocessorResult(content=content)
def broadcast_batch(self, batch, exclude=None):
gossip_message = GossipMessage(
content_type="BATCH",
content=batch.SerializeToString())
self.broadcast(
gossip_message, validator_pb2.Message.GOSSIP_MESSAGE, exclude)
def broadcast_block(self, block, exclude=None):
gossip_message = GossipMessage(
content_type="BLOCK",
content=block.SerializeToString())
self.broadcast(
gossip_message, validator_pb2.Message.GOSSIP_MESSAGE, exclude)
def broadcast_batch(self, batch, exclude=None):
time_to_live = self.get_time_to_live()
gossip_message = GossipMessage(content_type=GossipMessage.BATCH,
content=batch.SerializeToString(),
time_to_live=time_to_live)
self.broadcast(gossip_message, validator_pb2.Message.GOSSIP_MESSAGE,
exclude)
def handle(self, message, peer):
LOGGER.debug("GossipMessageHandler message: %s", message.sender)
request = GossipMessage()
request.ParseFromString(message.content)
LOGGER.debug("Got gossip message %s "
"from %s. sending ack", message.content, message.sender)
self._ingest_message(request)
ack = NetworkAcknowledgement()
ack.status = ack.OK
peer.send(
validator_pb2.Message(sender=message.sender,
message_type='gossip/ack',
correlation_id=message.correlation_id,
content=ack.SerializeToString()))
def broadcast_block(self, block, exclude=None, time_to_live=None):
if time_to_live is None:
time_to_live = self.get_time_to_live()
gossip_message = GossipMessage(content_type=GossipMessage.BLOCK,
content=block.SerializeToString(),
time_to_live=time_to_live)
self.broadcast(gossip_message, validator_pb2.Message.GOSSIP_MESSAGE,
exclude)
def handle(self, message, peer):
LOGGER.debug("GossipMessageHandler message: {}".format(message))
request = GossipMessage()
request.ParseFromString(message.content)
LOGGER.debug("Got gossip message {} "
"from {}. sending ack".format(message.content,
message.sender))
self._service._put_on_inbound(request)
ack = NetworkAcknowledgement()
ack.status = ack.OK
peer.send(validator_pb2.Message(
sender=message.sender,
message_type='gossip/ack',
correlation_id=message.correlation_id,
content=ack.SerializeToString()))
def send_consensus_message(self, peer_id, message_envelope):
connection_id = self._network.public_key_to_connection_id(peer_id)
self.send(
validator_pb2.Message.GOSSIP_MESSAGE,
GossipMessage(
content_type=GossipMessage.CONSENSUS,
content=message_envelope.SerializeToString(),
time_to_live=self.get_time_to_live()).SerializeToString(),
connection_id)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
if block.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated block: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_block(block):
LOGGER.debug("block signature is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("block passes signature verification %s",
block.header_signature)
self._seen_cache[block.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
if batch.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated batch: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_batch(batch):
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("batch passes signature verification %s",
batch.header_signature)
self._seen_cache[batch.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
# should drop the message if it does not have a valid content_type
return HandlerResult(status=HandlerStatus.DROP)
def handle(self, connection_id, message_content):
exclude = [connection_id]
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
# If we already have this batch, don't forward it
if not self._completer.get_batch(batch.header_signature):
self._gossip.broadcast_batch(batch, exclude)
elif gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
# If we already have this block, don't forward it
if not self._completer.get_block(block.header_signature):
self._gossip.broadcast_block(block, exclude)
else:
LOGGER.info("received %s, not BATCH or BLOCK",
gossip_message.content_type)
return HandlerResult(status=HandlerStatus.PASS)
def __init__(self, identity, endpoint, peer_list, dispatcher):
LOGGER.debug("Initializing Network service")
self._identity = identity
self._dispatcher = dispatcher
self._handlers = {}
self._peered_with_us = {}
self.inbound_queue = queue.Queue()
self._futures = future.FutureCollection()
self._send_receive_thread = _ServerSendReceiveThread(endpoint,
self._handlers,
self._futures)
self._send_receive_thread.daemon = True
self.add_handler('default', DefaultHandler())
self.add_handler('gossip/register',
PeerRegisterHandler(self))
self.add_handler('gossip/unregister',
PeerUnregisterHandler(self))
self.add_handler('gossip/msg',
GossipMessageHandler(self))
self.start()
if peer_list is not None:
for peer in peer_list:
self._send_receive_thread.add_connection(self._identity, peer)
LOGGER.info("Sleeping 5 seconds and then broadcasting messages "
"to connected peers")
time.sleep(5)
content = GossipMessage(content=bytes(
str("This is a gossip payload"), 'UTF-8')).SerializeToString()
for _ in range(1000):
message = validator_pb2.Message(
message_type=b'gossip/msg',
correlation_id=_generate_id(),
content=content)
self.broadcast_message(message)
# If we transmit as fast as possible, we populate the
# buffers and get faster throughput but worse avg. message
# latency. If we sleep here or otherwise throttle input,
# the overall duration is longer, but the per message
# latency is low. The process is CPU bound on Vagrant VM
# core at ~0.001-0.01 duration sleeps.
time.sleep(0.01)
def send_message(self, data):
if isinstance(data, str):
msg = GossipMessage(content_type="BlockRequest",
content=data.encode("utf-8"))
elif isinstance(data, block_pb2.Block):
msg = GossipMessage(content_type="Block",
content=data.SerializeToString())
elif isinstance(data, batch_pb2.Batch):
msg = GossipMessage(content_type="Batch",
content=data.SerializeToString())
content = msg.SerializeToString()
message = validator_pb2.Message(
message_type=validator_pb2.Message.GOSSIP_MESSAGE,
correlation_id=_generate_id(),
content=content)
self._put_on_inbound(message)
def test_signature_thread(self):
try:
self.verifier.start()
blocks = self._create_blocks(1, 1)
msg = GossipMessage(content_type="Block",
content=blocks[0].SerializeToString())
with self._in_condition:
self._in.put_nowait(msg)
self._in_condition.notify_all()
to = TimeOut(2)
while self._out.qsize() == 0:
if to.is_timed_out():
break
self.assertEqual(self._out.qsize(), 1)
finally:
with self._in_condition:
self.verifier.stop()
self._in_condition.notify_all()
def run(self):
while True:
if self._exit:
return
try:
message = self.incoming_msg_queue.get(block=False)
request = GossipMessage()
request.ParseFromString(message.content)
if request.content_type == "Block":
try:
block = Block()
block.ParseFromString(request.content)
status = self.validate_block(block)
if status:
LOGGER.debug("Pass block to dispatch %s",
block.header_signature)
self.dispatcher_msg_queue.put_nowait(request)
self.broadcast(message)
with self.dispatcher_condition:
self.dispatcher_condition.notify_all()
else:
LOGGER.debug("Block signature is invalid: %s",
block.header_signature)
except DecodeError as e:
# what to do with a bad msg
LOGGER.warning(
"Problem decoding GossipMessage for "
"Block, %s", e)
elif request.content_type == "Batch":
try:
batch = Batch()
batch.ParseFromString(request.content)
status = self.validate_batch(batch)
if status:
self.dispatcher_msg_queue.put_nowait(request)
self.broadcast(message)
with self.dispatcher_condition:
self.dispatcher_condition.notify_all()
else:
LOGGER.debug("Batch signature is invalid: %s",
batch.header_signature)
except DecodeError as e:
LOGGER.warning(
"Problem decoding GossipMessage for "
"Batch, %s", e)
elif request.content_type == "BlockRequest":
self.dispatcher_msg_queue.put_notwait(request)
with self.dispatcher_condition:
self.dispatcher_condition.notify_all()
elif request.content_type == "Test":
LOGGER.debug("Verifier Handle Test")
self.dispatcher_msg_queue.put_nowait(request)
with self.dispatcher_condition:
self.dispatcher_condition.notify_all()
except queue.Empty:
with self.in_condition:
self.in_condition.wait()
