def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
status = validate_block(block)
if status is True:
LOGGER.debug("block passes signature verification %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
LOGGER.debug("block signature is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
elif gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
status = validate_batch(batch)
if status is True:
LOGGER.debug("batch passes signature verification %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
def handle(self, connection_id, message_content):
exclude = [connection_id]
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.time_to_live == 0:
# Do not forward message if it has reached its time to live limit
return HandlerResult(status=HandlerStatus.PASS)
else:
# decrement time_to_live
time_to_live = gossip_message.time_to_live
gossip_message.time_to_live = time_to_live - 1
if gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
# If we already have this batch, don't forward it
if not self._completer.get_batch(batch.header_signature):
self._gossip.broadcast_batch(batch, exclude)
elif gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
# If we already have this block, don't forward it
if not self._completer.get_block(block.header_signature):
self._gossip.broadcast_block(block, exclude)
else:
LOGGER.info("received %s, not BATCH or BLOCK",
gossip_message.content_type)
return HandlerResult(status=HandlerStatus.PASS)
def _generate_batch(self, payload):
payload_encoded = payload.encode('utf-8')
hasher = hashlib.sha512()
hasher.update(payload_encoded)
header = TransactionHeader()
header.batcher_pubkey = self.public_key
# txn.dependencies not yet
header.family_name = 'test'
header.family_version = '1'
header.nonce = _generate_id(16)
header.payload_encoding = "text"
header.payload_sha512 = hasher.hexdigest().encode()
header.signer_pubkey = self.public_key
txn = Transaction()
header_bytes = header.SerializeToString()
txn.header = header_bytes
txn.header_signature = signing.sign(header_bytes, self.signing_key)
txn.payload = payload_encoded
batch_header = BatchHeader()
batch_header.signer_pubkey = self.public_key
batch_header.transaction_ids.extend([txn.header_signature])
batch = Batch()
header_bytes = batch_header.SerializeToString()
batch.header = header_bytes
batch.header_signature = signing.sign(header_bytes, self.signing_key)
batch.transactions.extend([txn])
return batch
def handle(self, connection_id, message_content):
gossip_message = network_pb2.GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == network_pb2.GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
LOGGER.debug("CompleterGossipHandler: BLOCK=%s",
block.header_signature[:8])
self._completer.add_block(block)
elif gossip_message.content_type == network_pb2.GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
#candidate_id = gossip_message.candidate_id.hex()
LOGGER.debug("CompleterGossipHandler: NEW BATCH=%s ",
batch.header_signature[:8])
# works in case batch from another node
self._completer.add_batch(batch)
elif gossip_message.content_type == network_pb2.GossipMessage.BATCHES:
batches = BatchList()
batches.ParseFromString(gossip_message.content)
candidate_id = batches.candidate_id.hex()
num = len(batches.batches)
#LOGGER.debug("CompleterGossipHandler: NEW BATCHES=%s candidate_id=%s", batches,candidate_id[:8])
block_num = batches.block_num
for batch in batches.batches:
LOGGER.debug(" => NEW BATCH[%s]=%s candidate_id=%s.%s", num,
batch.header_signature[:8], block_num,
candidate_id[:8])
self._completer.add_batch(batch,
(candidate_id, block_num, num))
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
gossip_message = network_pb2.GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
if not is_valid_block(block):
LOGGER.debug("block's batches structure is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("block passes batch structure verification %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
if not is_valid_batch(batch):
LOGGER.debug("batch structure is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("batch passes structure verification %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
return HandlerResult(status=HandlerStatus.PASS)
def test_publish(self, args=sys.argv[1:]):
gossip = GossipMock()
LOGGER.info(self.blocks)
publisher = BlockPublisher(
consensus=TestModePublisher(),
transaction_executor=TransactionExecutorMock(),
send_message=gossip.send_message,
squash_handler=None)
LOGGER.info("1")
# initial load of existing state
publisher.on_chain_updated(self.blocks.chain_head.block, [], [])
LOGGER.info("2")
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
LOGGER.info("3")
# current dev_mode consensus always claims blocks when asked.
# this will be called on a polling every so often or possibly triggered
# by events in the consensus it's self ... TBD
publisher.on_check_publish_block()
LOGGER.info("4")
LOGGER.info(self.blocks)
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
def test_publish(self):
LOGGER.info(self.blocks)
publisher = BlockPublisher(
consensus=TestModePublisher(),
transaction_executor=MockTransactionExecutor(),
block_sender=self.block_sender,
squash_handler=None,
chain_head=self.blocks.chain_head)
# initial load of existing state
publisher.on_chain_updated(self.blocks.chain_head, [], [])
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
# current dev_mode consensus always claims blocks when asked.
# this will be called on a polling every so often or possibly triggered
# by events in the consensus it's self ... TBD
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == gossip_message.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
has_block = False
if self._completer.get_block(block.header_signature) is not None:
has_block = True
if not has_block and self._has_block(block.header_signature):
has_block = True
if has_block:
LOGGER.debug("Drop duplicate block: %s",
block.header_signature)
return HandlerResult(HandlerStatus.DROP)
if gossip_message.content_type == gossip_message.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
has_batch = False
if self._completer.get_batch(batch.header_signature) is not None:
has_batch = True
if not has_batch and self._has_batch(batch.header_signature):
has_batch = True
if has_batch:
LOGGER.debug("Drop duplicate batch: %s",
batch.header_signature)
return HandlerResult(HandlerStatus.DROP)
return HandlerResult(HandlerStatus.PASS)
def handle(self, connection_id, message_content):
batch_response_message = network_pb2.GossipBatchResponse()
batch_response_message.ParseFromString(message_content)
batch = Batch()
batch.ParseFromString(batch_response_message.content)
self._completer.add_batch(batch)
return HandlerResult(status=HandlerStatus.PASS)
def gossip_batch_response_preprocessor(message_content_bytes):
batch_response = GossipBatchResponse()
batch_response.ParseFromString(message_content_bytes)
batch = Batch()
batch.ParseFromString(batch_response.content)
content = batch, message_content_bytes
return PreprocessorResult(content=content)
def test_no_batch(self):
"""
Test that if the block does not exist yet in the completer or the
chain controller, the gossip message is passed.
"""
batch= Batch(header_signature="Batch1")
message = GossipMessage(content_type=GossipMessage.BATCH,
content=batch.SerializeToString())
handler_status = self.handler.handle(
"connection_id", message.SerializeToString())
self.assertEqual(handler_status.status, HandlerStatus.PASS)
def run(self):
while True:
if self._exit:
return
try:
request = self.incoming_msg_queue.get(block=False)
if request.content_type == "Block":
try:
block = Block()
block.ParseFromString(request.content)
status = self.validate_block(block)
if status:
self.outgoing_msg_queue.put_nowait(request)
with self.out_condition:
self.out_condition.notify_all()
LOGGER.info("Block signature is invalid: %s",
block.header_signature)
except DecodeError as e:
# what to do with a bad msg
LOGGER.warning(
"Problem decoding GossipMessage for "
"Block, %s", e)
elif request.content_type == "Batch":
try:
batch = Batch()
batch.ParseFromString(request.content)
status = self.validate_batch(batch)
if status:
self.outgoing_msg_queue.put_nowait(request)
with self.out_condition:
self.out_condition.notify_all()
else:
LOGGER.info("Batch signature is invalid: %s",
batch.header_signature)
except DecodeError as e:
LOGGER.warning(
"Problem decoding GossipMessage for "
"Batch, %s", e)
elif request.content_type == "BlockRequest":
self.outgoing_msg_queue.put_notwait(request)
with self.out_condition:
self.out_condition.notify_all()
elif request.content_type == "Test":
LOGGER.debug("Verifier Handle Test")
self.outgoing_msg_queue.put_nowait(request)
with self.out_condition:
self.out_condition.notify_all()
except queue.Empty:
with self.in_condition:
self.in_condition.wait()
def handle(self, connection_id, message_content):
gossip_message = network_pb2.GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == network_pb2.GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
self._completer.add_block(block)
elif gossip_message.content_type == network_pb2.GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
self._completer.add_batch(batch)
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
batch_response_message = GossipBatchResponse()
batch_response_message.ParseFromString(message_content)
batch = Batch()
batch.ParseFromString(batch_response_message.content)
if not is_valid_batch(batch):
LOGGER.debug("requested batch's structure is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
return HandlerResult(status=HandlerStatus.PASS)
def run(self):
if self.incoming_msg_queue is None or self.condition is None:
LOGGER.warning("Dispatcher can not be used if incoming_msg_queue "
"or condition is not set.")
return
while True:
if self._exit:
return
try:
request = self.incoming_msg_queue.get(block=False)
if request.content_type == "Block":
try:
block = Block()
block.ParseFromString(request.content)
LOGGER.debug("Block given to Completer %s",
block.header_signature)
self.completer_queue.put_nowait(block)
with self.completer_conditon:
self.completer_conditon.notify_all()
except DecodeError as e:
# what to do with a bad msg
LOGGER.warning(
"Problem decoding GossipMessage for "
"Block, %s", e)
elif request.content_type == "Batch":
try:
batch = Batch()
batch.ParseFromString(request.content)
self.completer.add_batch(batch)
self.on_batch_received(batch)
except DecodeError as e:
LOGGER.warning(
"Problem decoding GossipMessage for "
"Batch, %s", e)
elif request.content_type == "BlockRequest":
block_id = str(request.content, "utf-8")
self.on_block_requested(block_id)
elif request.content_type == "Test":
LOGGER.debug("Dispatch Handle Test")
except queue.Empty:
with self.condition:
self.condition.wait()
def handle(self, identity, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
self._gossip.broadcast_batch(batch)
elif gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
self._gossip.broadcast_block(block)
else:
LOGGER.info("received %s, not BATCH or BLOCK",
gossip_message.content_type)
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
if block.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated block: %s cache=%s",
block.header_signature[:8], len(self._seen_cache)
) #[sig[:8] for sig in self._seen_cache])
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_block(block):
LOGGER.debug("block signature is invalid: %s",
block.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
self._seen_cache[block.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
if batch.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated batch: %s",
batch.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_batch(batch):
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
self._seen_cache[batch.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCHES:
# check batches signature
LOGGER.debug("Check BATCHES signature!!!")
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.ENDPOINTS:
# check endpoints
LOGGER.debug("Check ENDPOINTS signature!!!")
return HandlerResult(status=HandlerStatus.PASS)
# should drop the message if it does not have a valid content_type
return HandlerResult(status=HandlerStatus.DROP)
def create_batch(self, block_info):
payload = BlockInfoTxn(block=block_info).SerializeToString()
public_key = self._signer.get_public_key().as_hex()
header = TransactionHeader(
signer_public_key=public_key,
family_name=FAMILY_NAME,
family_version=FAMILY_VERSION,
inputs=[CONFIG_ADDRESS, BLOCK_INFO_NAMESPACE],
outputs=[CONFIG_ADDRESS, BLOCK_INFO_NAMESPACE],
dependencies=[],
payload_sha512=hashlib.sha512(payload).hexdigest(),
batcher_public_key=public_key,
).SerializeToString()
transaction_signature = self._signer.sign(header)
transaction = Transaction(
header=header,
payload=payload,
header_signature=transaction_signature,
)
header = BatchHeader(
signer_public_key=public_key,
transaction_ids=[transaction_signature],
).SerializeToString()
batch_signature = self._signer.sign(header)
return Batch(
header=header,
transactions=[transaction],
header_signature=batch_signature,
)
def handle(self, connection_id, message_content):
batch_response_message = GossipBatchResponse()
batch_response_message.ParseFromString(message_content)
batch = Batch()
batch.ParseFromString(batch_response_message.content)
status = validate_batch(batch)
if status is True:
LOGGER.debug("requested batch passes signature verification %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
else:
LOGGER.debug("requested batch's signature is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
def _create_batches(self, batch_count, txn_count,
missing_dep=False):
batch_list = []
for i in range(batch_count):
txn_list = self._create_transactions(txn_count,
missing_dep=missing_dep)
txn_sig_list = [txn.header_signature for txn in txn_list]
batch_header = BatchHeader(signer_pubkey=self.public_key)
batch_header.transaction_ids.extend(txn_sig_list)
header_bytes = batch_header.SerializeToString()
signature = signing.sign(
header_bytes,
self.private_key)
batch = Batch(header=header_bytes,
transactions=txn_list,
header_signature=signature)
batch_list.append(batch)
return batch_list
def _create_batches(self,
batch_count,
txn_count,
valid_batch=True,
valid_txn=True,
valid_structure=True,
valid_batcher=True):
batch_list = []
for i in range(batch_count):
txn_list = self._create_transactions(txn_count, valid_txn,
valid_batcher)
txn_sig_list = [txn.header_signature for txn in txn_list]
if not valid_structure:
txn_sig_list.pop()
batch_header = BatchHeader(signer_pubkey=self.public_key)
batch_header.transaction_ids.extend(txn_sig_list)
header_bytes = batch_header.SerializeToString()
if valid_batch:
signature = signing.sign(header_bytes, self.private_key)
else:
signature = "bad_signature"
batch = Batch(header=header_bytes,
transactions=txn_list,
header_signature=signature)
batch_list.append(batch)
return batch_list
def _generate_batch(self, txn_count=2, missing_deps=False, txns=None):
if txns is None:
txns = []
if txn_count != 0:
txns += [
self.generate_transaction('txn_' + str(i))
for i in range(txn_count)
]
if missing_deps:
target_txn = txns[-1]
txn_missing_deps = self.generate_transaction(
payload='this one has a missing dependency',
deps=[target_txn.header_signature])
# replace the targeted txn with the missing deps txn
txns[-1] = txn_missing_deps
batch_header = BatchHeader(
signer_public_key=self.signer.get_public_key().as_hex(),
transaction_ids=[txn.header_signature
for txn in txns]).SerializeToString()
batch = Batch(header=batch_header,
header_signature=self.signer.sign(batch_header),
transactions=txns)
return batch
def handle(self, connection_id, message_content):
batch_response_message = GossipBatchResponse()
batch_response_message.ParseFromString(message_content)
batch = Batch()
batch.ParseFromString(batch_response_message.content)
if batch.header_signature in self._seen_cache:
self._batch_dropped_count.inc()
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_batch(batch):
LOGGER.debug("requested batch's signature is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
self._seen_cache[batch.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
def get_batch(self, batch_id):
"""
Check to see if the requested batch_id is in the current chain. If so,
find the batch with the batch_id and return it. This is done by
finding the block and searching for the batch.
:param batch_id (string): The id of the batch requested.
:return:
The batch with the batch_id.
"""
payload = self._get_data_by_id(batch_id, 'commit_store_get_batch')
batch = Batch()
batch.ParseFromString(payload)
return batch
def test_publish_block(self):
"""
Test that the Journal will produce blocks and consume those blocks
to extend the chain.
:return:
"""
# construction and wire the journal to the
# gossip layer.
LOGGER.info("test_publish_block")
block_store = {}
journal = None
try:
journal = Journal(
consensus=test_mode_consensus,
block_store=block_store,
send_message=self.gossip.send_message,
transaction_executor=self.txn_executor,
squash_handler=None,
first_state_root="000000")
self.gossip.on_batch_received = \
journal.on_batch_received
self.gossip.on_block_received = \
journal.on_block_received
self.gossip.on_block_request = \
journal.on_block_request
journal.start()
# feed it a batch
batch = Batch()
journal.on_batch_received(batch)
# wait for a block message to arrive should be soon
to = TimeOut(2)
while len(self.gossip.messages) == 0:
time.sleep(0.1)
LOGGER.info("Batches: %s", self.gossip.messages)
self.assertTrue(len(self.gossip.messages) != 0)
block = self.gossip.messages[0]
# dispatch the message
self.gossip.dispatch_messages()
# wait for the chain_head to be updated.
to = TimeOut(2)
while block_store['chain_head_id'] != block.header_signature:
time.sleep(0.1)
self.assertTrue(block_store['chain_head_id'] ==
block.header_signature)
finally:
if journal is not None:
journal.stop()
def handle(self, connection_id, message_content):
batch_response_message = GossipBatchResponse()
batch_response_message.ParseFromString(message_content)
batch = Batch()
batch.ParseFromString(batch_response_message.content)
batch_id = batch.header_signature
if not self._has_open_requests(batch_id) and self._has_batch(batch_id):
LOGGER.debug('Drop duplicate batch: %s', batch_id)
return HandlerResult(HandlerStatus.RETURN)
ack = NetworkAcknowledgement()
ack.status = ack.OK
return HandlerResult(HandlerStatus.RETURN_AND_PASS,
message_out=ack,
message_type=validator_pb2.Message.NETWORK_ACK)
def _make_mock_batch(self, batch_id='batch_id'):
txn = _make_mock_transaction(batch_id)
header = BatchHeader(signer_pubkey='pubkey',
transaction_ids=[txn.header_signature])
return Batch(header=header.SerializeToString(),
header_signature=batch_id,
transactions=[txn])
def test_no_batch(self):
"""
Test that if the batch has not been received recently,
the gossip message is passed.
"""
batch = Batch(header_signature="Batch1")
handler_status = self.handler.handle("connection_id",
(batch, GossipMessage.BATCH, 2))
self.assertEqual(handler_status.status, HandlerStatus.PASS)
def test_no_batch(self):
"""
Test that if the block does not exist yet in the completer or the
chain controller, the gossip message is passed.
"""
batch = Batch(header_signature="Batch1")
handler_status = self.handler.handle("connection_id",
(batch, GossipMessage.BATCH, 2))
self.assertEqual(handler_status.status, HandlerStatus.PASS)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
if block.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated block: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_block(block):
LOGGER.debug("block signature is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("block passes signature verification %s",
block.header_signature)
self._seen_cache[block.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
if batch.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated batch: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_batch(batch):
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("batch passes signature verification %s",
batch.header_signature)
self._seen_cache[batch.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
# should drop the message if it does not have a valid content_type
return HandlerResult(status=HandlerStatus.DROP)
