def __init__(self, identity, endpoint, peer_list, dispatcher):
LOGGER.debug("Initializing Network service")
self._identity = identity
self._dispatcher = dispatcher
self._handlers = {}
self._peered_with_us = {}
self.inbound_queue = queue.Queue()
self.dispatcher_queue = queue.Queue()
self._signature_condition = Condition()
self._dispatcher_condition = Condition()
self._futures = future.FutureCollection()
self._send_receive_thread = _ServerSendReceiveThread(
endpoint, self._handlers, self._futures, self._put_on_inbound)
self._send_receive_thread.daemon = True
self._signature_verifier = SignatureVerifier(
self.inbound_queue, self.dispatcher_queue,
self._signature_condition, self._dispatcher_condition,
self.broadcast_message)
self._dispatcher.set_incoming_msg_queue(self.dispatcher_queue)
self._dispatcher.set_condition(self._dispatcher_condition)
self.add_handler(validator_pb2.Message.DEFAULT, DefaultHandler())
self.add_handler(validator_pb2.Message.GOSSIP_REGISTER,
PeerRegisterHandler(self))
self.add_handler(validator_pb2.Message.GOSSIP_UNREGISTER,
PeerUnregisterHandler(self))
self.add_handler(validator_pb2.Message.GOSSIP_MESSAGE,
GossipMessageHandler(self._put_on_inbound))
self.start()
self._dispatcher.start()
if peer_list is not None:
for peer in peer_list:
self._send_receive_thread.add_connection(self._identity, peer)
def setUp(self):
self.private_key = signing.generate_privkey()
self.public_key = signing.encode_pubkey(
signing.generate_pubkey(self.private_key), "hex")
self._out = queue.Queue()
self._in = queue.Queue()
self._in_condition = Condition()
self._out_condition = Condition()
self.verifier = SignatureVerifier(self._in, self._out,
self._in_condition,
self._out_condition)
class Network(object):
def __init__(self, identity, endpoint, peer_list, dispatcher):
LOGGER.debug("Initializing Network service")
self._identity = identity
self._dispatcher = dispatcher
self._handlers = {}
self._peered_with_us = {}
self.inbound_queue = queue.Queue()
self.dispatcher_queue = queue.Queue()
self._signature_condition = Condition()
self._dispatcher_condition = Condition()
self._futures = future.FutureCollection()
self._send_receive_thread = _ServerSendReceiveThread(
endpoint, self._handlers, self._futures, self._put_on_inbound)
self._send_receive_thread.daemon = True
self._signature_verifier = SignatureVerifier(
self.inbound_queue, self.dispatcher_queue,
self._signature_condition, self._dispatcher_condition,
self.broadcast_message)
self._dispatcher.set_incoming_msg_queue(self.dispatcher_queue)
self._dispatcher.set_condition(self._dispatcher_condition)
self.add_handler(validator_pb2.Message.DEFAULT, DefaultHandler())
self.add_handler(validator_pb2.Message.GOSSIP_REGISTER,
PeerRegisterHandler(self))
self.add_handler(validator_pb2.Message.GOSSIP_UNREGISTER,
PeerUnregisterHandler(self))
self.add_handler(validator_pb2.Message.GOSSIP_MESSAGE,
GossipMessageHandler(self._put_on_inbound))
self.start()
self._dispatcher.start()
if peer_list is not None:
for peer in peer_list:
self._send_receive_thread.add_connection(self._identity, peer)
def add_handler(self, message_type, handler):
LOGGER.debug("Network service adding " "handler for %s", message_type)
self._handlers[message_type] = handler
def _put_on_inbound(self, item):
self.inbound_queue.put_nowait(item)
with self._signature_condition:
self._signature_condition.notify_all()
def send_message(self, data):
if isinstance(data, str):
msg = GossipMessage(content_type="BlockRequest",
content=data.encode("utf-8"))
elif isinstance(data, block_pb2.Block):
msg = GossipMessage(content_type="Block",
content=data.SerializeToString())
elif isinstance(data, batch_pb2.Batch):
msg = GossipMessage(content_type="Batch",
content=data.SerializeToString())
content = msg.SerializeToString()
message = validator_pb2.Message(
message_type=validator_pb2.Message.GOSSIP_MESSAGE,
correlation_id=_generate_id(),
content=content)
self._put_on_inbound(message)
def broadcast_message(self, message):
self._send_receive_thread.broadcast_message(message)
fut = future.Future(message.correlation_id)
self._futures.put(fut)
return fut
def register_peer(self, sender, identity):
data = sender
LOGGER.debug("Registering peer: "
"sender %s, identity %s", sender, identity)
if sender not in self._peered_with_us.keys():
self._peered_with_us[sender] = []
self._peered_with_us[sender].append(data)
LOGGER.debug("Peers: %s", self._peered_with_us)
def unregister_peer(self, sender, identity):
LOGGER.debug("Unregistering peer: "
"sender %s, identity %s", sender, identity)
if sender in self._peered_with_us.keys():
del self._peered_with_us[sender]
LOGGER.debug("Peers: %s", self._peered_with_us)
def start(self):
self._send_receive_thread.start()
self._signature_verifier.start()
def stop(self):
self._signature_verifier.stop()
self._dispatcher.stop()
with self._signature_condition:
self._signature_condition.notify_all()
with self._dispatcher_condition:
self._dispatcher_condition.notify_all()
self._send_receive_thread.join()
class TestMessageValidation(unittest.TestCase):
def setUp(self):
self.private_key = signing.generate_privkey()
self.public_key = signing.encode_pubkey(
signing.generate_pubkey(self.private_key), "hex")
self._out = queue.Queue()
self._in = queue.Queue()
self._broadcast = self.broadcast
self._in_condition = Condition()
self._out_condition = Condition()
self.verifier = SignatureVerifier(self._in,
self._out,
self._in_condition,
self._out_condition,
self._broadcast)
def broadcast(self, msg):
pass
def _create_transactions(self, count, valid=True, valid_batcher=True):
txn_list = []
for i in range(count):
payload = {'Verb': 'set',
'Name': 'name' + str(random.randint(0, 100)),
'Value': random.randint(0, 100)}
intkey_prefix = \
hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6]
addr = intkey_prefix + \
hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest()
payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest()
header = TransactionHeader(
signer_pubkey=self.public_key,
family_name='intkey',
family_version='1.0',
inputs=[addr],
outputs=[addr],
dependencies=[],
payload_encoding="application/cbor",
payload_sha512=payload_encode)
if valid_batcher:
header.batcher_pubkey = self.public_key
else:
header.batcher_pubkey = "bad_batcher"
header_bytes = header.SerializeToString()
if valid:
signature = signing.sign(
header_bytes,
self.private_key)
else:
signature = "bad_signature"
transaction = Transaction(
header=header_bytes,
payload=cbor.dumps(payload),
header_signature=signature)
txn_list.append(transaction)
return txn_list
def _generate_id(self):
return hashlib.sha512(''.join(
[random.choice(string.ascii_letters)
for _ in range(0, 1024)]).encode()).hexdigest()
def _create_batches(self, batch_count, txn_count,
valid_batch=True, valid_txn=True,
valid_batcher=True):
batch_list = []
for i in range(batch_count):
txn_list = self._create_transactions(txn_count, valid_txn,
valid_batcher)
txn_sig_list = [txn.header_signature for txn in txn_list]
batch_header = BatchHeader(signer_pubkey=self.public_key)
batch_header.transaction_ids.extend(txn_sig_list)
header_bytes = batch_header.SerializeToString()
if valid_batch:
signature = signing.sign(
header_bytes,
self.private_key)
else:
signature = "bad_signature"
batch = Batch(header=header_bytes,
transactions=txn_list,
header_signature=signature)
batch_list.append(batch)
return batch_list
def _create_blocks(self, block_count, batch_count,
valid_block=True, valid_batch=True):
block_list = []
for i in range(block_count):
batch_list = self._create_batches(
batch_count, 2, valid_batch=valid_batch)
batch_ids = [batch.header_signature for batch in batch_list]
block_header = BlockHeader(signer_pubkey=self.public_key,
batch_ids=batch_ids)
header_bytes = block_header.SerializeToString()
if valid_block:
signature = signing.sign(
header_bytes,
self.private_key)
else:
signature = "bad_signature"
block = Block(header=header_bytes,
batches=batch_list,
header_signature=signature)
block_list.append(block)
return block_list
def test_valid_transaction(self):
txn_list = self._create_transactions(1)
txn = txn_list[0]
valid = self.verifier.validate_transaction(txn)
self.assertTrue(valid)
def test_invalid_transaction(self):
# add invalid flag to _create transaction
txn_list = self._create_transactions(1, valid=False)
txn = txn_list[0]
valid = self.verifier.validate_transaction(txn)
self.assertFalse(valid)
def test_valid_batch(self):
batch_list = self._create_batches(1, 10)
batch = batch_list[0]
valid = self.verifier.validate_batch(batch)
self.assertTrue(valid)
def test_invalid_batch(self):
# add invalid flag to create_batches
batch_list = self._create_batches(1, 1, valid_batch=False)
batch = batch_list[0]
valid = self.verifier.validate_batch(batch)
self.assertFalse(valid)
# create an invalid txn in the batch
batch_list = self._create_batches(1, 1, valid_txn=False)
batch = batch_list[0]
valid = self.verifier.validate_batch(batch)
self.assertFalse(valid)
# create an invalid txn with bad batcher
batch_list = self._create_batches(1, 1, valid_batcher=False)
batch = batch_list[0]
valid = self.verifier.validate_batch(batch)
self.assertFalse(valid)
def test_valid_block(self):
block_list = self._create_blocks(1, 1)
block = block_list[0]
valid = self.verifier.validate_block(block)
self.assertTrue(valid)
def test_invalid_block(self):
block_list = self._create_blocks(1, 1, valid_batch=False)
block = block_list[0]
valid = self.verifier.validate_block(block)
self.assertFalse(valid)
block_list = self._create_blocks(1, 1, valid_block=False)
block = block_list[0]
valid = self.verifier.validate_block(block)
self.assertFalse(valid)
def test_signature_thread(self):
try:
self.verifier.start()
blocks = self._create_blocks(1, 1)
message = GossipMessage(content_type="Block",
content=blocks[0].SerializeToString())
content = message.SerializeToString()
msg = validator_pb2.Message(
message_type=validator_pb2.Message.GOSSIP_MESSAGE,
correlation_id=self._generate_id(),
content=content)
with self._in_condition:
self._in.put_nowait(msg)
self._in_condition.notify_all()
to = TimeOut(2)
while self._out.qsize() == 0:
if to.is_timed_out():
break
self.assertEqual(self._out.qsize(), 1)
finally:
with self._in_condition:
self.verifier.stop()
self._in_condition.notify_all()
def __init__(self, identity, endpoint, peer_list, dispatcher):
LOGGER.debug("Initializing Network service")
self._identity = identity
self._dispatcher = dispatcher
self._handlers = {}
self._peered_with_us = {}
self.inbound_queue = queue.Queue()
self.outbound_queue = queue.Queue()
self._signature_condition = Condition()
self._dispatcher_condition = Condition()
self._futures = future.FutureCollection()
self._send_receive_thread = _ServerSendReceiveThread(
endpoint, self._handlers, self._futures, self._put_on_inbound)
self._send_receive_thread.daemon = True
self._signature_verifier = SignatureVerifier(
self.inbound_queue, self.outbound_queue, self._signature_condition,
self._dispatcher_condition)
self._dispatcher.set_incoming_msg_queue(self.outbound_queue)
self._dispatcher.set_condition(self._dispatcher_condition)
self.add_handler('default', DefaultHandler())
self.add_handler('gossip/register', PeerRegisterHandler(self))
self.add_handler('gossip/unregister', PeerUnregisterHandler(self))
self.add_handler('gossip/msg',
GossipMessageHandler(self._put_on_inbound))
self.start()
self._dispatcher.start()
if peer_list is not None:
for peer in peer_list:
self._send_receive_thread.add_connection(self._identity, peer)
LOGGER.info("Sleeping 5 seconds and then broadcasting messages "
"to connected peers")
time.sleep(5)
content = GossipMessage(content=bytes(
str("This is a gossip payload"), 'UTF-8')).SerializeToString()
for _ in range(1000):
message = validator_pb2.Message(message_type=b'gossip/msg',
correlation_id=_generate_id(),
content=content)
self.broadcast_message(message)
# If we transmit as fast as possible, we populate the
# buffers and get faster throughput but worse avg. message
# latency. If we sleep here or otherwise throttle input,
# the overall duration is longer, but the per message
# latency is low. The process is CPU bound on Vagrant VM
# core at ~0.001-0.01 duration sleeps.
time.sleep(0.01)
# Send messages to :
# inbound queue -> SignatureVerifier -> outbound queue -> Dispatcher
for _ in range(20):
msg = GossipMessage(content=bytes(str("This is a gossip payload"),
'UTF-8'),
content_type="Test")
self._put_on_inbound(msg)
time.sleep(.01)