class TestCompleter(unittest.TestCase):
def setUp(self):
self.block_store = BlockStore(DictDatabase(
indexes=BlockStore.create_index_configuration()))
self.gossip = MockGossip()
self.completer = Completer(self.block_store, self.gossip)
self.completer._on_block_received = self._on_block_received
self.completer._on_batch_received = self._on_batch_received
self.completer._has_block = self._has_block
self._has_block_value = True
context = create_context('secp256k1')
private_key = context.new_random_private_key()
crypto_factory = CryptoFactory(context)
self.signer = crypto_factory.new_signer(private_key)
self.blocks = []
self.batches = []
def _on_block_received(self, block):
return self.blocks.append(block.header_signature)
def _on_batch_received(self, batch):
return self.batches.append(batch.header_signature)
def _has_block(self, batch):
return self._has_block_value
def _create_transactions(self, count, missing_dep=False):
txn_list = []
for _ in range(count):
payload = {
'Verb': 'set',
'Name': 'name' + str(random.randint(0, 100)),
'Value': random.randint(0, 100)
}
intkey_prefix = \
hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6]
addr = intkey_prefix + \
hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest()
payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest()
header = TransactionHeader(
signer_public_key=self.signer.get_public_key().as_hex(),
family_name='intkey',
family_version='1.0',
inputs=[addr],
outputs=[addr],
dependencies=[],
batcher_public_key=self.signer.get_public_key().as_hex(),
payload_sha512=payload_encode)
if missing_dep:
header.dependencies.extend(["Missing"])
header_bytes = header.SerializeToString()
signature = self.signer.sign(header_bytes)
transaction = Transaction(
header=header_bytes,
payload=cbor.dumps(payload),
header_signature=signature)
txn_list.append(transaction)
return txn_list
def _create_batches(self, batch_count, txn_count,
missing_dep=False):
batch_list = []
for _ in range(batch_count):
txn_list = self._create_transactions(txn_count,
missing_dep=missing_dep)
txn_sig_list = [txn.header_signature for txn in txn_list]
batch_header = BatchHeader(
signer_public_key=self.signer.get_public_key().as_hex())
batch_header.transaction_ids.extend(txn_sig_list)
header_bytes = batch_header.SerializeToString()
signature = self.signer.sign(header_bytes)
batch = Batch(
header=header_bytes,
transactions=txn_list,
header_signature=signature)
batch_list.append(batch)
return batch_list
def _create_blocks(self,
block_count,
batch_count,
missing_predecessor=False,
missing_batch=False,
find_batch=True):
block_list = []
for i in range(0, block_count):
batch_list = self._create_batches(batch_count, 2)
batch_ids = [batch.header_signature for batch in batch_list]
if missing_predecessor:
predecessor = "Missing"
else:
predecessor = (block_list[i - 1].header_signature if i > 0 else
NULL_BLOCK_IDENTIFIER)
block_header = BlockHeader(
signer_public_key=self.signer.get_public_key().as_hex(),
batch_ids=batch_ids,
block_num=i,
previous_block_id=predecessor)
header_bytes = block_header.SerializeToString()
signature = self.signer.sign(header_bytes)
if missing_batch:
if find_batch:
self.completer.add_batch(batch_list[-1])
batch_list = batch_list[:-1]
block = Block(
header=header_bytes,
batches=batch_list,
header_signature=signature)
block_list.append(block)
return block_list
def test_good_block(self):
"""
Add completed block to completer. Block should be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 1)[0]
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
def test_duplicate_block(self):
"""
Submit same block twice.
"""
block = self._create_blocks(1, 1)[0]
self.completer.add_block(block)
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
self.assertEqual(len(self.blocks), 1)
def test_block_missing_predecessor(self):
"""
The block is completed but the predecessor is missing.
"""
block = self._create_blocks(1, 1, missing_predecessor=True)[0]
self._has_block_value = False
self.completer.add_block(block)
self.assertEqual(len(self.blocks), 0)
self.assertIn("Missing", self.gossip.requested_blocks)
header = BlockHeader(previous_block_id=NULL_BLOCK_IDENTIFIER)
missing_block = Block(header_signature="Missing",
header=header.SerializeToString())
self._has_block_value = True
self.completer.add_block(missing_block)
self.assertIn(block.header_signature, self.blocks)
self.assertEqual(
block,
self.completer.get_block(block.header_signature).get_block())
def test_block_with_extra_batch(self):
"""
The block has a batch that is not in the batch_id list.
"""
block = self._create_blocks(1, 1)[0]
batches = self._create_batches(1, 1, True)
block.batches.extend(batches)
self.completer.add_block(block)
self.assertEqual(len(self.blocks), 0)
def test_block_missing_batch(self):
"""
The block is a missing batch and the batch is in the cache. The Block
will be build and passed to on_block_recieved. This puts the block
in the self.blocks list.
"""
block = self._create_blocks(1, 2, missing_batch=True)[0]
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
self.assertEqual(
block,
self.completer.get_block(block.header_signature).get_block())
def test_block_missing_batch_not_in_cache(self):
"""
The block is a missing batch and the batch is not in the cache.
The batch will be requested and the block will not be passed to
on_block_recieved.
"""
block = self._create_blocks(
1, 3, missing_batch=True, find_batch=False)[0]
self.completer.add_block(block)
header = BlockHeader()
header.ParseFromString(block.header)
self.assertIn(header.batch_ids[-1], self.gossip.requested_batches)
def test_block_batches_wrong_order(self):
"""
The block has all of its batches but they are in the wrong order. The
batches will be reordered and the block will be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 6)[0]
batches = list(block.batches)
random.shuffle(batches)
del block.batches[:]
block.batches.extend(batches)
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
def test_block_batches_wrong_batch(self):
"""
The block has all the correct number of batches but one is not in the
batch_id list. This block should be dropped.
"""
block = self._create_blocks(1, 6)[0]
batch = Batch(header_signature="Extra")
batches = list(block.batches)
batches[-1] = batch
block.batches.extend(batches)
self.completer.add_block(block)
self.assertEqual(len(self.blocks), 0)
def test_good_batch(self):
"""
Add complete batch to completer. The batch should be passed to
on_batch_received.
"""
batch = self._create_batches(1, 1)[0]
self.completer.add_batch(batch)
self.assertIn(batch.header_signature, self.batches)
self.assertEqual(batch,
self.completer.get_batch(batch.header_signature))
def test_batch_with_missing_dep(self):
"""
Add batch to completer that has a missing dependency. The missing
transaction's batch should be requested add the missing batch is then
added to the completer. The incomplete batch should be rechecked
and passed to on_batch_received.
"""
batch = self._create_batches(1, 1, missing_dep=True)[0]
self.completer.add_batch(batch)
self.assertIn("Missing",
self.gossip.requested_batches_by_txn_id)
missing = Transaction(header_signature="Missing")
missing_batch = Batch(header_signature="Missing_batch",
transactions=[missing])
self.completer.add_batch(missing_batch)
self.assertIn(missing_batch.header_signature, self.batches)
self.assertIn(batch.header_signature, self.batches)
self.assertEqual(missing_batch,
self.completer.get_batch_by_transaction("Missing"))
class TestCompleter(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.block_db = NativeLmdbDatabase(
os.path.join(self.dir, 'block.lmdb'),
BlockStore.create_index_configuration())
self.block_store = BlockStore(self.block_db)
self.block_manager = BlockManager()
self.block_manager.add_commit_store(self.block_store)
self.gossip = MockGossip()
self.completer = Completer(
block_manager=self.block_manager,
transaction_committed=self.block_store.has_transaction,
get_committed_batch_by_id=self.block_store.get_batch,
get_committed_batch_by_txn_id=(
self.block_store.get_batch_by_transaction),
gossip=self.gossip)
self.completer.set_get_chain_head(lambda: self.block_store.chain_head)
self.completer.set_on_block_received(self._on_block_received)
self.completer.set_on_batch_received(self._on_batch_received)
self._has_block_value = True
context = create_context('secp256k1')
private_key = context.new_random_private_key()
crypto_factory = CryptoFactory(context)
self.signer = crypto_factory.new_signer(private_key)
self.blocks = []
self.batches = []
def _on_block_received(self, block_id):
return self.blocks.append(block_id)
def _on_batch_received(self, batch):
return self.batches.append(batch.header_signature)
def _has_block(self, batch):
return self._has_block_value
def _create_transactions(self, count, missing_dep=False):
txn_list = []
for _ in range(count):
payload = {
'Verb': 'set',
'Name': 'name' + str(random.randint(0, 100)),
'Value': random.randint(0, 100)
}
intkey_prefix = \
hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6]
addr = intkey_prefix + \
hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest()
payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest()
header = TransactionHeader(
signer_public_key=self.signer.get_public_key().as_hex(),
family_name='intkey',
family_version='1.0',
inputs=[addr],
outputs=[addr],
dependencies=[],
batcher_public_key=self.signer.get_public_key().as_hex(),
payload_sha512=payload_encode)
if missing_dep:
header.dependencies.extend(["Missing"])
header_bytes = header.SerializeToString()
signature = self.signer.sign(header_bytes)
transaction = Transaction(header=header_bytes,
payload=cbor.dumps(payload),
header_signature=signature)
txn_list.append(transaction)
return txn_list
def _create_batches(self, batch_count, txn_count, missing_dep=False):
batch_list = []
for _ in range(batch_count):
txn_list = self._create_transactions(txn_count,
missing_dep=missing_dep)
txn_sig_list = [txn.header_signature for txn in txn_list]
batch_header = BatchHeader(
signer_public_key=self.signer.get_public_key().as_hex())
batch_header.transaction_ids.extend(txn_sig_list)
header_bytes = batch_header.SerializeToString()
signature = self.signer.sign(header_bytes)
batch = Batch(header=header_bytes,
transactions=txn_list,
header_signature=signature)
batch_list.append(batch)
return batch_list
def _create_blocks(self,
block_count,
batch_count,
missing_predecessor=False,
missing_batch=False,
find_batch=True):
block_list = []
for i in range(0, block_count):
batch_list = self._create_batches(batch_count, 2)
batch_ids = [batch.header_signature for batch in batch_list]
if missing_predecessor:
predecessor = "Missing"
else:
predecessor = (block_list[i - 1].header_signature
if i > 0 else NULL_BLOCK_IDENTIFIER)
block_header = BlockHeader(
signer_public_key=self.signer.get_public_key().as_hex(),
batch_ids=batch_ids,
block_num=i,
previous_block_id=predecessor)
header_bytes = block_header.SerializeToString()
signature = self.signer.sign(header_bytes)
if missing_batch:
if find_batch:
self.completer.add_batch(batch_list[-1])
batch_list = batch_list[:-1]
block = Block(header=header_bytes,
batches=batch_list,
header_signature=signature)
block_list.append(block)
return block_list
def test_good_block(self):
"""
Add completed block to completer. Block should be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 1)[0]
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
def test_duplicate_block(self):
"""
Submit same block twice.
"""
block = self._create_blocks(1, 1)[0]
self.completer.add_block(block)
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
self.assertEqual(len(self.blocks), 1)
def test_block_missing_predecessor(self):
"""
The block is completed but the predecessor is missing.
"""
block = self._create_blocks(1, 1, missing_predecessor=True)[0]
self._has_block_value = False
self.completer.add_block(block)
self.assertEqual(len(self.blocks), 0)
self.assertIn("Missing", self.gossip.requested_blocks)
header = BlockHeader(previous_block_id=NULL_BLOCK_IDENTIFIER)
missing_block = Block(header_signature="Missing",
header=header.SerializeToString())
self._has_block_value = True
self.completer.add_block(missing_block)
self.assertIn(block.header_signature, self.blocks)
self.assertEqual(block,
self.completer.get_block(block.header_signature))
def test_block_with_extra_batch(self):
"""
The block has a batch that is not in the batch_id list.
"""
block = self._create_blocks(1, 1)[0]
batches = self._create_batches(1, 1, True)
block.batches.extend(batches)
self.completer.add_block(block)
self.assertEqual(len(self.blocks), 0)
def test_block_missing_batch(self):
"""
The block is a missing batch and the batch is in the cache. The Block
will be build and passed to on_block_recieved. This puts the block
in the self.blocks list.
"""
block = self._create_blocks(1, 2, missing_batch=True)[0]
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
self.assertEqual(block,
self.completer.get_block(block.header_signature))
def test_block_missing_batch_not_in_cache(self):
"""
The block is a missing batch and the batch is not in the cache.
The batch will be requested and the block will not be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 3, missing_batch=True,
find_batch=False)[0]
self.completer.add_block(block)
header = BlockHeader()
header.ParseFromString(block.header)
self.assertIn(header.batch_ids[-1], self.gossip.requested_batches)
def test_block_batches_wrong_order(self):
"""
The block has all of its batches but they are in the wrong order. The
batches will be reordered and the block will be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 6)[0]
batches = list(block.batches)
random.shuffle(batches)
del block.batches[:]
block.batches.extend(batches)
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
def test_block_batches_wrong_batch(self):
"""
The block has all the correct number of batches but one is not in the
batch_id list. This block should be dropped.
"""
block = self._create_blocks(1, 6)[0]
batch = Batch(header_signature="Extra")
batches = list(block.batches)
batches[-1] = batch
block.batches.extend(batches)
self.completer.add_block(block)
self.assertEqual(len(self.blocks), 0)
def test_good_batch(self):
"""
Add complete batch to completer. The batch should be passed to
on_batch_received.
"""
batch = self._create_batches(1, 1)[0]
self.completer.add_batch(batch)
self.assertIn(batch.header_signature, self.batches)
self.assertEqual(batch,
self.completer.get_batch(batch.header_signature))
def test_batch_with_missing_dep(self):
"""
Add batch to completer that has a missing dependency. The missing
transaction's batch should be requested add the missing batch is then
added to the completer. The incomplete batch should be rechecked
and passed to on_batch_received.
"""
batch = self._create_batches(1, 1, missing_dep=True)[0]
self.completer.add_batch(batch)
self.assertIn("Missing", self.gossip.requested_batches_by_txn_id)
missing = Transaction(header_signature="Missing")
missing_batch = Batch(header_signature="Missing_batch",
transactions=[missing])
self.completer.add_batch(missing_batch)
self.assertIn(missing_batch.header_signature, self.batches)
self.assertIn(batch.header_signature, self.batches)
self.assertEqual(missing_batch,
self.completer.get_batch_by_transaction("Missing"))
class TestCompleter(unittest.TestCase):
def setUp(self):
self.block_store = BlockStore({})
self.gossip = MockGossip()
self.completer = Completer(self.block_store, self.gossip)
self.completer._on_block_received = self._on_block_received
self.completer._on_batch_received = self._on_batch_received
self.private_key = signing.generate_privkey()
self.public_key = signing.encode_pubkey(
signing.generate_pubkey(self.private_key), "hex")
self.blocks = []
self.batches = []
def _on_block_received(self, block):
print("Block received")
return self.blocks.append(block.header_signature)
def _on_batch_received(self, batch):
print("Batch received")
return self.batches.append(batch.header_signature)
def _create_transactions(self, count, missing_dep=False):
txn_list = []
for i in range(count):
payload = {
'Verb': 'set',
'Name': 'name' + str(random.randint(0, 100)),
'Value': random.randint(0, 100)
}
intkey_prefix = \
hashlib.sha512('intkey'.encode('utf-8')).hexdigest()[0:6]
addr = intkey_prefix + \
hashlib.sha512(payload["Name"].encode('utf-8')).hexdigest()
payload_encode = hashlib.sha512(cbor.dumps(payload)).hexdigest()
header = TransactionHeader(signer_pubkey=self.public_key,
family_name='intkey',
family_version='1.0',
inputs=[addr],
outputs=[addr],
dependencies=[],
payload_encoding="application/cbor",
payload_sha512=payload_encode)
if missing_dep:
header.dependencies.extend(["Missing"])
header_bytes = header.SerializeToString()
signature = signing.sign(header_bytes, self.private_key)
transaction = Transaction(header=header_bytes,
payload=cbor.dumps(payload),
header_signature=signature)
txn_list.append(transaction)
return txn_list
def _generate_id(self):
return hashlib.sha512(''.join([
random.choice(string.ascii_letters) for _ in range(0, 1024)
]).encode()).hexdigest()
def _create_batches(self, batch_count, txn_count, missing_dep=False):
batch_list = []
for i in range(batch_count):
txn_list = self._create_transactions(txn_count,
missing_dep=missing_dep)
txn_sig_list = [txn.header_signature for txn in txn_list]
batch_header = BatchHeader(signer_pubkey=self.public_key)
batch_header.transaction_ids.extend(txn_sig_list)
header_bytes = batch_header.SerializeToString()
signature = signing.sign(header_bytes, self.private_key)
batch = Batch(header=header_bytes,
transactions=txn_list,
header_signature=signature)
batch_list.append(batch)
return batch_list
def _create_blocks(self,
block_count,
batch_count,
missing_predecessor=False,
missing_batch=False,
find_batch=True):
block_list = []
pred = 0
for i in range(0, block_count):
batch_list = self._create_batches(batch_count, 2)
batch_ids = [batch.header_signature for batch in batch_list]
if missing_predecessor:
predecessor = "Missing"
else:
predecessor = (block_list[i - 1].header_signature
if i > 0 else NULL_BLOCK_IDENTIFIER)
block_header = BlockHeader(signer_pubkey=self.public_key,
batch_ids=batch_ids,
block_num=i,
previous_block_id=predecessor)
header_bytes = block_header.SerializeToString()
signature = signing.sign(header_bytes, self.private_key)
if missing_batch:
if find_batch:
self.completer.add_batch(batch_list[-1])
batch_list = batch_list[:-1]
block = Block(header=header_bytes,
batches=batch_list,
header_signature=signature)
block_list.append(block)
return block_list
def test_good_block(self):
"""
Add completed block to completer. Block should be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 1)[0]
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
def test_duplicate_block(self):
"""
Submit same block twice.
"""
block = block = self._create_blocks(1, 1)[0]
self.completer.add_block(block)
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
self.assertEquals(len(self.blocks), 1)
def test_block_missing_predecessor(self):
"""
The block is completed but the predecessor is missing.
"""
block = self._create_blocks(1, 1, missing_predecessor=True)[0]
self.completer.add_block(block)
self.assertEquals(len(self.blocks), 0)
self.assertIn("Missing", self.gossip.requested_blocks)
header = BlockHeader(previous_block_id=NULL_BLOCK_IDENTIFIER)
missing_block = Block(header_signature="Missing",
header=header.SerializeToString())
self.completer.add_block(missing_block)
self.assertIn(block.header_signature, self.blocks)
self.assertEquals(
block,
self.completer.get_block(block.header_signature).get_block())
def test_block_with_extra_batch(self):
"""
The block has a batch that is not in the batch_id list.
"""
block = self._create_blocks(1, 1)[0]
batches = self._create_batches(1, 1, True)
block.batches.extend(batches)
self.completer.add_block(block)
self.assertEquals(len(self.blocks), 0)
def test_block_missing_batch(self):
"""
The block is a missing batch and the batch is in the cache. The Block
will be build and passed to on_block_recieved. This puts the block
in the self.blocks list.
"""
block = self._create_blocks(1, 2, missing_batch=True)[0]
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
self.assertEquals(
block,
self.completer.get_block(block.header_signature).get_block())
def test_block_missing_batch_not_in_cache(self):
"""
The block is a missing batch and the batch is not in the cache.
The batch will be requested and the block will not be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 3, missing_batch=True,
find_batch=False)[0]
self.completer.add_block(block)
header = BlockHeader()
header.ParseFromString(block.header)
self.assertIn(header.batch_ids[-1], self.gossip.requested_batches)
def test_block_batches_wrong_order(self):
"""
The block has all of its batches but they are in the wrong order. The
batches will be reordered and the block will be passed to
on_block_recieved.
"""
block = self._create_blocks(1, 6)[0]
batches = list(block.batches)
random.shuffle(batches)
del block.batches[:]
block.batches.extend(batches)
self.completer.add_block(block)
self.assertIn(block.header_signature, self.blocks)
def test_block_batches_wrong_batch(self):
"""
The block has all the correct number of batches but one is not in the
batch_id list. This block should be dropped.
"""
block = self._create_blocks(1, 6)[0]
batch = Batch(header_signature="Extra")
batches = list(block.batches)
batches[-1] = batch
block.batches.extend(batches)
self.completer.add_block(block)
self.assertEquals(len(self.blocks), 0)
def test_good_batch(self):
"""
Add complete batch to completer. The batch should be passed to
on_batch_received.
"""
batch = self._create_batches(1, 1)[0]
self.completer.add_batch(batch)
self.assertIn(batch.header_signature, self.batches)
self.assertEquals(batch,
self.completer.get_batch(batch.header_signature))
def test_batch_with_missing_dep(self):
"""
Add batch to completer that has a missing dependency. The missing
transaction's batch should be requested add the missing batch is then
added to the completer. The incomplete batch should be rechecked
and passed to on_batch_received.
"""
batch = self._create_batches(1, 1, missing_dep=True)[0]
self.completer.add_batch(batch)
self.assertIn("Missing",
self.gossip.requested_batches_by_transactin_id)
missing = Transaction(header_signature="Missing")
missing_batch = Batch(header_signature="Missing_batch",
transactions=[missing])
self.completer.add_batch(missing_batch)
self.assertIn(missing_batch.header_signature, self.batches)
self.assertIn(batch.header_signature, self.batches)
self.assertEquals(missing_batch,
self.completer.get_batch_by_transaction("Missing"))