def setUp(self):
opts = gateway_helpers.get_gateway_opts(8000,
include_default_btc_args=True)
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.node = MockGatewayNode(opts)
self.node.neutrality_service = MagicMock()
self.connection = BtcNodeConnection(
MockSocketConnection(1,
node=self.node,
ip_address=LOCALHOST,
port=123), self.node)
gateway_helpers.add_blockchain_peer(self.node, self.connection)
self.tx_hash = BtcObjectHash(buf=helpers.generate_bytearray(32),
length=BTC_SHA_HASH_LEN)
self.block_hash = BtcObjectHash(buf=helpers.generate_bytearray(32),
length=BTC_SHA_HASH_LEN)
self.sut = BtcNodeConnectionProtocol(self.connection)
while self.connection.outputbuf.length > 0:
initial_bytes = self.connection.get_bytes_to_send()
self.connection.advance_sent_bytes(len(initial_bytes))
def _test_block_cleanup(self):
block_msg = self._get_sample_block(self._get_file_path())
transactions = block_msg.txns()[:]
block_hash = typing.cast(BtcObjectHash, block_msg.block_hash())
random.shuffle(transactions)
short_len = int(len(transactions) * 0.9)
transactions_short = transactions[:short_len]
unknown_transactions = transactions[short_len:]
transaction_hashes = []
for idx, tx in enumerate(transactions_short):
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=btc_constants.BTC_SHA_HASH_LEN)
transaction_hashes.append(tx_hash)
self.transaction_service.set_transaction_contents(tx_hash, tx)
self.transaction_service.assign_short_id(tx_hash, idx + 1)
for idx, tx in enumerate(unknown_transactions):
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=btc_constants.BTC_SHA_HASH_LEN)
transaction_hashes.append(tx_hash)
if idx % 2 == 0:
self.transaction_service.set_transaction_contents(tx_hash, tx)
self.cleanup_service._block_hash_marked_for_cleanup.add(block_hash)
self.cleanup_service.clean_block_transactions(block_msg,
self.transaction_service)
self.assertEqual(0, self.transaction_service._total_tx_contents_size)
for tx_hash in transaction_hashes:
self.assertFalse(
self.transaction_service.has_transaction_contents(tx_hash))
def _test_mark_blocks_and_request_cleanup(self):
marked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
prev_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
tracked_blocks = []
self.cleanup_service.on_new_block_received(marked_block, prev_block)
self.transaction_service.track_seen_short_ids(marked_block, [])
for _ in range(self.block_confirmations_count - 1):
tracked_block = BtcObjectHash(binary=helpers.generate_bytearray(
btc_constants.BTC_SHA_HASH_LEN))
self.transaction_service.track_seen_short_ids(tracked_block, [])
tracked_blocks.append(tracked_block)
unmarked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
self.assertIsNone(self.cleanup_service.last_confirmed_block)
self.cleanup_service.mark_blocks_and_request_cleanup(
[marked_block, *tracked_blocks])
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
self.assertTrue(
self.cleanup_service.is_marked_for_cleanup(marked_block))
self.assertFalse(
self.cleanup_service.is_marked_for_cleanup(unmarked_block))
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
msg = self.node.send_to_node_messages.pop(-1)
self.assertEqual(1, msg.count())
self.assertEqual((InventoryType.MSG_BLOCK, marked_block),
next(iter(msg)))
def setUp(self):
super().setUp()
self.node1.alarm_queue = AlarmQueue()
self.node2.alarm_queue = AlarmQueue()
self.network_num = 1
self.magic = 12345
self.version = 23456
self.prev_block_hash = bytearray(crypto.double_sha256(b"123"))
self.prev_block = BtcObjectHash(self.prev_block_hash,
length=crypto.SHA256_HASH_LEN)
self.merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
self.merkle_root = BtcObjectHash(self.merkle_root_hash,
length=crypto.SHA256_HASH_LEN)
self.bits = 2
self.nonce = 3
opts = self.gateway_1_opts()
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.btc_message_converter = btc_message_converter_factory.create_btc_message_converter(
self.magic, opts)
self.btc_transactions = [
TxBtcMessage(self.magic, self.version, [], [], i)
for i in range(self.TRANSACTIONS_COUNT)
]
self.btc_transactions_for_block = [
tx_btc_message.rawbytes()[btc_constants.BTC_HDR_COMMON_OFF:]
for tx_btc_message in self.btc_transactions
]
self.transactions = [
self.btc_message_converter.tx_to_bx_txs(tx_btc_message,
self.network_num)[0][0]
for tx_btc_message in self.btc_transactions
]
self.transactions_with_short_ids = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1, tx_message.tx_val())
for i, tx_message in enumerate(self.transactions)
]
self.transactions_with_no_content = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1) for i, tx_message in enumerate(self.transactions)
]
self.transactions_by_short_id = {
tx_message.short_id(): tx_message
for tx_message in self.transactions_with_short_ids
}
self.block = BlockBtcMessage(self.magic, self.version,
self.prev_block, self.merkle_root,
int(time.time()), self.bits, self.nonce,
self.btc_transactions_for_block)
def block_to_bx_block(self, block_msg, tx_service) -> Tuple[memoryview, BlockInfo]:
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
self._default_block_size = max(self._default_block_size, len(block_msg.buf))
tsk = self.compression_tasks.borrow_task()
tsk.init(tpe.InputBytes(block_msg.buf), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compression_tasks.return_task(tsk)
raise message_conversion_error.btc_block_compression_error(block_msg.block_hash(), e)
bx_block = tsk.bx_block()
block = memoryview(bx_block)
compressed_size = len(block)
original_size = len(block_msg.rawbytes())
block_hash = BtcObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string())
)
block_info = BlockInfo(
block_hash,
tsk.short_ids(),
compress_start_datetime,
datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tsk.txn_count(),
tsk.compressed_block_hash().hex_string(),
tsk.prev_block_hash().hex_string(),
original_size,
compressed_size,
100 - float(compressed_size) / original_size * 100
)
self.compression_tasks.return_task(tsk)
return block, block_info
def parse_bx_block_header(
bx_block: memoryview,
block_pieces: Deque[Union[bytearray, memoryview]]) -> BlockHeaderInfo:
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block)
short_ids, short_ids_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block, block_offsets.short_id_offset)
# Compute block header hash
block_header_size = \
block_offsets.block_begin_offset + \
btc_constants.BTC_HDR_COMMON_OFF + \
btc_constants.BTC_BLOCK_HDR_SIZE
block_hash = BtcObjectHash(buf=crypto.bitcoin_hash(
bx_block[block_offsets.block_begin_offset +
btc_constants.BTC_HDR_COMMON_OFF:block_header_size]),
length=btc_constants.BTC_SHA_HASH_LEN)
offset = block_header_size
# Add header piece
txn_count, txn_count_size = btc_common_utils.btc_varint_to_int(
bx_block, block_header_size)
offset += txn_count_size
block_pieces.append(bx_block[block_offsets.block_begin_offset:offset])
return BlockHeaderInfo(block_offsets, short_ids, short_ids_len, block_hash,
offset, txn_count)
def block_hash(self) -> BtcObjectHash:
if self._block_hash is None:
self._block_hash = BtcObjectHash(buf=self.buf,
offset=BTC_HDR_COMMON_OFF,
length=BTC_SHA_HASH_LEN)
# pyre-fixme[7]: Expected `BtcObjectHash` but got `None`.
return self._block_hash
def block_hash(self):
if self._block_hash is None:
header = self._memoryview[:
BTC_BLOCK_HDR_SIZE] # remove the tx count at the end
raw_hash = crypto.bitcoin_hash(header)
self._hash_val = BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN)
return self._hash_val
def block_hash(self) -> BtcObjectHash:
if self._hash_val is None:
header = self._memoryview[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE]
raw_hash = crypto.bitcoin_hash(header)
self._hash_val = BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN)
# pyre-fixme[7]: Expected `BtcObjectHash` but got `None`.
return self._hash_val
def __iter__(self):
off = BTC_HDR_COMMON_OFF + 4 # For the version field.
b_count, size = btc_varint_to_int(self.buf, off)
off += size
for i in range(b_count):
yield BtcObjectHash(buf=self.buf,
offset=off,
length=BTC_SHA_HASH_LEN)
off += 32
def version(self):
if self._version is None:
off = BTC_HDR_COMMON_OFF
self._version = struct.unpack_from('<I', self.buf, off)[0]
off += 4
self._prev_block = BtcObjectHash(self.buf, off, 32)
off += 32
self._merkle_root = BtcObjectHash(self.buf, off, 32)
off += 32
self._timestamp, self._bits, self._nonce = struct.unpack_from(
'<III', self.buf, off)
off += 12
self._txn_count, size = btc_varint_to_int(self.buf, off)
off += size
self._tx_offset = off
self._header = self._memoryview[0:self._tx_offset]
return self._version
class BtcConnectionProtocolTest(AbstractTestCase):
HASH = BtcObjectHash(binary=crypto.double_sha256(b"123"))
def setUp(self):
opts = gateway_helpers.get_gateway_opts(8000,
include_default_btc_args=True)
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.node = MockGatewayNode(opts)
self.node.block_processing_service = MagicMock()
self.connection = MagicMock()
gateway_helpers.add_blockchain_peer(self.node, self.connection)
self.connection.node = self.node
self.connection.peer_ip = LOCALHOST
self.connection.peer_port = 8001
self.connection.network_num = 2
self.connection.endpoint = IpEndpoint(self.connection.peer_ip,
self.connection.peer_port)
self.node.blockchain_peers.add(
BlockchainPeerInfo(self.connection.peer_ip,
self.connection.peer_port))
gateway_bdn_performance_stats_service.set_node(self.node)
self.sut = BtcBaseConnectionProtocol(self.connection)
def test_msg_block_success(self):
block_timestamp = int(
time.time()
) + 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval
txns = [
TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
message = BlockBtcMessage(0, 0, self.HASH, self.HASH, block_timestamp,
0, 0, txns)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_called_once(
)
def test_msg_block_too_old(self):
block_timestamp = int(
time.time()
) - 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval
txns = [
TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
message = BlockBtcMessage(0, 0, self.HASH, self.HASH, 0,
block_timestamp, 0, txns)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_not_called(
)
def test_btc_block_to_bloxroute_block_and_back_sids_found(self):
prev_block_hash = bytearray(crypto.bitcoin_hash(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
timestamp = 1
bits = 2
nonce = 3
btc_block = BlockBtcMessage(
self.magic, self.version, prev_block, merkle_root, timestamp, bits, nonce, self.txns
)
block_hash = btc_block.block_hash()
bloxroute_block, block_info = self.btc_message_converter.block_to_bx_block(
btc_block, self.tx_service, True, 0
)
self.assertEqual(10, block_info.txn_count)
self.assertEqual("5a77d1e9612d350b3734f6282259b7ff0a3f87d62cfef5f35e91a5604c0490a3",
block_info.prev_block_hash)
self.assertEqual(self.short_ids, list(block_info.short_ids))
self.assertEqual(btc_block.block_hash(), block_info.block_hash)
# TODO: if we convert bloxroute block to a class, add some tests here
parsed_btc_block, block_info, _, _ = self.btc_message_converter.bx_block_to_block(
bloxroute_block,
self.tx_service)
self.assertIsNotNone(block_info)
self.assertEqual(parsed_btc_block.rawbytes().tobytes(), btc_block.rawbytes().tobytes())
self.assertEqual(self.version, parsed_btc_block.version())
self.assertEqual(self.magic, parsed_btc_block.magic())
self.assertEqual(prev_block_hash, parsed_btc_block.prev_block_hash().get_little_endian())
self.assertEqual(merkle_root_hash, parsed_btc_block.merkle_root().get_little_endian())
self.assertEqual(timestamp, parsed_btc_block.timestamp())
self.assertEqual(bits, parsed_btc_block.bits())
self.assertEqual(nonce, parsed_btc_block.nonce())
self.assertEqual(len(self.txns), parsed_btc_block.txn_count())
self.assertEqual(btc_block.checksum(), parsed_btc_block.checksum())
self.assertEqual(block_hash, parsed_btc_block.block_hash())
self.assertEqual(block_hash.binary, block_info.block_hash.binary)
self.assertEqual(list(block_info.short_ids), self.short_ids)
def clean_block_transactions(
self, block_msg: BlockBtcMessage,
transaction_service: TransactionService) -> None:
block_short_ids = []
block_unknown_tx_hashes = []
start_time = time.time()
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
for tx in block_msg.txns():
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=BTC_SHA_HASH_LEN)
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
block_hash = block_msg.block_hash()
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, block_msg.txn_count(),
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=block_short_ids,
unknown_tx_hashes=block_unknown_tx_hashes)
def version(self):
if self._version is None:
off = 0
self._version = struct.unpack_from('<I', self.buf, off)[0]
off += 4
self._prev_block = BtcObjectHash(self.buf, off, 32)
off += 32
self._merkle_root = self._memoryview[off:off + 32]
off += 32
self._timestamp, self._bits, self._nonce = struct.unpack_from(
'<III', self.buf, off)
self._txn_count, size = btc_varint_to_int(self.buf, off)
return self._version
def btc_block(timestamp=None, real_block=None):
if real_block is not None:
return BlockBtcMessage(buf=bytearray(convert.hex_to_bytes(real_block)))
if timestamp is None:
timestamp = int(time.time())
magic = 12345
version = 23456
prev_block_hash = bytearray(crypto.double_sha256(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
bits = 2
nonce = 3
txns = [
TxBtcMessage(magic, version, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
return BlockBtcMessage(magic, version, prev_block, merkle_root, timestamp,
bits, nonce, txns)
def test_get_data_only_new_data(self):
seen_block_hash = BtcObjectHash(
buf=helpers.generate_bytearray(BTC_SHA_HASH_LEN),
length=BTC_SHA_HASH_LEN)
not_seen_block_hash = BtcObjectHash(
buf=helpers.generate_bytearray(BTC_SHA_HASH_LEN),
length=BTC_SHA_HASH_LEN)
self.node.blocks_seen.add(seen_block_hash)
inv_message = InvBtcMessage(
magic=123,
inv_vects=[(InventoryType.MSG_TX, seen_block_hash),
(InventoryType.MSG_BLOCK, not_seen_block_hash),
(InventoryType.MSG_BLOCK, seen_block_hash)])
self.sut.msg_inv(inv_message)
get_data_msg_bytes = self.sut.connection.get_bytes_to_send()
get_data_msg = GetDataBtcMessage(buf=get_data_msg_bytes)
self.assertEqual(2, get_data_msg.count())
self.assertIn((InventoryType.MSG_TX, seen_block_hash), get_data_msg)
self.assertIn((InventoryType.MSG_BLOCK, not_seen_block_hash),
get_data_msg)
def peek_block(input_buffer):
buf = input_buffer.peek_message(BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE)
is_here = False
if input_buffer.length < BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE:
return is_here, None, None
header = buf[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE]
raw_hash = crypto.bitcoin_hash(header)
payload_len = struct.unpack_from('<L', buf, 16)[0]
is_here = True
return is_here, BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN), payload_len
def get_txid(buffer: Union[memoryview, bytearray]) -> BtcObjectHash:
"""
Actually gets the txid, which is the same as the hash for non segwit transactions
:param buffer: the bytes of the transaction contents
:return: hash object
"""
flag_len = btc_common_constants.TX_SEGWIT_FLAG_LEN if is_segwit(buffer) else 0
txid = sha256(buffer[:btc_common_constants.TX_VERSION_LEN])
end = btc_common_constants.TX_VERSION_LEN + flag_len
io_size, _, _ = get_tx_io_count_and_size(buffer, end, tail=-1)
txid.update(buffer[end:end + io_size])
txid.update(buffer[-btc_common_constants.TX_LOCK_TIME_LEN:])
return BtcObjectHash(buf=sha256(txid.digest()).digest(), length=btc_common_constants.BTC_SHA_HASH_LEN)
def _test_mark_blocks_and_request_cleanup(self):
node_conn = MockConnection(
MockSocketConnection(0, self.node, ip_address=LOCALHOST,
port=9000), self.node)
self.node.get_any_active_blockchain_connection = MagicMock(
return_value=node_conn)
marked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
prev_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
tracked_blocks = []
self.cleanup_service.on_new_block_received(marked_block, prev_block)
self.transaction_service.track_seen_short_ids(marked_block, [])
for _ in range(self.block_confirmations_count - 1):
tracked_block = BtcObjectHash(binary=helpers.generate_bytearray(
btc_constants.BTC_SHA_HASH_LEN))
self.transaction_service.track_seen_short_ids(tracked_block, [])
tracked_blocks.append(tracked_block)
unmarked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
self.assertIsNone(self.cleanup_service.last_confirmed_block)
self.cleanup_service.mark_blocks_and_request_cleanup(
[marked_block, *tracked_blocks])
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
self.assertTrue(
self.cleanup_service.is_marked_for_cleanup(marked_block))
self.assertFalse(
self.cleanup_service.is_marked_for_cleanup(unmarked_block))
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
msg = node_conn.enqueued_messages[0]
self.assertEqual(1, msg.count())
self.assertEqual((InventoryType.MSG_BLOCK, marked_block),
next(iter(msg)))
def test_message_preview_success_all_gateway_types(self):
self.get_message_preview_successfully(
GatewayHelloMessage(123, 1, "127.0.0.1", 40000, 1),
GatewayMessageType.HELLO,
GatewayHelloMessage.PAYLOAD_LENGTH,
)
self.get_message_preview_successfully(
BlockReceivedMessage(self.HASH),
GatewayMessageType.BLOCK_RECEIVED,
BlockReceivedMessage.PAYLOAD_LENGTH,
)
self.get_message_preview_successfully(
BlockPropagationRequestMessage(self.BLOCK),
GatewayMessageType.BLOCK_PROPAGATION_REQUEST,
len(self.BLOCK) + constants.CONTROL_FLAGS_LEN,
)
self.get_message_preview_successfully(
BlockHoldingMessage(self.HASH, network_num=123),
BloxrouteMessageType.BLOCK_HOLDING,
BlockHoldingMessage.PAYLOAD_LENGTH,
)
self.get_message_preview_successfully(
ConfirmedTxMessage(self.HASH, self.TX_VAL),
GatewayMessageType.CONFIRMED_TX,
ConfirmedTxMessage.PAYLOAD_LENGTH + len(self.TX_VAL),
)
self.get_message_preview_successfully(
RequestTxStreamMessage(),
GatewayMessageType.REQUEST_TX_STREAM,
constants.CONTROL_FLAGS_LEN,
)
hash_val = BtcObjectHash(buf=crypto.double_sha256(b"123"),
length=crypto.SHA256_HASH_LEN)
blockchain_message = GetBlocksBtcMessage(12345, 23456, [hash_val],
hash_val).rawbytes()
self.get_message_preview_successfully(
BlockchainSyncRequestMessage(GetBlocksBtcMessage.MESSAGE_TYPE,
blockchain_message),
BlockchainSyncRequestMessage.MESSAGE_TYPE,
MSG_TYPE_LEN + len(blockchain_message),
)
self.get_message_preview_successfully(
BlockchainSyncResponseMessage(GetBlocksBtcMessage.MESSAGE_TYPE,
blockchain_message),
BlockchainSyncResponseMessage.MESSAGE_TYPE,
MSG_TYPE_LEN + len(blockchain_message),
)
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.decompression_tasks.return_task(tsk)
header_info = btc_normal_message_converter.parse_bx_block_header(bx_block_msg, deque())
raise message_conversion_error.btc_block_decompression_error(header_info.block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = BtcObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string())
)
if tsk.success():
btc_block_msg = BlockBtcMessage(buf=memoryview(tsk.block_message()))
logger.debug(
"Successfully parsed block broadcast message. {} transactions "
"in block {}",
total_tx_count,
block_hash
)
else:
btc_block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}",
block_hash,
len(unknown_tx_sids),
len(unknown_tx_hashes),
total_tx_count
)
block_info = get_block_info(
bx_block_msg,
block_hash,
tsk.short_ids(),
decompress_start_datetime,
decompress_start_timestamp,
total_tx_count,
btc_block_msg
)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(btc_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
def __iter__(self) -> Iterator[Tuple[int, BtcObjectHash]]:
off = btc_constants.BTC_HDR_COMMON_OFF
num_items, size = btc_common_utils.btc_varint_to_int(self.buf, off)
off += size
# pyre-fixme[16]: `InventoryBtcMessage` has no attribute `_num_items`.
self._num_items = num_items
# pyre-fixme[16]: `InventoryBtcMessage` has no attribute `_items`.
self._items = list()
for _ in range(num_items):
inv_type, = struct.unpack_from("<L", self.buf, off)
off += 4
yield (inv_type,
BtcObjectHash(buf=self.buf,
offset=off,
length=btc_constants.BTC_SHA_HASH_LEN))
off += 32
def test_compact_block_full_compression(self):
compact_block = get_sample_compact_block()
recovered_block = get_recovered_compact_block()
for short_id, txn in enumerate(recovered_block.txns()):
bx_tx_hash = BtcObjectHash(buf=crypto.double_sha256(txn),
length=BTC_SHA_HASH_LEN)
self.tx_service.set_transaction_contents(bx_tx_hash, txn)
self.tx_service.assign_short_id(bx_tx_hash, short_id + 1)
result = self.btc_message_converter.compact_block_to_bx_block(
compact_block, self.tx_service
)
self.assertTrue(result.success)
ref_block, _, _, _ = self.btc_message_converter.bx_block_to_block(
result.bx_block, self.tx_service
)
self.assertEqual(recovered_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes())
def message(self):
if self._message is None:
off = BTC_HDR_COMMON_OFF
self._message = struct.unpack_from('%dp' % (len(self.buf) - off, ),
self.buf, off)[0]
off += len(self._message) + 1
self._ccode = struct.unpack_from('B', self.buf, off)[0]
off += 1
self._reason = struct.unpack_from('%dp' % (len(self.buf) - off, ),
self.buf, off)[0]
off += len(self._reason) + 1
self._data = self.buf[off:]
self._obj_hash = BtcObjectHash(
buf=self.buf,
offset=len(self.buf) - btc_constants.BTC_SHA_HASH_LEN,
length=btc_constants.BTC_SHA_HASH_LEN)
return self._message
class BlockchainSyncBtcTest(AbstractTestCase):
HASH = BtcObjectHash(binary=crypto.bitcoin_hash(b"hi"))
def setUp(self):
self.local_node_fileno = 1
self.remote_node_fileno = 2
self.gateway_node = spies.make_spy_node(BtcGatewayNode,
8000,
include_default_btc_args=True)
self.btc_node_connection = spies.make_spy_connection(
BtcNodeConnection, self.local_node_fileno, 8001, self.gateway_node)
self.btc_remote_node_connection = spies.make_spy_connection(
BtcRemoteConnection, self.remote_node_fileno, 8002,
self.gateway_node)
self.gateway_node.node_conn = self.btc_node_connection
self.gateway_node.remote_node_conn = self.btc_remote_node_connection
self.gateway_node.connection_pool.add(self.local_node_fileno,
LOCALHOST, 8001,
self.btc_node_connection)
self.gateway_node.connection_pool.add(self.remote_node_fileno,
LOCALHOST, 8002,
self.btc_remote_node_connection)
def test_block_headers_request(self):
sent_get_headers = GetHeadersBtcMessage(12345, 23456, [self.HASH],
self.HASH)
helpers.receive_node_message(self.gateway_node, self.local_node_fileno,
sent_get_headers.rawbytes())
self.btc_remote_node_connection.enqueue_msg.assert_called_once_with(
sent_get_headers)
response_headers = HeadersBtcMessage(12345, [])
helpers.receive_node_message(self.gateway_node,
self.remote_node_fileno,
response_headers.rawbytes())
self.btc_node_connection.enqueue_msg.assert_called_once_with(
response_headers)
def get_object_hash(bytearr):
return BtcObjectHash(bytearr, 0, 32)
class BtcMessageFactoryTest(MessageFactoryTestCase):
MAGIC = 12345
VERSION = 11111
HASH = BtcObjectHash(binary=crypto.bitcoin_hash(b"123"))
VERSION_BTC_MESSAGE = VersionBtcMessage(MAGIC, VERSION, "127.0.0.1", 8000,
"127.0.0.1", 8001, 123, 0,
"hello".encode("utf-8"))
def get_message_factory(self):
return btc_message_factory
def test_peek_message_success_all_types(self):
# TODO: pull these numbers into constants, along with all the BTC messages
self.get_message_preview_successfully(self.VERSION_BTC_MESSAGE,
VersionBtcMessage.MESSAGE_TYPE,
90)
self.get_message_preview_successfully(VerAckBtcMessage(self.MAGIC),
VerAckBtcMessage.MESSAGE_TYPE, 0)
self.get_message_preview_successfully(PingBtcMessage(self.MAGIC),
PingBtcMessage.MESSAGE_TYPE, 8)
self.get_message_preview_successfully(PongBtcMessage(self.MAGIC, 123),
PongBtcMessage.MESSAGE_TYPE, 8)
self.get_message_preview_successfully(GetAddrBtcMessage(self.MAGIC),
GetAddrBtcMessage.MESSAGE_TYPE,
0)
self.get_message_preview_successfully(
AddrBtcMessage(self.MAGIC,
[(int(time.time()), "127.0.0.1", 8000)]),
AddrBtcMessage.MESSAGE_TYPE, 23)
inv_vector = [(1, self.HASH), (2, self.HASH)]
self.get_message_preview_successfully(
InvBtcMessage(self.MAGIC, inv_vector), InvBtcMessage.MESSAGE_TYPE,
73)
self.get_message_preview_successfully(
GetDataBtcMessage(self.MAGIC, inv_vector),
GetDataBtcMessage.MESSAGE_TYPE, 73)
self.get_message_preview_successfully(
NotFoundBtcMessage(self.MAGIC, inv_vector),
NotFoundBtcMessage.MESSAGE_TYPE, 73)
hashes = [self.HASH, self.HASH]
self.get_message_preview_successfully(
GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetHeadersBtcMessage.MESSAGE_TYPE, 101)
self.get_message_preview_successfully(
GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetBlocksBtcMessage.MESSAGE_TYPE, 101)
self.get_message_preview_successfully(
TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0),
TxBtcMessage.MESSAGE_TYPE, 10)
txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5
self.get_message_preview_successfully(
BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0,
0, 0, txs), BlockBtcMessage.MESSAGE_TYPE, 131)
self.get_message_preview_successfully(
HeadersBtcMessage(self.MAGIC,
[helpers.generate_bytearray(81)] * 2),
HeadersBtcMessage.MESSAGE_TYPE, 163)
self.get_message_preview_successfully(
RejectBtcMessage(self.MAGIC, b"a message",
RejectBtcMessage.REJECT_MALFORMED, b"test break",
helpers.generate_bytearray(10)),
RejectBtcMessage.MESSAGE_TYPE, 32)
self.get_message_preview_successfully(
SendHeadersBtcMessage(self.MAGIC),
SendHeadersBtcMessage.MESSAGE_TYPE, 0)
self.get_message_preview_successfully(
FeeFilterBtcMessage(self.MAGIC, fee_rate=100),
FeeFilterBtcMessage.MESSAGE_TYPE, 8)
self.get_message_preview_successfully(
BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)),
XversionBtcMessage.MESSAGE_TYPE, 0)
def test_peek_message_incomplete(self):
is_full_message, command, payload_length = btc_message_factory.get_message_header_preview_from_input_buffer(
create_input_buffer_with_bytes(
self.VERSION_BTC_MESSAGE.rawbytes()[:-10]))
self.assertFalse(is_full_message)
self.assertEqual(b"version", command)
self.assertEqual(90, payload_length)
is_full_message, command, payload_length = btc_message_factory.get_message_header_preview_from_input_buffer(
create_input_buffer_with_bytes(
self.VERSION_BTC_MESSAGE.rawbytes()[:1]))
self.assertFalse(is_full_message)
self.assertIsNone(command)
self.assertIsNone(payload_length)
def test_parse_message_success_all_types(self):
# TODO: pull these numbers into constants, along with all the BTC messages
self.create_message_successfully(self.VERSION_BTC_MESSAGE,
VersionBtcMessage)
self.create_message_successfully(VerAckBtcMessage(self.MAGIC),
VerAckBtcMessage)
self.create_message_successfully(PingBtcMessage(self.MAGIC),
PingBtcMessage)
self.create_message_successfully(PongBtcMessage(self.MAGIC, 123),
PongBtcMessage)
self.create_message_successfully(GetAddrBtcMessage(self.MAGIC),
GetAddrBtcMessage)
self.create_message_successfully(
AddrBtcMessage(self.MAGIC,
[(int(time.time()), "127.0.0.1", 8000)]),
AddrBtcMessage)
inv_vector = [(1, self.HASH), (2, self.HASH)]
self.create_message_successfully(InvBtcMessage(self.MAGIC, inv_vector),
InvBtcMessage)
self.create_message_successfully(
GetDataBtcMessage(self.MAGIC, inv_vector), GetDataBtcMessage)
self.create_message_successfully(
NotFoundBtcMessage(self.MAGIC, inv_vector), NotFoundBtcMessage)
hashes = [self.HASH, self.HASH]
self.create_message_successfully(
GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetHeadersBtcMessage)
self.create_message_successfully(
GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetBlocksBtcMessage)
self.create_message_successfully(
TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0), TxBtcMessage)
txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5
self.create_message_successfully(
BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0,
0, 0, txs), BlockBtcMessage)
self.create_message_successfully(
HeadersBtcMessage(self.MAGIC,
[helpers.generate_bytearray(81)] * 2),
HeadersBtcMessage)
self.create_message_successfully(
RejectBtcMessage(self.MAGIC, b"a message",
RejectBtcMessage.REJECT_MALFORMED, b"test break",
helpers.generate_bytearray(10)), RejectBtcMessage)
self.create_message_successfully(SendHeadersBtcMessage(self.MAGIC),
SendHeadersBtcMessage)
self.create_message_successfully(
FeeFilterBtcMessage(self.MAGIC, fee_rate=100), FeeFilterBtcMessage)
self.create_message_successfully(
BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)),
XversionBtcMessage)
def test_parse_message_incomplete(self):
with self.assertRaises(PayloadLenError):
btc_message_factory.create_message_from_buffer(
PingBtcMessage(self.MAGIC).rawbytes()[:-1])
ping_message = PingBtcMessage(self.MAGIC)
for i in range(BTC_HEADER_MINUS_CHECKSUM, BTC_HDR_COMMON_OFF):
ping_message.buf[i] = 0
with self.assertRaises(ChecksumError):
btc_message_factory.create_message_from_buffer(
ping_message.rawbytes())
def test_segwit_tx_hash(self):
seg_tx = "010000000001024668fcfeba861f7f1bf4d386f15cc6923bd8425e0214686671775359d17a51d50100000000ffffffffdc5530f864de2fac86246426094d7b8586a452d6bd8c209bb891646afb3548770000000000ffffffff0220040000000000001600147771a1cab96e36344b1693d3d9f29180ca900482f5c40100000000001976a91483121cc1ea476c25d91191ff735a5e90518c732788ac02473044022006db5e6aa36dafb5d89a8522675d304a228d39ede1450aaab04f84b1fb57db2902203efb537cca9738c599d95d5c0ddcec6ebd11c6001541a89a468246318e0bd6fe012102d6b8b2ba44eb621ac9537ed7e11553bb02060abca88a9e6faf7697df5ac6d30c02483045022100b04869e06930db5d4e8e4d453d9aed1097a8dae57eef0274ebdc99a106796335022037a6b744900b9b6392448c961e8d793367a0caf675b9ca80349c593e505d8e9d0121034ef9635ae7cd714b2cf8af7e72f23b8b07c7f75d75df95da8d682ae17459091b00000000"
seg_tx_bytes = convert.hex_to_bytes(seg_tx)
self.assertTrue(btc_common_utils.is_segwit(seg_tx_bytes))
self.assertEqual(
convert.bytes_to_hex(
btc_common_utils.get_txid(seg_tx_bytes).binary),
"d9a057f11a21cf8afd32278e23fd2290660f05a3ffb582466eb5a1a5ece4ce85")
def test_non_segwit_tx_hash(self):
non_seg_tx = "0100000002d8c8df6a6fdd2addaf589a83d860f18b44872d13ee6ec3526b2b470d42a96d4d000000008b483045022100b31557e47191936cb14e013fb421b1860b5e4fd5d2bc5ec1938f4ffb1651dc8902202661c2920771fd29dd91cd4100cefb971269836da4914d970d333861819265ba014104c54f8ea9507f31a05ae325616e3024bd9878cb0a5dff780444002d731577be4e2e69c663ff2da922902a4454841aa1754c1b6292ad7d317150308d8cce0ad7abffffffff2ab3fa4f68a512266134085d3260b94d3b6cfd351450cff021c045a69ba120b2000000008b4830450220230110bc99ef311f1f8bda9d0d968bfe5dfa4af171adbef9ef71678d658823bf022100f956d4fcfa0995a578d84e7e913f9bb1cf5b5be1440bcede07bce9cd5b38115d014104c6ec27cffce0823c3fecb162dbd576c88dd7cda0b7b32b0961188a392b488c94ca174d833ee6a9b71c0996620ae71e799fc7c77901db147fa7d97732e49c8226ffffffff02c0175302000000001976a914a3d89c53bb956f08917b44d113c6b2bcbe0c29b788acc01c3d09000000001976a91408338e1d5e26db3fce21b011795b1c3c8a5a5d0788ac00000000"
non_seg_tx_bytes = convert.hex_to_bytes(non_seg_tx)
self.assertFalse(btc_common_utils.is_segwit(non_seg_tx_bytes))
self.assertEqual(
convert.bytes_to_hex(
btc_common_utils.get_txid(non_seg_tx_bytes).binary),
"9021b49d445c719106c95d561b9c3fac7bcb3650db67684a9226cd7fa1e1c1a0")
def test_create_message_success_all_gateway_types(self):
hello_message = self.create_message_successfully(
GatewayHelloMessage(123, 1, "127.0.0.1", 40001, 1),
GatewayHelloMessage)
self.assertEqual(123, hello_message.protocol_version())
self.assertEqual(1, hello_message.network_num())
self.assertEqual("127.0.0.1", hello_message.ip())
self.assertEqual(40001, hello_message.port())
self.assertEqual(1, hello_message.ordering())
block_recv_message = self.create_message_successfully(
BlockReceivedMessage(self.HASH), BlockReceivedMessage)
self.assertEqual(self.HASH, block_recv_message.block_hash())
block_holding_message = self.create_message_successfully(
BlockHoldingMessage(self.HASH, network_num=123),
BlockHoldingMessage)
self.assertEqual(self.HASH, block_holding_message.block_hash())
block_propagation_request_message = self.create_message_successfully(
BlockPropagationRequestMessage(self.BLOCK),
BlockPropagationRequestMessage)
self.assertEqual(self.BLOCK, block_propagation_request_message.blob())
hash_val = BtcObjectHash(buf=crypto.double_sha256(b"123"),
length=crypto.SHA256_HASH_LEN)
blockchain_message_in = GetBlocksBtcMessage(12345, 23456, [hash_val],
hash_val).rawbytes()
sync_request_message = self.create_message_successfully(
BlockchainSyncRequestMessage(GetBlocksBtcMessage.MESSAGE_TYPE,
blockchain_message_in),
BlockchainSyncRequestMessage,
)
blockchain_message_out = GetBlocksBtcMessage(
buf=sync_request_message.payload())
self.assertEqual(12345, blockchain_message_out.magic())
self.assertEqual(23456, blockchain_message_out.version())
self.assertEqual(1, blockchain_message_out.hash_count())
self.assertEqual(hash_val, blockchain_message_out.hash_stop())
self.create_message_successfully(
BlockchainSyncResponseMessage(GetBlocksBtcMessage.MESSAGE_TYPE,
blockchain_message_in),
BlockchainSyncResponseMessage,
)
blockchain_message_out = GetBlocksBtcMessage(
buf=sync_request_message.payload())
self.assertEqual(12345, blockchain_message_out.magic())
self.assertEqual(23456, blockchain_message_out.version())
self.assertEqual(1, blockchain_message_out.hash_count())
self.assertEqual(hash_val, blockchain_message_out.hash_stop())
confirmed_tx = self.create_message_successfully(
ConfirmedTxMessage(self.HASH, self.TX_VAL), ConfirmedTxMessage)
self.assertEqual(self.HASH, confirmed_tx.tx_hash())
self.assertEqual(self.TX_VAL, confirmed_tx.tx_val())
confirmed_tx_no_content = self.create_message_successfully(
ConfirmedTxMessage(self.HASH), ConfirmedTxMessage)
self.assertEqual(self.HASH, confirmed_tx_no_content.tx_hash())
self.assertEqual(TxMessage.EMPTY_TX_VAL,
confirmed_tx_no_content.tx_val())
self.create_message_successfully(RequestTxStreamMessage(),
RequestTxStreamMessage)
class BtcRelayConnectionTest(AbstractTestCase):
BTC_HASH = BtcObjectHash(crypto.double_sha256(b"123"), length=SHA256_HASH_LEN)
MAGIC = 12345
VERSION = 23456
TEST_NETWORK_NUM = 12345
def setUp(self):
opts = gateway_helpers.get_gateway_opts(8000, include_default_btc_args=True)
if opts.use_extensions:
helpers.set_extensions_parallelism(opts.thread_pool_parallelism_degree)
node_ssl_service = MockNodeSSLService(BtcGatewayNode.NODE_TYPE, MagicMock())
self.gateway_node = BtcGatewayNode(opts, node_ssl_service)
self.gateway_node.opts.has_fully_updated_tx_service = True
self.sut = BtcRelayConnection(MockSocketConnection(
node=self.gateway_node, ip_address=LOCALHOST, port=8001), self.gateway_node
)
self.gateway_node.node_conn = MockConnection(MockSocketConnection(
1, self.gateway_node, ip_address=LOCALHOST, port=8002), self.gateway_node
)
self.gateway_node.message_converter = converter_factory.create_btc_message_converter(
12345, self.gateway_node.opts
)
self.gateway_node.node_conn.state = ConnectionState.ESTABLISHED
self.gateway_node.send_msg_to_node = MagicMock()
self.sut.enqueue_msg = MagicMock()
gateway_transaction_stats_service.set_node(self.gateway_node)
def btc_transactions(self):
return [TxBtcMessage(self.MAGIC, self.VERSION, [], [], i) for i in range(10)]
def btc_transactions_bytes(self):
return [transaction.rawbytes()[BTC_HDR_COMMON_OFF:] for transaction in self.btc_transactions()]
def btc_block(self, txns=None):
if txns is None:
txns = self.btc_transactions_bytes()
return BlockBtcMessage(self.MAGIC, self.VERSION, self.BTC_HASH, self.BTC_HASH, 0, 0, 0, txns)
def bx_block(self, btc_block=None):
if btc_block is None:
btc_block = self.btc_block()
return bytes(self.gateway_node.message_converter.block_to_bx_block(btc_block, self.gateway_node.get_tx_service())[0])
def bx_transactions(self, transactions=None, assign_short_ids=False):
if transactions is None:
transactions = self.btc_transactions()
bx_transactions = []
for i, transaction in enumerate(transactions):
transaction = self.gateway_node.message_converter.tx_to_bx_txs(transaction, self.TEST_NETWORK_NUM)[0][0]
if assign_short_ids:
transaction._short_id = i + 1 # 0 is null SID
bx_transactions.append(transaction)
return bx_transactions
def test_msg_broadcast_wait_for_key(self):
btc_block = self.btc_block()
bx_block = self.bx_block(btc_block)
key, ciphertext = symmetric_encrypt(bx_block)
block_hash = crypto.double_sha256(ciphertext)
broadcast_message = BroadcastMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True, ciphertext)
self.sut.msg_broadcast(broadcast_message)
# handle duplicate messages
self.sut.msg_broadcast(broadcast_message)
self.sut.msg_broadcast(broadcast_message)
self.gateway_node.send_msg_to_node.assert_not_called()
self.assertEqual(1, len(self.gateway_node.in_progress_blocks))
key_message = KeyMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "", key)
self.sut.msg_key(key_message)
self._assert_block_sent(btc_block)
def test_msg_broadcast_duplicate_block_with_different_short_id(self):
# test scenario when first received block is compressed with unknown short ids,
# but second received block is compressed with known short ids
btc_block = self.btc_block()
block_hash = btc_block.block_hash()
transactions = self.bx_transactions()
unknown_sid_transaction_service = ExtensionTransactionService(MockNode(
gateway_helpers.get_gateway_opts(8999)), 0)
for i, transaction in enumerate(transactions):
unknown_sid_transaction_service.assign_short_id(transaction.tx_hash(), i)
unknown_sid_transaction_service.set_transaction_contents(transaction.tx_hash(), transaction.tx_val())
unknown_short_id_block = bytes(self.gateway_node.message_converter.block_to_bx_block(btc_block,
unknown_sid_transaction_service)[0])
unknown_key, unknown_cipher = symmetric_encrypt(unknown_short_id_block)
unknown_block_hash = crypto.double_sha256(unknown_cipher)
unknown_message = BroadcastMessage(Sha256Hash(unknown_block_hash), self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True, unknown_cipher)
unknown_key_message = KeyMessage(Sha256Hash(unknown_block_hash), self.TEST_NETWORK_NUM, "", unknown_key)
local_transaction_service = self.gateway_node.get_tx_service()
for i, transaction in enumerate(transactions):
local_transaction_service.assign_short_id(transaction.tx_hash(), i + 20)
local_transaction_service.set_transaction_contents(transaction.tx_hash(), transaction.tx_val())
known_short_id_block = bytes(self.gateway_node.message_converter.block_to_bx_block(btc_block,
local_transaction_service)[0])
known_key, known_cipher = symmetric_encrypt(known_short_id_block)
known_block_hash = crypto.double_sha256(known_cipher)
known_message = BroadcastMessage(Sha256Hash(known_block_hash), self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True, known_cipher)
known_key_message = KeyMessage(Sha256Hash(known_block_hash), self.TEST_NETWORK_NUM, "", known_key)
self.sut.msg_broadcast(unknown_message)
self.sut.msg_key(unknown_key_message)
self.assertEqual(1, len(self.gateway_node.block_queuing_service))
self.assertEqual(True, self.gateway_node.block_queuing_service._blocks_waiting_for_recovery[block_hash])
self.assertEqual(1, len(self.gateway_node.block_recovery_service._block_hash_to_bx_block_hashes))
self.assertNotIn(block_hash, self.gateway_node.blocks_seen.contents)
self.sut.msg_broadcast(known_message)
self.sut.msg_key(known_key_message)
self.assertEqual(0, len(self.gateway_node.block_queuing_service))
self.assertEqual(0, len(self.gateway_node.block_recovery_service._block_hash_to_bx_block_hashes))
self.assertIn(block_hash, self.gateway_node.blocks_seen.contents)
def test_msg_key_wait_for_broadcast(self):
btc_block = self.btc_block()
bx_block = self.bx_block(btc_block)
key, ciphertext = symmetric_encrypt(bx_block)
block_hash = crypto.double_sha256(ciphertext)
self.gateway_node.send_msg_to_node.assert_not_called()
key_message = KeyMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "", key)
self.sut.msg_key(key_message)
# handle duplicate broadcasts
self.sut.msg_key(key_message)
self.sut.msg_key(key_message)
self.assertEqual(1, len(self.gateway_node.in_progress_blocks))
broadcast_message = BroadcastMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True, ciphertext)
self.sut.msg_broadcast(broadcast_message)
self._assert_block_sent(btc_block)
def test_msg_tx(self):
transactions = self.bx_transactions(assign_short_ids=True)
for transaction in transactions:
self.sut.msg_tx(transaction)
for i, transaction in enumerate(transactions):
transaction_hash = transaction.tx_hash()
self.assertTrue(self.gateway_node.get_tx_service().has_transaction_contents(transaction_hash))
self.assertTrue(self.gateway_node.get_tx_service().has_transaction_short_id(transaction_hash))
self.assertEqual(i + 1, self.gateway_node.get_tx_service().get_short_id(transaction_hash))
stored_hash, stored_content, _ = self.gateway_node.get_tx_service().get_transaction(i + 1)
self.assertEqual(transaction_hash, stored_hash)
self.assertEqual(transaction.tx_val(), stored_content)
self.assertEqual(len(transactions), self.gateway_node.send_msg_to_node.call_count)
def test_msg_tx_additional_sid(self):
transactions = self.bx_transactions(assign_short_ids=True)
for transaction in transactions:
self.sut.msg_tx(transaction)
# rebroadcast transactions with more sids
for i, transaction in enumerate(transactions):
transaction._short_id += 20
self.sut.msg_tx(transaction)
for i, transaction in enumerate(transactions):
transaction_hash = transaction.tx_hash()
self.assertTrue(self.gateway_node.get_tx_service().has_transaction_contents(transaction_hash))
self.assertTrue(self.gateway_node.get_tx_service().has_transaction_short_id(transaction_hash))
stored_hash, stored_content, _ = self.gateway_node.get_tx_service().get_transaction(i + 1)
self.assertEqual(transaction_hash, stored_hash)
self.assertEqual(transaction.tx_val(), stored_content)
stored_hash2, stored_content2, _ = self.gateway_node.get_tx_service().get_transaction(i + 21)
self.assertEqual(transaction_hash, stored_hash2)
self.assertEqual(transaction.tx_val(), stored_content2)
# only 10 times even with rebroadcast SID
self.assertEqual(len(transactions), self.gateway_node.send_msg_to_node.call_count)
def test_msg_tx_duplicate_ignore(self):
transactions = self.bx_transactions(assign_short_ids=True)
for transaction in transactions:
self.sut.msg_tx(transaction)
for transaction in transactions:
self.sut.msg_tx(transaction)
self.assertEqual(len(transactions), self.gateway_node.send_msg_to_node.call_count)
def test_get_txs_block_recovery(self):
btc_block = self.btc_block()
transactions = self.btc_transactions()
# assign short ids that the local connection won't know about until it gets the txs message
remote_transaction_service = ExtensionTransactionService(MockNode(
gateway_helpers.get_gateway_opts(8999)), 0)
short_id_mapping = {}
for i, transaction in enumerate(transactions):
remote_transaction_service.assign_short_id(transaction.tx_hash(), i + 1)
remote_transaction_service.set_transaction_contents(transaction.tx_hash(), transaction.tx())
short_id_mapping[transaction.tx_hash()] = TransactionInfo(transaction.tx_hash(), transaction.tx(), i + 1)
bx_block = bytes(self.gateway_node.message_converter.block_to_bx_block(btc_block, remote_transaction_service)[0])
self.gateway_node.block_recovery_service.add_block = \
MagicMock(wraps=self.gateway_node.block_recovery_service.add_block)
self.gateway_node.send_msg_to_node = MagicMock()
self.gateway_node.broadcast = MagicMock()
key, ciphertext = symmetric_encrypt(bx_block)
block_hash = crypto.double_sha256(ciphertext)
key_message = KeyMessage(Sha256Hash(block_hash), DEFAULT_NETWORK_NUM, "", key)
broadcast_message = BroadcastMessage(Sha256Hash(block_hash), DEFAULT_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True, ciphertext)
self.sut.msg_broadcast(broadcast_message)
self.gateway_node.broadcast.reset_mock()
self.sut.msg_key(key_message)
self.gateway_node.block_recovery_service.add_block.assert_called_once()
self.assertEqual(2, self.gateway_node.broadcast.call_count)
recovery_broadcast = self.gateway_node.broadcast.call_args_list[0]
((gettxs_message,), recovery_kwargs) = recovery_broadcast
self.assertIsInstance(gettxs_message, GetTxsMessage)
self.assertIn(ConnectionType.RELAY_TRANSACTION, recovery_kwargs["connection_types"])
key_broadcast = self.gateway_node.broadcast.call_args_list[1]
((key_message, _conn), recovery_kwargs) = key_broadcast
self.assertIsInstance(key_message, KeyMessage)
self.assertIn(ConnectionType.GATEWAY, recovery_kwargs["connection_types"])
txs = [tx for tx in short_id_mapping.values()]
txs_message = TxsMessage(txs=txs)
self.sut.msg_txs(txs_message)
self._assert_block_sent(btc_block)
def test_get_txs_multiple_sid_assignments(self):
transactions = self.btc_transactions()
# assign short ids that the local connection won't know about until it gets the txs message
remote_transaction_service1 = ExtensionTransactionService(MockNode(
gateway_helpers.get_gateway_opts(8999)), 0)
short_id_mapping1 = {}
for i, transaction in enumerate(transactions):
remote_transaction_service1.assign_short_id(transaction.tx_hash(), i + 1)
remote_transaction_service1.set_transaction_contents(transaction.tx_hash(), transaction.tx())
short_id_mapping1[transaction.tx_hash()] = TransactionInfo(transaction.tx_hash(), transaction.tx(), i + 1)
txs_message_1 = TxsMessage([tx for tx in short_id_mapping1.values()])
self.sut.msg_txs(txs_message_1)
for transaction_hash, tx_info in short_id_mapping1.items():
self.assertEqual(tx_info.short_id, self.gateway_node.get_tx_service().get_short_id(transaction_hash))
stored_hash, stored_content, _ = self.gateway_node.get_tx_service().get_transaction(tx_info.short_id)
self.assertEqual(transaction_hash, stored_hash)
self.assertEqual(tx_info.contents, stored_content)
remote_transaction_service2 = ExtensionTransactionService(MockNode(
gateway_helpers.get_gateway_opts(8999)), 0)
short_id_mapping2 = {}
for i, transaction in enumerate(transactions):
remote_transaction_service2.assign_short_id(transaction.tx_hash(), i + 101)
remote_transaction_service2.set_transaction_contents(transaction.tx_hash(), transaction.tx())
short_id_mapping2[transaction.tx_hash()] = TransactionInfo(transaction.tx_hash(), transaction.tx(), i + 101)
txs_message_2 = TxsMessage([tx for tx in short_id_mapping2.values()])
self.sut.msg_txs(txs_message_2)
for transaction_hash, tx_info in short_id_mapping2.items():
stored_hash, stored_content, _ = self.gateway_node.get_tx_service().get_transaction(tx_info.short_id)
self.assertEqual(transaction_hash, stored_hash)
self.assertEqual(tx_info.contents, stored_content)
def _assert_block_sent(self, btc_block):
self.gateway_node.send_msg_to_node.assert_called()
calls = self.gateway_node.send_msg_to_node.call_args_list
self.assertEqual(2, len(calls))
((sent_block_msg,), _) = calls[0]
self.assertEqual(btc_block.version(), sent_block_msg.version())
self.assertEqual(btc_block.magic(), sent_block_msg.magic())
self.assertEqual(btc_block.prev_block_hash(), sent_block_msg.prev_block_hash())
self.assertEqual(btc_block.merkle_root(), sent_block_msg.merkle_root())
self.assertEqual(btc_block.timestamp(), sent_block_msg.timestamp())
self.assertEqual(btc_block.bits(), sent_block_msg.bits())
self.assertEqual(btc_block.nonce(), sent_block_msg.nonce())
self.assertEqual(btc_block.txn_count(), sent_block_msg.txn_count())
((sent_inv_msg,), _) = calls[1]
self.assertIsInstance(sent_inv_msg, InvBtcMessage)
sent_inv_msg = cast(InvBtcMessage, sent_inv_msg)
inv_items = list(iter(sent_inv_msg))
self.assertEqual(1, len(inv_items))
block_inv = inv_items[0]
self.assertEqual(InventoryType.MSG_BLOCK, block_inv[0])
self.assertEqual(btc_block.block_hash(), block_inv[1])