def _test_block_cleanup(self):
block_msg = self._get_sample_block(self._get_file_path())
transactions = block_msg.txns()[:]
block_hash = typing.cast(BtcObjectHash, block_msg.block_hash())
random.shuffle(transactions)
short_len = int(len(transactions) * 0.9)
transactions_short = transactions[:short_len]
unknown_transactions = transactions[short_len:]
transaction_hashes = []
for idx, tx in enumerate(transactions_short):
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=btc_constants.BTC_SHA_HASH_LEN)
transaction_hashes.append(tx_hash)
self.transaction_service.set_transaction_contents(tx_hash, tx)
self.transaction_service.assign_short_id(tx_hash, idx + 1)
for idx, tx in enumerate(unknown_transactions):
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=btc_constants.BTC_SHA_HASH_LEN)
transaction_hashes.append(tx_hash)
if idx % 2 == 0:
self.transaction_service.set_transaction_contents(tx_hash, tx)
self.cleanup_service._block_hash_marked_for_cleanup.add(block_hash)
self.cleanup_service.clean_block_transactions(block_msg,
self.transaction_service)
self.assertEqual(0, self.transaction_service._total_tx_contents_size)
for tx_hash in transaction_hashes:
self.assertFalse(
self.transaction_service.has_transaction_contents(tx_hash))
def start_transaction_recovery(
self,
unknown_sids: Iterable[int],
unknown_hashes: Iterable[Sha256Hash],
block_hash: Sha256Hash,
connection: Optional[AbstractRelayConnection] = None) -> None:
all_unknown_sids = []
all_unknown_sids.extend(unknown_sids)
tx_service = self._node.get_tx_service()
# retrieving sids of txs with unknown contents
for tx_hash in unknown_hashes:
transaction_key = tx_service.get_transaction_key(tx_hash)
tx_sid = tx_service.get_short_id_by_key(transaction_key)
all_unknown_sids.append(tx_sid)
if not self._node.opts.request_recovery:
if connection is not None:
network_num = connection.network_num
else:
network_num = self._node.network_num
# log recovery started to match with recovery completing
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_RECOVERY_STARTED,
network_num=network_num,
txs_count=len(all_unknown_sids),
more_info="recovery from relay is disabled",
)
return
get_txs_message = GetTxsMessage(short_ids=all_unknown_sids)
self._node.broadcast(
get_txs_message,
connection_types=(ConnectionType.RELAY_TRANSACTION, ))
if connection is not None:
tx_stats.add_txs_by_short_ids_event(
all_unknown_sids,
TransactionStatEventType.
TX_UNKNOWN_SHORT_IDS_REQUESTED_BY_GATEWAY_FROM_RELAY,
network_num=self._node.network_num,
peers=[connection],
block_hash=convert.bytes_to_hex(block_hash.binary))
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_RECOVERY_STARTED,
network_num=connection.network_num,
txs_count=len(all_unknown_sids),
request_hash=convert.bytes_to_hex(
crypto.double_sha256(get_txs_message.rawbytes())))
else:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_RECOVERY_REPEATED,
network_num=self._node.network_num,
txs_count=len(all_unknown_sids),
request_hash=convert.bytes_to_hex(
crypto.double_sha256(get_txs_message.rawbytes())))
def test_msg_broadcast_duplicate_block_with_different_short_id(self):
# test scenario when first received block is compressed with unknown short ids,
# but second received block is compressed with known short ids
ont_block = self.ont_block()
block_hash = ont_block.block_hash()
transactions = self.bx_transactions()
unknown_sid_transaction_service = ExtensionTransactionService(MockNode(
gateway_helpers.get_gateway_opts(8999)), 0)
for i, transaction in enumerate(transactions):
unknown_sid_transaction_service.assign_short_id(transaction.tx_hash(), i)
unknown_sid_transaction_service.set_transaction_contents(transaction.tx_hash(), transaction.tx_val())
unknown_short_id_block = bytes(
self.gateway_node.message_converter.block_to_bx_block(
ont_block, unknown_sid_transaction_service, True, self.gateway_node.network.min_tx_age_seconds
)[0]
)
unknown_key, unknown_cipher = symmetric_encrypt(unknown_short_id_block)
unknown_block_hash = crypto.double_sha256(unknown_cipher)
unknown_message = BroadcastMessage(Sha256Hash(unknown_block_hash), self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, False, bytearray(unknown_short_id_block))
unknown_key_message = KeyMessage(Sha256Hash(unknown_block_hash), self.TEST_NETWORK_NUM, "", unknown_key)
local_transaction_service = self.gateway_node.get_tx_service()
for i, transaction in enumerate(transactions):
local_transaction_service.assign_short_id(transaction.tx_hash(), i + 20)
local_transaction_service.set_transaction_contents(transaction.tx_hash(), transaction.tx_val())
known_short_id_block = bytes(
self.gateway_node.message_converter.block_to_bx_block(
ont_block, local_transaction_service, True, self.gateway_node.network.min_tx_age_seconds
)[0]
)
known_key, known_cipher = symmetric_encrypt(known_short_id_block)
known_block_hash = crypto.double_sha256(known_cipher)
known_message = BroadcastMessage(Sha256Hash(known_block_hash), self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, False, bytearray(known_short_id_block))
known_key_message = KeyMessage(Sha256Hash(known_block_hash), self.TEST_NETWORK_NUM, "", known_key)
self.sut.msg_broadcast(unknown_message)
self.sut.msg_key(unknown_key_message)
self.assertEqual(1, len(self.gateway_node.block_queuing_service))
self.assertEqual(True, self.gateway_node.block_queuing_service._blocks_waiting_for_recovery[block_hash])
self.assertEqual(1, len(self.gateway_node.block_recovery_service._block_hash_to_bx_block_hashes))
self.assertNotIn(block_hash, self.gateway_node.blocks_seen.contents)
self.sut.msg_broadcast(known_message)
self.sut.msg_key(known_key_message)
self.gateway_node_sut.msg_get_data(GetDataOntMessage(self.magic, InventoryOntType.MSG_BLOCK.value, block_hash))
self.gateway_node.broadcast.assert_called()
self.assertEqual(0, len(self.gateway_node.block_queuing_service))
self.assertEqual(0, len(self.gateway_node.block_recovery_service._block_hash_to_bx_block_hashes))
self.assertIn(block_hash, self.gateway_node.blocks_seen.contents)
def setUp(self):
super().setUp()
self.node1.alarm_queue = AlarmQueue()
self.node2.alarm_queue = AlarmQueue()
self.network_num = 1
self.magic = 12345
self.version = 23456
self.prev_block_hash = bytearray(crypto.double_sha256(b"123"))
self.prev_block = BtcObjectHash(self.prev_block_hash,
length=crypto.SHA256_HASH_LEN)
self.merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
self.merkle_root = BtcObjectHash(self.merkle_root_hash,
length=crypto.SHA256_HASH_LEN)
self.bits = 2
self.nonce = 3
opts = self.gateway_1_opts()
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.btc_message_converter = btc_message_converter_factory.create_btc_message_converter(
self.magic, opts)
self.btc_transactions = [
TxBtcMessage(self.magic, self.version, [], [], i)
for i in range(self.TRANSACTIONS_COUNT)
]
self.btc_transactions_for_block = [
tx_btc_message.rawbytes()[btc_constants.BTC_HDR_COMMON_OFF:]
for tx_btc_message in self.btc_transactions
]
self.transactions = [
self.btc_message_converter.tx_to_bx_txs(tx_btc_message,
self.network_num)[0][0]
for tx_btc_message in self.btc_transactions
]
self.transactions_with_short_ids = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1, tx_message.tx_val())
for i, tx_message in enumerate(self.transactions)
]
self.transactions_with_no_content = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1) for i, tx_message in enumerate(self.transactions)
]
self.transactions_by_short_id = {
tx_message.short_id(): tx_message
for tx_message in self.transactions_with_short_ids
}
self.block = BlockBtcMessage(self.magic, self.version,
self.prev_block, self.merkle_root,
int(time.time()), self.bits, self.nonce,
self.btc_transactions_for_block)
def bdn_tx_to_bx_tx(
self,
raw_tx: Union[bytes, bytearray, memoryview],
network_num: int,
quota_type: Optional[QuotaType] = None
) -> TxMessage:
return TxMessage(Sha256Hash(crypto.double_sha256(raw_tx)), network_num, tx_val=raw_tx)
def add_block(self, bx_block: memoryview, block_hash: Sha256Hash,
unknown_tx_sids: List[int],
unknown_tx_hashes: List[Sha256Hash]):
"""
Adds a block that needs to recovery. Tracks unknown short ids and contents as they come in.
:param bx_block: bytearray representation of compressed block
:param block_hash: original ObjectHash of block
:param unknown_tx_sids: list of unknown short ids
:param unknown_tx_hashes: list of unknown tx ObjectHashes
"""
logger.trace(
"Recovering block with {} unknown short ids and {} contents: {}",
len(unknown_tx_sids), len(unknown_tx_hashes), block_hash)
bx_block_hash = Sha256Hash(crypto.double_sha256(bx_block))
self._bx_block_hash_to_block[bx_block_hash] = bx_block
self._bx_block_hash_to_block_hash[bx_block_hash] = block_hash
self._bx_block_hash_to_sids[bx_block_hash] = set(unknown_tx_sids)
self._bx_block_hash_to_tx_hashes[bx_block_hash] = set(
unknown_tx_hashes)
self._block_hash_to_bx_block_hashes[block_hash].add(bx_block_hash)
for sid in unknown_tx_sids:
self._sid_to_bx_block_hashes[sid].add(bx_block_hash)
for tx_hash in unknown_tx_hashes:
self._tx_hash_to_bx_block_hashes[tx_hash].add(bx_block_hash)
self._blocks_expiration_queue.add(bx_block_hash)
self._schedule_cleanup()
def test_bdn_stats_block_new_from_bdn_ignore_from_node(self):
block_msg = NewBlockEthProtocolMessage(
None,
_block_with_timestamp(time.time() + 1 - (
self.node.opts.blockchain_ignore_block_interval_count *
self.node.opts.blockchain_block_interval)), 10)
internal_new_block_msg = InternalEthBlockInfo.from_new_block_msg(
block_msg)
msg_bytes, block_info = self.node.message_converter.block_to_bx_block(
internal_new_block_msg, self.node._tx_service, True,
self.node.network.min_tx_age_seconds)
msg_hash = Sha256Hash(crypto.double_sha256(msg_bytes))
broadcast_msg = BroadcastMessage(message_hash=msg_hash,
network_num=1,
is_encrypted=False,
blob=msg_bytes)
self.relay_connection.msg_broadcast(broadcast_msg)
block_msg.serialize()
self.block_blockchain_connection_protocol.msg_block(block_msg)
self.assertEqual(
3,
len(gateway_bdn_performance_stats_service.interval_data.
blockchain_node_to_bdn_stats))
for stats in gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats.values(
):
self.assertEqual(1, stats.new_blocks_received_from_bdn)
self.assertEqual(0, stats.new_blocks_received_from_blockchain_node)
def test_msg_broadcast_wait_for_key(self):
ont_block = self.ont_block()
bx_block = self.bx_block(ont_block)
key, ciphertext = symmetric_encrypt(bx_block)
block_hash = crypto.double_sha256(ciphertext)
broadcast_message = BroadcastMessage(Sha256Hash(block_hash),
self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True,
ciphertext)
self.sut.msg_broadcast(broadcast_message)
# handle duplicate messages
self.sut.msg_broadcast(broadcast_message)
self.sut.msg_broadcast(broadcast_message)
self.assertEqual(3, len(self.gateway_node.broadcast.call_args_list))
for call, conn_type in self.gateway_node.broadcast.call_args_list:
msg, conn = call
self.assertTrue(isinstance(msg, BlockReceivedMessage))
self.gateway_node.broadcast.reset_mock()
self.assertEqual(1, len(self.gateway_node.in_progress_blocks))
key_message = KeyMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM,
"", key)
self.sut.msg_key(key_message)
self._assert_block_sent(ont_block)
def get_block_info(
bx_block: memoryview,
block_hash: Sha256Hash,
short_ids: List[int],
decompress_start_datetime: datetime,
decompress_start_timestamp: float,
total_tx_count: Optional[int] = None,
btc_block_msg: Optional[BlockBtcMessage] = None
) -> BlockInfo:
if btc_block_msg is not None:
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(bx_block))
compressed_size = len(bx_block)
prev_block_hash = convert.bytes_to_hex(btc_block_msg.prev_block_hash().binary)
btc_block_len = len(btc_block_msg.rawbytes())
compression_rate = 100 - float(compressed_size) / btc_block_len * 100
else:
bx_block_hash = None
compressed_size = None
prev_block_hash = None
btc_block_len = None
compression_rate = None
return BlockInfo(
block_hash,
short_ids,
decompress_start_datetime,
datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
total_tx_count,
bx_block_hash,
prev_block_hash,
btc_block_len,
compressed_size,
compression_rate
)
def _propagate_block_to_gateway_peers(self, cipher_hash, bx_block):
"""
Propagates unencrypted bx_block to all gateway peers for encryption and sending to bloXroute.
Also sends keys to bloXroute in case this was user error (e.g. no gateway peers).
Called after a timeout. This invalidates all future bx_block receipts.
"""
bx_block_hash = crypto.double_sha256(bx_block)
hex_bx_block_hash = convert.bytes_to_hex(bx_block_hash)
logger.debug("Did not receive enough receipts for: {}. Propagating compressed block to other gateways: {}",
cipher_hash, hex_bx_block_hash)
self._send_key(cipher_hash)
request = BlockPropagationRequestMessage(bx_block)
conns = self._node.broadcast(request, None, connection_types=(ConnectionType.GATEWAY,))
block_stats.add_block_event_by_block_hash(cipher_hash,
BlockStatEventType.ENC_BLOCK_PROPAGATION_NEEDED,
network_num=self._node.network_num,
compressed_block_hash=hex_bx_block_hash,
peers=conns,
more_info="Peers: {}, {} receipts".format(
stats_format.connections(conns),
self._receipt_tracker[cipher_hash]))
del self._receipt_tracker[cipher_hash]
del self._alarms[cipher_hash]
return constants.CANCEL_ALARMS
def __init__(self,
magic: Optional[int] = None,
command: Optional[bytes] = None,
payload_len: Optional[int] = None,
buf: Optional[bytearray] = None):
self.buf = buf
# pyre-fixme[6]: Expected `Union[bytearray, bytes, memoryview]` for 1st
# param but got `Optional[bytearray]`.
self._memoryview = memoryview(buf)
# pyre-fixme[6]: Expected `int` for 1st param but got `Optional[int]`.
checksum = crypto.double_sha256(
self._memoryview[ont_constants.ONT_HDR_COMMON_OFF:payload_len +
ont_constants.ONT_HDR_COMMON_OFF])
off = 0
# pyre-fixme[6]: Expected `Union[array.array[typing.Any], bytearray,
# memoryview, mmap.mmap]` for 2nd param but got `Optional[bytearray]`.
struct.pack_into("<L12sL", buf, off, magic, command, payload_len)
off += ont_constants.ONT_HEADER_MINUS_CHECKSUM
# pyre-fixme[16]: `Optional` has no attribute `__setitem__`.
buf[off:off + ont_constants.ONT_MSG_CHECKSUM_LEN] = checksum[0:4]
self._magic = magic
self._command = command
self._payload_len = payload_len
self._payload = None
self._checksum = None
def test_get_txs_block_recovery_encrypted(self):
block: BlockOntMessage = self.ont_block()
transactions: List[TxOntMessage] = self.ont_transactions(block)
# assign short ids that the local connection won't know about until it gets the txs message
remote_transaction_service = ExtensionTransactionService(
MockNode(gateway_helpers.get_gateway_opts(8999)), 0)
short_id_mapping = {}
for i, transaction in enumerate(transactions):
tx_hash = transaction.tx_hash()
remote_transaction_service.assign_short_id(tx_hash, i + 1)
remote_transaction_service.set_transaction_contents(
tx_hash, transaction.rawbytes())
short_id_mapping[tx_hash] = TransactionInfo(
tx_hash, transaction.rawbytes(), i + 1)
bx_block = bytes(
self.gateway_node.message_converter.block_to_bx_block(
block, remote_transaction_service)[0])
self.gateway_node.block_recovery_service.add_block = \
MagicMock(wraps=self.gateway_node.block_recovery_service.add_block)
self.gateway_node.send_msg_to_node = MagicMock()
self.gateway_node.broadcast = MagicMock()
key, ciphertext = symmetric_encrypt(bx_block)
block_hash = crypto.double_sha256(ciphertext)
key_message = KeyMessage(Sha256Hash(block_hash), DEFAULT_NETWORK_NUM,
"", key)
broadcast_message = BroadcastMessage(Sha256Hash(block_hash),
DEFAULT_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True,
ciphertext)
self.sut.msg_broadcast(broadcast_message)
self.gateway_node.broadcast.reset_mock()
self.sut.msg_key(key_message)
self.gateway_node.block_recovery_service.add_block.assert_called_once()
self.assertEqual(2, self.gateway_node.broadcast.call_count)
recovery_broadcast = self.gateway_node.broadcast.call_args_list[0]
((gettxs_message, ), recovery_kwargs) = recovery_broadcast
self.assertIsInstance(gettxs_message, GetTxsMessage)
self.assertIn(ConnectionType.RELAY_TRANSACTION,
recovery_kwargs["connection_types"])
key_broadcast = self.gateway_node.broadcast.call_args_list[1]
((key_message, _conn), recovery_kwargs) = key_broadcast
self.assertIsInstance(key_message, KeyMessage)
self.assertIn(ConnectionType.GATEWAY,
recovery_kwargs["connection_types"])
txs = [tx for tx in short_id_mapping.values()]
txs_message = TxsMessage(txs=txs)
self.sut.msg_txs(txs_message)
self._assert_block_sent(block)
def get_txid(
buffer: Union[memoryview, bytearray]) -> Tuple[OntObjectHash, int]:
if not isinstance(buffer, memoryview):
buffer = memoryview(buffer)
off = 1
txtype = buffer[off:off + 1]
off += 1 + 40
# Deploy type
if txtype == ont_common_constants.ONT_TX_DEPLOY_TYPE_INDICATOR_AS_BYTEARRAY:
buffer_length, size = ont_varint_to_int(buffer, off)
off += buffer_length + size
off += 1
buffer_length, size = ont_varint_to_int(buffer, off)
off += buffer_length + size
buffer_length, size = ont_varint_to_int(buffer, off)
off += buffer_length + size
buffer_length, size = ont_varint_to_int(buffer, off)
off += buffer_length + size
buffer_length, size = ont_varint_to_int(buffer, off)
off += buffer_length + size
buffer_length, size = ont_varint_to_int(buffer, off)
off += buffer_length + size
# Invoke type
elif txtype == ont_common_constants.ONT_TX_INVOKE_TYPE_INDICATOR_AS_BYTEARRAY:
buffer_length, size = ont_varint_to_int(buffer, off)
off += buffer_length + size
_, size = ont_varint_to_int(buffer, off)
off += size
return OntObjectHash(buf=crypto.double_sha256(buffer[:off]),
length=ont_common_constants.ONT_HASH_LEN), off
def test_msg_key_wait_for_broadcast(self):
ont_block = self.ont_block()
bx_block = self.bx_block(ont_block)
key, ciphertext = symmetric_encrypt(bx_block)
block_hash = crypto.double_sha256(ciphertext)
self.gateway_node.broadcast.assert_not_called()
key_message = KeyMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM,
"", key)
self.sut.msg_key(key_message)
# handle duplicate broadcasts
self.sut.msg_key(key_message)
self.sut.msg_key(key_message)
self.assertEqual(1, len(self.gateway_node.in_progress_blocks))
broadcast_message = BroadcastMessage(Sha256Hash(block_hash),
self.TEST_NETWORK_NUM, "",
BroadcastMessageType.BLOCK, True,
ciphertext)
self.sut.msg_broadcast(broadcast_message)
self._assert_block_sent(ont_block)
def block_to_bx_block(
self, block_msg, tx_service
) -> Tuple[memoryview, BlockInfo]:
"""
Compresses a Bitcoin block's transactions and packs it into a bloXroute block.
"""
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
size = 0
buf = deque()
short_ids = []
header = block_msg.header()
size += len(header)
buf.append(header)
for tx in block_msg.txns():
tx_hash = btc_common_utils.get_txid(tx)
short_id = tx_service.get_short_id(tx_hash)
if short_id == constants.NULL_TX_SID:
buf.append(tx)
size += len(tx)
else:
short_ids.append(short_id)
buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY)
size += 1
serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids)
buf.append(serialized_short_ids)
size += constants.UL_ULL_SIZE_IN_BYTES
offset_buf = struct.pack("<Q", size)
buf.appendleft(offset_buf)
size += len(serialized_short_ids)
block = bytearray(size)
off = 0
for blob in buf:
next_off = off + len(blob)
block[off:next_off] = blob
off = next_off
prev_block_hash = convert.bytes_to_hex(block_msg.prev_block_hash().binary)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
original_size = len(block_msg.rawbytes())
block_info = BlockInfo(
block_msg.block_hash(),
short_ids,
compress_start_datetime,
datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
block_msg.txn_count(),
bx_block_hash,
prev_block_hash,
original_size,
size,
100 - float(size) / original_size * 100
)
return memoryview(block), block_info
def block_to_bx_block(
self, block_msg, tx_service, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
"""
Compresses a Bitcoin block's transactions and packs it into a bloXroute block.
"""
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
size = 0
buf = deque()
short_ids = []
original_size = len(block_msg.rawbytes())
ignored_sids = []
header = block_msg.header()
size += len(header)
buf.append(header)
max_timestamp_for_compression = time.time() - min_tx_age_seconds
for tx in block_msg.txns():
tx_hash = btc_common_utils.get_txid(tx)
transaction_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(transaction_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(
short_id)
if short_id == constants.NULL_TX_SID or \
not enable_block_compression or \
short_id_assign_time > max_timestamp_for_compression:
if short_id != constants.NULL_TX_SIDS:
ignored_sids.append(short_id)
buf.append(tx)
size += len(tx)
else:
short_ids.append(short_id)
buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY)
size += 1
block = finalize_block_bytes(buf, size, short_ids)
prev_block_hash = convert.bytes_to_hex(
block_msg.prev_block_hash().binary)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(block_msg.block_hash(), short_ids,
compress_start_datetime, datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
block_msg.txn_count(), bx_block_hash,
prev_block_hash, original_size, size,
100 - float(size) / original_size * 100,
ignored_sids)
return memoryview(block), block_info
def msg_headers(self, msg: HeadersOntMessage):
if not self.node.should_process_block_hash():
return
header = msg.headers()[-1]
raw_hash = crypto.double_sha256(header)
reply = GetDataOntMessage(self.magic, InventoryOntType.MSG_BLOCK.value,
OntObjectHash(buf=raw_hash, length=ont_constants.ONT_HASH_LEN))
self.connection.enqueue_msg(reply)
def encrypt_and_add_payload(self, payload):
"""
Encrypts payload, computing a hash and storing it along with the key for later release.
If encryption is disabled for dev, store ciphertext identical to hash_key.
"""
key, ciphertext = symmetric_encrypt(bytes(payload))
hash_key = crypto.double_sha256(ciphertext)
self._add(hash_key, key, ciphertext, payload)
return ciphertext, hash_key
def bdn_tx_to_bx_tx(
self,
raw_tx: Union[bytes, bytearray, memoryview],
network_num: int,
transaction_flag: Optional[TransactionFlag] = None) -> TxMessage:
return TxMessage(Sha256Hash(crypto.double_sha256(raw_tx)),
network_num,
tx_val=raw_tx,
transaction_flag=transaction_flag)
class BtcConnectionProtocolTest(AbstractTestCase):
HASH = BtcObjectHash(binary=crypto.double_sha256(b"123"))
def setUp(self):
opts = gateway_helpers.get_gateway_opts(8000,
include_default_btc_args=True)
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.node = MockGatewayNode(opts)
self.node.block_processing_service = MagicMock()
self.connection = MagicMock()
gateway_helpers.add_blockchain_peer(self.node, self.connection)
self.connection.node = self.node
self.connection.peer_ip = LOCALHOST
self.connection.peer_port = 8001
self.connection.network_num = 2
self.connection.endpoint = IpEndpoint(self.connection.peer_ip,
self.connection.peer_port)
self.node.blockchain_peers.add(
BlockchainPeerInfo(self.connection.peer_ip,
self.connection.peer_port))
gateway_bdn_performance_stats_service.set_node(self.node)
self.sut = BtcBaseConnectionProtocol(self.connection)
def test_msg_block_success(self):
block_timestamp = int(
time.time()
) + 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval
txns = [
TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
message = BlockBtcMessage(0, 0, self.HASH, self.HASH, block_timestamp,
0, 0, txns)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_called_once(
)
def test_msg_block_too_old(self):
block_timestamp = int(
time.time()
) - 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval
txns = [
TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
message = BlockBtcMessage(0, 0, self.HASH, self.HASH, 0,
block_timestamp, 0, txns)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_not_called(
)
def test_get_connection_protocol_version__wrong_message(self):
wrong_message = BroadcastMessage(message_hash=Sha256Hash(
crypto.double_sha256(b"hello")),
network_num=1,
source_id="",
blob=bytearray(1))
input_buffer = InputBuffer()
input_buffer.add_bytes(wrong_message.rawbytes())
self.assertEqual(
3,
self.version_manager.get_connection_protocol_version(input_buffer))
def start_transaction_recovery(
self,
unknown_sids: Iterable[int],
unknown_hashes: Iterable[Sha256Hash],
block_hash: Sha256Hash,
connection: Optional[AbstractRelayConnection] = None):
all_unknown_sids = []
all_unknown_sids.extend(unknown_sids)
tx_service = self._node.get_tx_service()
# retrieving sids of txs with unknown contents
for tx_hash in unknown_hashes:
tx_sid = tx_service.get_short_id(tx_hash)
all_unknown_sids.append(tx_sid)
get_txs_message = GetTxsMessage(short_ids=all_unknown_sids)
self._node.broadcast(
get_txs_message,
connection_types=[ConnectionType.RELAY_TRANSACTION])
if connection is not None:
tx_stats.add_txs_by_short_ids_event(
all_unknown_sids,
TransactionStatEventType.
TX_UNKNOWN_SHORT_IDS_REQUESTED_BY_GATEWAY_FROM_RELAY,
network_num=self._node.network_num,
peer=connection.peer_desc,
block_hash=convert.bytes_to_hex(block_hash.binary))
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_RECOVERY_STARTED,
network_num=connection.network_num,
request_hash=convert.bytes_to_hex(
crypto.double_sha256(get_txs_message.rawbytes())))
else:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_RECOVERY_REPEATED,
network_num=self._node.network_num,
request_hash=convert.bytes_to_hex(
crypto.double_sha256(get_txs_message.rawbytes())))
def test_msg_broadcast_encrypted(self, mock_handle_decrypted_block):
msg_bytes = helpers.generate_bytearray(50)
msg_hash = Sha256Hash(crypto.double_sha256(msg_bytes))
broadcast_msg = BroadcastMessage(message_hash=msg_hash, network_num=1, is_encrypted=True, blob=msg_bytes)
self.connection.msg_broadcast(broadcast_msg)
self.assertTrue(self.node.in_progress_blocks.has_ciphertext_for_hash(msg_hash))
self.assertEqual(1, len(self.node.broadcast_messages))
self.assertIsInstance(self.node.broadcast_messages[0][0], BlockReceivedMessage)
mock_handle_decrypted_block.assert_not_called()
def bdn_tx_to_bx_tx(
self,
raw_tx: Union[bytes, bytearray, memoryview],
network_num: int,
transaction_flag: Optional[TransactionFlag] = None,
account_id: str = common_constants.DECODED_EMPTY_ACCOUNT_ID
) -> TxMessage:
return TxMessage(Sha256Hash(crypto.double_sha256(raw_tx)),
network_num,
tx_val=raw_tx,
transaction_flag=transaction_flag,
account_id=account_id)
def msg_block_propagation_request(self, msg):
bx_block = msg.blob()
bx_block_hash = crypto.double_sha256(bx_block)
block_stats.add_block_event_by_block_hash(
bx_block_hash,
BlockStatEventType.BX_BLOCK_PROPAGATION_REQUESTED_BY_PEER,
network_num=self.network_num,
peers=[self],
)
self.node.neutrality_service.propagate_block_to_network(bx_block,
self,
from_peer=True)
def test_msg_broadcast_unencrypted(self, mock_handle_decrypted_block):
msg_bytes = helpers.generate_bytearray(50)
msg_hash = Sha256Hash(crypto.double_sha256(msg_bytes))
broadcast_msg = BroadcastMessage(message_hash=msg_hash, network_num=1, is_encrypted=False, blob=msg_bytes)
self.connection.msg_broadcast(broadcast_msg)
self.assertFalse(self.node.in_progress_blocks.has_ciphertext_for_hash(msg_hash))
self.assertEqual(0, len(self.node.broadcast_messages))
mock_handle_decrypted_block.assert_called_once()
class OntConnectionProtocolTest(AbstractTestCase):
HASH = OntObjectHash(binary=crypto.double_sha256(b"123"))
def setUp(self):
opts = gateway_helpers.get_gateway_opts(8000,
include_default_ont_args=True)
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.node = MockOntGatewayNode(opts)
self.node.block_processing_service = MagicMock()
self.connection = MagicMock()
gateway_helpers.add_blockchain_peer(self.node, self.connection)
self.connection.node = self.node
self.connection.peer_ip = LOCALHOST
self.connection.peer_port = 8001
self.connection.network_num = 2
self.connection.endpoint = IpEndpoint(self.connection.peer_ip,
self.connection.peer_port)
self.node.blockchain_peers.add(
BlockchainPeerInfo(self.connection.peer_ip,
self.connection.peer_port))
gateway_bdn_performance_stats_service.set_node(self.node)
self.sut = OntBaseConnectionProtocol(self.connection)
def test_msg_block_success(self):
block_timestamp = int(time.time()) + 1 - \
self.node.opts.blockchain_ignore_block_interval_count * \
self.node.opts.blockchain_block_interval
message = BlockOntMessage(0, 0, self.HASH, self.HASH,
self.HASH, block_timestamp, 0, 0, bytes(10),
bytes(20), [bytes(33)] * 5, [bytes(2)] * 3,
[bytes(32)] * 5, self.HASH)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_called_once(
)
def test_msg_block_too_old(self):
block_timestamp = int(time.time()) - 1 - \
self.node.opts.blockchain_ignore_block_interval_count * \
self.node.opts.blockchain_block_interval
message = BlockOntMessage(0, 0, self.HASH, self.HASH,
self.HASH, block_timestamp, 0, 0, bytes(10),
bytes(20), [bytes(33)] * 5, [bytes(2)] * 3,
[bytes(32)] * 5, self.HASH)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_not_called(
)
def clean_block_transactions(
self, block_msg: BlockBtcMessage,
transaction_service: TransactionService) -> None:
block_short_ids = []
block_unknown_tx_hashes = []
start_time = time.time()
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
for tx in block_msg.txns():
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=BTC_SHA_HASH_LEN)
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
block_hash = block_msg.block_hash()
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, block_msg.txn_count(),
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=block_short_ids,
unknown_tx_hashes=block_unknown_tx_hashes)
def btc_block(timestamp=None, real_block=None):
if real_block is not None:
return BlockBtcMessage(buf=bytearray(convert.hex_to_bytes(real_block)))
if timestamp is None:
timestamp = int(time.time())
magic = 12345
version = 23456
prev_block_hash = bytearray(crypto.double_sha256(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
bits = 2
nonce = 3
txns = [
TxBtcMessage(magic, version, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
return BlockBtcMessage(magic, version, prev_block, merkle_root, timestamp,
bits, nonce, txns)
def block_hash(self) -> OntObjectHash:
if self._hash_val is None:
if self._header_offset is None:
self.parse_message()
assert self._header_offset is not None
header = self._memoryview[ont_constants.ONT_HDR_COMMON_OFF:self.
_header_offset]
raw_hash = crypto.double_sha256(header)
self._hash_val = OntObjectHash(buf=raw_hash,
length=ont_constants.ONT_HASH_LEN)
hash_val = self._hash_val
assert isinstance(hash_val, OntObjectHash)
return hash_val