def _test_mark_blocks_and_request_cleanup(self): self.node.has_active_blockchain_peer = MagicMock(return_value=True) self.cleanup_service._request_block = MagicMock() marked_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) prev_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) tracked_blocks = [] self.cleanup_service.on_new_block_received(marked_block, prev_block) self.transaction_service.track_seen_short_ids(marked_block, []) for _ in range(self.block_confirmations_count - 1): tracked_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) self.transaction_service.track_seen_short_ids(tracked_block, []) tracked_blocks.append(tracked_block) unmarked_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) self.assertIsNone(self.cleanup_service.last_confirmed_block) self.cleanup_service.mark_blocks_and_request_cleanup( [marked_block, *tracked_blocks]) self.assertEqual(marked_block, self.cleanup_service.last_confirmed_block) self.assertTrue( self.cleanup_service.is_marked_for_cleanup(marked_block)) self.assertFalse( self.cleanup_service.is_marked_for_cleanup(unmarked_block)) self.assertEqual(marked_block, self.cleanup_service.last_confirmed_block) self.cleanup_service._request_block.assert_called_once()
def _test_mark_blocks_and_request_cleanup(self): marked_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) prev_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) tracked_blocks = [] self.cleanup_service.on_new_block_received(marked_block, prev_block) self.transaction_service.track_seen_short_ids(marked_block, []) for _ in range(self.block_confirmations_count - 1): tracked_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) self.transaction_service.track_seen_short_ids(tracked_block, []) tracked_blocks.append(tracked_block) unmarked_block = OntObjectHash( binary=helpers.generate_bytearray(ont_constants.ONT_HASH_LEN)) self.assertIsNone(self.cleanup_service.last_confirmed_block) self.cleanup_service.mark_blocks_and_request_cleanup( [marked_block, *tracked_blocks]) self.assertEqual(marked_block, self.cleanup_service.last_confirmed_block) self.assertTrue( self.cleanup_service.is_marked_for_cleanup(marked_block)) self.assertFalse( self.cleanup_service.is_marked_for_cleanup(unmarked_block)) self.assertEqual(marked_block, self.cleanup_service.last_confirmed_block) msg = self.node.send_to_node_messages.pop(-1) self.assertEqual((InventoryOntType.MSG_BLOCK.value, marked_block), msg.inv_type())
def setUp(self): opts = gateway_helpers.get_gateway_opts(8000, include_default_ont_args=True) if opts.use_extensions: helpers.set_extensions_parallelism() self.node = MockOntGatewayNode(opts) self.node.neutrality_service = MagicMock() self.connection = OntNodeConnection( MockSocketConnection(1, node=self.node, ip_address=LOCALHOST, port=123), self.node) gateway_helpers.add_blockchain_peer(self.node, self.connection) self.tx_hash = OntObjectHash(buf=helpers.generate_bytearray(32), length=ONT_HASH_LEN) self.block_hash = OntObjectHash(buf=helpers.generate_bytearray(32), length=ONT_HASH_LEN) self.sut = OntNodeConnectionProtocol(self.connection) while self.connection.outputbuf.length > 0: initial_bytes = self.connection.get_bytes_to_send() self.connection.advance_sent_bytes(len(initial_bytes))
def parse_message(self) -> int: if not self._parsed: self._parsed = True off = ont_constants.ONT_HDR_COMMON_OFF self._version, = struct.unpack_from("<I", self.buf, off) off += ont_constants.ONT_INT_LEN self._prev_block = OntObjectHash(self.buf, off, ont_constants.ONT_HASH_LEN) off += ont_constants.ONT_HASH_LEN self._txns_root = OntObjectHash(self.buf, off, ont_constants.ONT_HASH_LEN) off += ont_constants.ONT_HASH_LEN self._block_root = OntObjectHash(self.buf, off, ont_constants.ONT_HASH_LEN) off += ont_constants.ONT_HASH_LEN self._timestamp, self._height, self._consensus_data = struct.unpack_from( "<IIQ", self.buf, off) off += ont_constants.ONT_BLOCK_TIME_HEIGHT_CONS_DATA_LEN self._consensus_payload_length, size = ont_varint_to_int( self.buf, off) # pyre-fixme[6]: Expected `int` for 1st param but got `None`. off += self._consensus_payload_length + size self._next_bookkeeper, = struct.unpack_from("<20s", self.buf, off) off += ont_constants.ONT_BLOCK_NEXT_BOOKKEEPER_LEN self._header_offset = off self._header = self._memoryview[0:self._header_offset] self._bookkeepers_length, size = ont_varint_to_int(self.buf, off) off += size # pyre-fixme[6]: Expected `int` for 1st param but got `None`. for _ in range(self._bookkeepers_length): _, size = ont_varint_to_int(self.buf, off) off += size + ont_constants.ONT_BOOKKEEPER_LEN self._sig_data_length, size = ont_varint_to_int(self.buf, off) off += size # pyre-fixme[6]: Expected `int` for 1st param but got `None`. for _ in range(self._sig_data_length): sig_length, size = ont_varint_to_int(self.buf, off) off += size + sig_length self._header_with_program_offset = off self._header_with_program = self._memoryview[ ont_constants.ONT_HDR_COMMON_OFF:self. _header_with_program_offset] self._txn_count, = struct.unpack_from("<L", self.buf, off) off += ont_constants.ONT_INT_LEN self._tx_offset = off self._txn_header = self._memoryview[0:self._tx_offset] version = self._version assert isinstance(version, int) return version
def parse_bx_block_header(bx_block: memoryview, block_pieces: Deque[Union[bytearray, memoryview]]) -> \ BlockHeaderInfo: block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(bx_block) short_ids, short_ids_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer( bx_block, block_offsets.short_id_offset ) block_hash = OntObjectHash(binary=bx_block[block_offsets.block_begin_offset + 1: block_offsets.block_begin_offset + ont_constants.ONT_HASH_LEN + 1]) offset = block_offsets.block_begin_offset + ont_constants.ONT_HASH_LEN + 1 txn_count, = struct.unpack_from("<L", bx_block, offset) offset += ont_constants.ONT_INT_LEN payload_tail_len, = struct.unpack_from("<L", bx_block, offset) offset += ont_constants.ONT_INT_LEN + payload_tail_len owner_and_signature_len, = struct.unpack_from("<L", bx_block, offset) offset += ont_constants.ONT_INT_LEN + owner_and_signature_len consensus_payload_header_len, = struct.unpack_from("<L", bx_block, offset) offset += ont_constants.ONT_INT_LEN block_pieces.append(bx_block[offset: offset + consensus_payload_header_len]) offset += consensus_payload_header_len block_pieces.append(bx_block[offset: offset + ont_constants.ONT_CHAR_LEN]) offset += ont_constants.ONT_CHAR_LEN block_pieces.append(bx_block[offset: offset + ont_constants.ONT_INT_LEN]) offset += ont_constants.ONT_INT_LEN block_start_len_and_txn_header_total_len, = struct.unpack_from("<L", bx_block, offset) offset += ont_constants.ONT_INT_LEN block_pieces.append(bx_block[offset: offset + block_start_len_and_txn_header_total_len]) offset += block_start_len_and_txn_header_total_len return BlockHeaderInfo(block_offsets, short_ids, short_ids_len, block_hash, offset, txn_count)
def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() self._default_block_size = max(self._default_block_size, len(block_msg.buf)) tsk = self.compression_tasks.borrow_task() tsk.init(tpe.InputBytes(block_msg.buf), tx_service.proxy, enable_block_compression, min_tx_age_seconds) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compression_tasks.return_task(tsk) raise message_conversion_error.btc_block_compression_error( block_msg.block_hash(), e) bx_block = tsk.bx_block() block = memoryview(bx_block) compressed_size = len(block) original_size = len(block_msg.rawbytes()) block_hash = OntObjectHash( binary=convert.hex_to_bytes(tsk.block_hash().hex_string())) block_info = BlockInfo( block_hash, tsk.short_ids(), compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tsk.txn_count(), tsk.compressed_block_hash().hex_string(), tsk.prev_block_hash().hex_string(), original_size, compressed_size, 100 - float(compressed_size) / original_size * 100, tsk.ignored_short_ids()) self.compression_tasks.return_task(tsk) return block, block_info
def get_txid( buffer: Union[memoryview, bytearray]) -> Tuple[OntObjectHash, int]: if not isinstance(buffer, memoryview): buffer = memoryview(buffer) off = 1 txtype = buffer[off:off + 1] off += 1 + 40 # Deploy type if txtype == ont_common_constants.ONT_TX_DEPLOY_TYPE_INDICATOR_AS_BYTEARRAY: buffer_length, size = ont_varint_to_int(buffer, off) off += buffer_length + size off += 1 buffer_length, size = ont_varint_to_int(buffer, off) off += buffer_length + size buffer_length, size = ont_varint_to_int(buffer, off) off += buffer_length + size buffer_length, size = ont_varint_to_int(buffer, off) off += buffer_length + size buffer_length, size = ont_varint_to_int(buffer, off) off += buffer_length + size buffer_length, size = ont_varint_to_int(buffer, off) off += buffer_length + size # Invoke type elif txtype == ont_common_constants.ONT_TX_INVOKE_TYPE_INDICATOR_AS_BYTEARRAY: buffer_length, size = ont_varint_to_int(buffer, off) off += buffer_length + size _, size = ont_varint_to_int(buffer, off) off += size return OntObjectHash(buf=crypto.double_sha256(buffer[:off]), length=ont_common_constants.ONT_HASH_LEN), off
def txns(self) -> List[memoryview]: if self._txns is None: if self._tx_offset is None: self.parse_message() assert self._tx_offset is not None start = self._tx_offset self._txns = [] for _ in range(self.txn_count()): _, off = get_txid(self._memoryview[start:]) sig_length, size = ont_varint_to_int(self._memoryview[start:], off) off += size for _ in range(sig_length): invoke_length, size = ont_varint_to_int( self._memoryview[start:], off) off += size + invoke_length verify_length, size = ont_varint_to_int( self._memoryview[start:], off) off += size + verify_length # pyre-fixme[16]: Optional type has no attribute `append`. self._txns.append(self._memoryview[start:start + off]) start += off self._merkle_root = OntObjectHash(self._memoryview[start:], 0, ont_constants.ONT_HASH_LEN) self._merkle_root_memoryview = self._memoryview[start:start + ont_constants. ONT_HASH_LEN] txns = self._txns assert isinstance(txns, list) return txns
def parse_block_core_type( self, buf: Union[bytearray, memoryview]) -> List[memoryview]: if isinstance(buf, bytearray): buf = memoryview(buf) off = ont_constants.ONT_INT_LEN self._prev_block = OntObjectHash(buf, off, ont_constants.ONT_HASH_LEN) off += ont_constants.ONT_HASH_LEN * 3 + ont_constants.ONT_BLOCK_TIME_HEIGHT_CONS_DATA_LEN consensus_payload_length, size = ont_varint_to_int(buf, off) off += consensus_payload_length + size + ont_constants.ONT_BLOCK_NEXT_BOOKKEEPER_LEN self._header_offset = off hash_header = buf[:self._header_offset] self._hash_val = OntObjectHash(buf=crypto.double_sha256(hash_header), length=ont_constants.ONT_HASH_LEN) bookkeepers_length, size = ont_varint_to_int(buf, off) off += size for _ in range(bookkeepers_length): _, size = ont_varint_to_int(buf, off) off += size + ont_constants.ONT_BOOKKEEPER_LEN sig_data_length, size = ont_varint_to_int(buf, off) off += size for _ in range(sig_data_length): sig_length, size = ont_varint_to_int(buf, off) off += size + sig_length self._txn_count, = struct.unpack_from("<L", buf, off) off += ont_constants.ONT_INT_LEN self._tx_offset = off self._txn_header = buf[:off] txns = [] start = self._tx_offset txn_count = self._txn_count assert isinstance(txn_count, int) for _ in range(txn_count): _, off = get_txid(buf[start:]) sig_length, size = ont_varint_to_int(buf[start:], off) off += size for i in range(sig_length): invoke_length, size = ont_varint_to_int(buf[start:], off) off += size + invoke_length verify_length, size = ont_varint_to_int(buf[start:], off) off += size + verify_length txns.append(buf[start:start + off]) start += off return txns
def test_inv_and_get_data(self): seen_block_hash = OntObjectHash(buf=helpers.generate_bytearray(ONT_HASH_LEN), length=ONT_HASH_LEN) not_seen_block_hash = OntObjectHash(buf=helpers.generate_bytearray(ONT_HASH_LEN), length=ONT_HASH_LEN) self.node.blocks_seen.add(seen_block_hash) inv_msg_seen_block = InvOntMessage(123, InventoryOntType.MSG_BLOCK, [seen_block_hash, seen_block_hash]) self.sut.msg_inv(inv_msg_seen_block) get_data_msg_msg_seen_bytes = self.sut.connection.get_bytes_to_send() self.assertEqual(0, len(get_data_msg_msg_seen_bytes)) inv_msg = InvOntMessage(123, InventoryOntType.MSG_BLOCK, [seen_block_hash, not_seen_block_hash]) self.sut.msg_inv(inv_msg) get_data_msg_bytes = self.sut.connection.get_bytes_to_send() get_data_msg = GetDataOntMessage(buf=get_data_msg_bytes) data_msg_inv_type, data_msg_block = get_data_msg.inv_type() self.assertEqual(InventoryOntType.MSG_BLOCK.value, data_msg_inv_type) self.assertEqual(not_seen_block_hash, data_msg_block)
def msg_headers(self, msg: HeadersOntMessage): if not self.node.should_process_block_hash(): return header = msg.headers()[-1] raw_hash = crypto.double_sha256(header) reply = GetDataOntMessage(self.magic, InventoryOntType.MSG_BLOCK.value, OntObjectHash(buf=raw_hash, length=ont_constants.ONT_HASH_LEN)) self.connection.enqueue_msg(reply)
def test_ont_block_to_bloxroute_block_and_back_sids_found(self): prev_block_hash = bytearray(crypto.bitcoin_hash(b"123")) prev_block = OntObjectHash(prev_block_hash, length=SHA256_HASH_LEN) merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234")) merkle_root = OntObjectHash(merkle_root_hash, length=SHA256_HASH_LEN) txns_root_hash = bytearray(crypto.bitcoin_hash(b"345")) txns_root = OntObjectHash(txns_root_hash, length=SHA256_HASH_LEN) block_root_hash = bytearray(crypto.bitcoin_hash(b"456")) block_root = OntObjectHash(block_root_hash, length=SHA256_HASH_LEN) consensus_payload = bytes(b'111') next_bookkeeper = bytes(b'222') bookkeepers = [bytes(33)] * 5 sig_data = [bytes(2)] * 3 txns = [] timestamp = 1 height = 2 consensus_data = 3 ont_block = BlockOntMessage(self.magic, self.version, prev_block, txns_root, block_root, timestamp, height, consensus_data, consensus_payload, next_bookkeeper, bookkeepers, sig_data, txns, merkle_root) block_hash = ont_block.block_hash() bloxroute_block, block_info = self.ont_message_converter.block_to_bx_block( ont_block, self.tx_service, True, 0) self.assertEqual(0, block_info.txn_count) self.assertEqual(self.short_ids, list(block_info.short_ids)) self.assertEqual(ont_block.block_hash(), block_info.block_hash) parsed_ont_block, block_info, _, _ = self.ont_message_converter.bx_block_to_block( bloxroute_block, self.tx_service) self.assertIsNotNone(block_info) self.assertEqual(ont_block.rawbytes().tobytes(), parsed_ont_block.rawbytes().tobytes()) self.assertEqual(self.magic, parsed_ont_block.magic()) self.assertEqual( prev_block_hash, parsed_ont_block.prev_block_hash().get_little_endian()) self.assertEqual(ont_block.checksum(), parsed_ont_block.checksum()) self.assertEqual(block_hash, parsed_ont_block.block_hash()) self.assertEqual(block_hash.binary, block_info.block_hash.binary) self.assertEqual(timestamp, parsed_ont_block.timestamp())
class OntConnectionProtocolTest(AbstractTestCase): HASH = OntObjectHash(binary=crypto.double_sha256(b"123")) def setUp(self): opts = gateway_helpers.get_gateway_opts(8000, include_default_ont_args=True) if opts.use_extensions: helpers.set_extensions_parallelism() self.node = MockOntGatewayNode(opts) self.node.block_processing_service = MagicMock() self.connection = MagicMock() gateway_helpers.add_blockchain_peer(self.node, self.connection) self.connection.node = self.node self.connection.peer_ip = LOCALHOST self.connection.peer_port = 8001 self.connection.network_num = 2 self.connection.endpoint = IpEndpoint(self.connection.peer_ip, self.connection.peer_port) self.node.blockchain_peers.add( BlockchainPeerInfo(self.connection.peer_ip, self.connection.peer_port)) gateway_bdn_performance_stats_service.set_node(self.node) self.sut = OntBaseConnectionProtocol(self.connection) def test_msg_block_success(self): block_timestamp = int(time.time()) + 1 - \ self.node.opts.blockchain_ignore_block_interval_count * \ self.node.opts.blockchain_block_interval message = BlockOntMessage(0, 0, self.HASH, self.HASH, self.HASH, block_timestamp, 0, 0, bytes(10), bytes(20), [bytes(33)] * 5, [bytes(2)] * 3, [bytes(32)] * 5, self.HASH) self.sut.msg_block(message) self.node.block_processing_service.queue_block_for_processing.assert_called_once( ) def test_msg_block_too_old(self): block_timestamp = int(time.time()) - 1 - \ self.node.opts.blockchain_ignore_block_interval_count * \ self.node.opts.blockchain_block_interval message = BlockOntMessage(0, 0, self.HASH, self.HASH, self.HASH, block_timestamp, 0, 0, bytes(10), bytes(20), [bytes(33)] * 5, [bytes(2)] * 3, [bytes(32)] * 5, self.HASH) self.sut.msg_block(message) self.node.block_processing_service.queue_block_for_processing.assert_not_called( )
def clean_block_transactions( self, block_msg: BlockOntMessage, transaction_service: TransactionService) -> None: block_short_ids = [] block_unknown_tx_hashes = [] start_time = time.time() short_ids_count = 0 unknown_tx_hashes_count = 0 transactions_processed = 0 tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_before_cleanup = transaction_service.get_short_id_count( ) for tx in block_msg.txns(): tx_hash = OntObjectHash(buf=crypto.double_sha256(tx), length=ont_constants.ONT_HASH_LEN) short_ids = transaction_service.remove_transaction_by_tx_hash( tx_hash, force=True) if short_ids is None: unknown_tx_hashes_count += 1 block_unknown_tx_hashes.append(tx_hash) else: short_ids_count += len(short_ids) block_short_ids.extend(short_ids) transactions_processed += 1 block_hash = block_msg.block_hash() transaction_service.on_block_cleaned_up(block_hash) end_time = time.time() duration = end_time - start_time tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_after_cleanup = transaction_service.get_short_id_count() logger.debug( "Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} " "short ids. Took {:.3f}s.", block_hash, transactions_processed, unknown_tx_hashes_count, short_ids_count, duration) transaction_service.log_block_transaction_cleanup_stats( block_hash, block_msg.txn_count(), tx_hash_to_contents_len_before_cleanup, tx_hash_to_contents_len_after_cleanup, short_id_count_before_cleanup, short_id_count_after_cleanup) self._block_hash_marked_for_cleanup.discard(block_hash) self.node.post_block_cleanup_tasks( block_hash=block_hash, short_ids=block_short_ids, unknown_tx_hashes=block_unknown_tx_hashes)
def inv_type(self) -> Tuple[int, OntObjectHash]: if self._inv_type is None: off = ont_constants.ONT_HDR_COMMON_OFF self._inv_type, = struct.unpack_from("<B", self.buf, off) off += ont_constants.ONT_CHAR_LEN self._block = OntObjectHash(buf=self.buf, offset=off, length=ont_constants.ONT_HASH_LEN) inv_type, block = self._inv_type, self._block assert isinstance(inv_type, int) assert isinstance(block, OntObjectHash) return inv_type, block
def block_hash(self) -> OntObjectHash: if self._hash_val is None: if self._header_offset is None: self.parse_message() assert self._header_offset is not None header = self._memoryview[ont_constants.ONT_HDR_COMMON_OFF:self. _header_offset] raw_hash = crypto.double_sha256(header) self._hash_val = OntObjectHash(buf=raw_hash, length=ont_constants.ONT_HASH_LEN) hash_val = self._hash_val assert isinstance(hash_val, OntObjectHash) return hash_val
def bx_block_to_block( self, bx_block_msg: memoryview, tx_service: ExtensionTransactionService ) -> BlockDecompressionResult: decompress_start_datetime = datetime.utcnow() decompress_start_timestamp = time.time() tsk = self.decompression_tasks.borrow_task() tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.decompression_tasks.return_task(tsk) header_info = ont_normal_message_converter.parse_bx_block_header( bx_block_msg, deque()) raise message_conversion_error.btc_block_decompression_error( header_info.block_hash, e) total_tx_count = tsk.tx_count() unknown_tx_hashes = [ Sha256Hash(bytearray(unknown_tx_hash.binary())) for unknown_tx_hash in tsk.unknown_tx_hashes() ] unknown_tx_sids = tsk.unknown_tx_sids() block_hash = OntObjectHash( binary=convert.hex_to_bytes(tsk.block_hash().hex_string())) if tsk.success(): ont_block_msg = BlockOntMessage( buf=memoryview(tsk.block_message())) logger.debug( "Successfully parsed block broadcast message. {} transactions " "in block {}", total_tx_count, block_hash) else: ont_block_msg = None logger.debug( "Block recovery needed for {}. Missing {} sids, {} tx hashes. " "Total txs in block: {}", block_hash, len(unknown_tx_sids), len(unknown_tx_hashes), total_tx_count) block_info = get_block_info(bx_block_msg, block_hash, tsk.short_ids(), decompress_start_datetime, decompress_start_timestamp, total_tx_count, ont_block_msg) self.decompression_tasks.return_task(tsk) return BlockDecompressionResult(ont_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
class OntMessageFactoryTest(MessageFactoryTestCase): MAGIC = 12345 VERSION = 111 HASH = OntObjectHash(binary=crypto.double_sha256(b"123")) VERSION_ONT_MESSAGE = VersionOntMessage(MAGIC, VERSION, 20330, 20330, 20330, bytes(32), 123, 0, True, True, "v1.0.0".encode("utf-8")) def get_message_factory(self): return ont_message_factory def test_peek_message_success_all_types(self): self.get_message_preview_successfully(self.VERSION_ONT_MESSAGE, VersionOntMessage.MESSAGE_TYPE, 83) self.get_message_preview_successfully( VerAckOntMessage(self.MAGIC, True), VerAckOntMessage.MESSAGE_TYPE, 1) self.get_message_preview_successfully(PingOntMessage(self.MAGIC), PingOntMessage.MESSAGE_TYPE, 8) self.get_message_preview_successfully(PongOntMessage(self.MAGIC, 123), PongOntMessage.MESSAGE_TYPE, 8) self.get_message_preview_successfully(GetAddrOntMessage(self.MAGIC), GetAddrOntMessage.MESSAGE_TYPE, 0) self.get_message_preview_successfully( AddrOntMessage( self.MAGIC, [(int(time.time()), 123, "127.0.0.1", 20300, 20200, 1234)]), AddrOntMessage.MESSAGE_TYPE, 52) self.get_message_preview_successfully( OntConsensusMessage(self.MAGIC, self.VERSION, bytes(20)), OntConsensusMessage.MESSAGE_TYPE, 24) self.get_message_preview_successfully( InvOntMessage(self.MAGIC, InventoryOntType.MSG_TX, [self.HASH, self.HASH]), InvOntMessage.MESSAGE_TYPE, 69) self.get_message_preview_successfully( GetDataOntMessage(self.MAGIC, 1, self.HASH), GetDataOntMessage.MESSAGE_TYPE, 33) self.get_message_preview_successfully( GetHeadersOntMessage(self.MAGIC, 1, self.HASH, self.HASH), GetHeadersOntMessage.MESSAGE_TYPE, 65) self.get_message_preview_successfully( GetBlocksOntMessage(self.MAGIC, 1, self.HASH, self.HASH), GetBlocksOntMessage.MESSAGE_TYPE, 65) self.get_message_preview_successfully( TxOntMessage(self.MAGIC, self.VERSION, bytes(20)), TxOntMessage.MESSAGE_TYPE, 21) self.get_message_preview_successfully( BlockOntMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, self.HASH, 0, 0, 0, bytes(10), bytes(20), [bytes(33)] * 5, [bytes(2)] * 3, [bytes(32)] * 5, self.HASH), BlockOntMessage.MESSAGE_TYPE, 524) self.get_message_preview_successfully( HeadersOntMessage(self.MAGIC, [bytes(1)] * 2), HeadersOntMessage.MESSAGE_TYPE, 6) self.get_message_preview_successfully( NotFoundOntMessage(self.MAGIC, self.HASH), NotFoundOntMessage.MESSAGE_TYPE, 32) def test_peek_message_incomplete(self): is_full_message, command, payload_length = ont_message_factory.get_message_header_preview_from_input_buffer( create_input_buffer_with_bytes( self.VERSION_ONT_MESSAGE.rawbytes()[:-10])) self.assertFalse(is_full_message) self.assertEqual(b"version", command) self.assertEqual(83, payload_length) is_full_message, command, payload_length = ont_message_factory.get_message_header_preview_from_input_buffer( create_input_buffer_with_bytes( self.VERSION_ONT_MESSAGE.rawbytes()[:1])) self.assertFalse(is_full_message) self.assertIsNone(command) self.assertIsNone(payload_length) def test_parse_message_success_all_types(self): self.create_message_successfully(self.VERSION_ONT_MESSAGE, VersionOntMessage) self.create_message_successfully(VerAckOntMessage(self.MAGIC, False), VerAckOntMessage) self.create_message_successfully(PingOntMessage(self.MAGIC), PingOntMessage) self.create_message_successfully(PongOntMessage(self.MAGIC, 123), PongOntMessage) self.create_message_successfully(GetAddrOntMessage(self.MAGIC), GetAddrOntMessage) self.create_message_successfully( AddrOntMessage( self.MAGIC, [(int(time.time()), 123, "127.0.0.1", 20300, 20200, 1234)]), AddrOntMessage) self.create_message_successfully( OntConsensusMessage(self.MAGIC, self.VERSION, bytes(20)), OntConsensusMessage) self.create_message_successfully( InvOntMessage(self.MAGIC, InventoryOntType.MSG_TX, [self.HASH, self.HASH]), InvOntMessage) self.create_message_successfully( GetDataOntMessage(self.MAGIC, 1, self.HASH), GetDataOntMessage) self.create_message_successfully( GetHeadersOntMessage(self.MAGIC, 1, self.HASH, self.HASH), GetHeadersOntMessage) self.create_message_successfully( GetBlocksOntMessage(self.MAGIC, 1, self.HASH, self.HASH), GetBlocksOntMessage) self.create_message_successfully( TxOntMessage(self.MAGIC, self.VERSION, bytes(20)), TxOntMessage) self.create_message_successfully( BlockOntMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, self.HASH, 0, 0, 0, bytes(10), bytes(20), [bytes(33)] * 5, [bytes(2)] * 3, [bytes(32)] * 5, self.HASH), BlockOntMessage) self.create_message_successfully( HeadersOntMessage(self.MAGIC, [bytes(1)] * 2), HeadersOntMessage) self.create_message_successfully( NotFoundOntMessage(self.MAGIC, self.HASH), NotFoundOntMessage) def test_parse_message_incomplete(self): with self.assertRaises(PayloadLenError): ont_message_factory.create_message_from_buffer( PingOntMessage(self.MAGIC).rawbytes()[:-1]) ping_message = PingOntMessage(self.MAGIC) for i in range(ONT_HEADER_MINUS_CHECKSUM, ONT_HDR_COMMON_OFF): ping_message.buf[i] = 0 with self.assertRaises(ChecksumError): ont_message_factory.create_message_from_buffer( ping_message.rawbytes())
class OntRelayConnectionTest(AbstractTestCase): ONT_HASH = OntObjectHash(crypto.double_sha256(b"123"), length=SHA256_HASH_LEN) magic = 12345 version = 111 TEST_NETWORK_NUM = 12345 def setUp(self): opts = gateway_helpers.get_gateway_opts(8000, include_default_ont_args=True) if opts.use_extensions: helpers.set_extensions_parallelism( opts.thread_pool_parallelism_degree) node_ssl_service = MockNodeSSLService(OntGatewayNode.NODE_TYPE, MagicMock()) self.gateway_node = OntGatewayNode(opts, node_ssl_service) self.gateway_node.opts.has_fully_updated_tx_service = True self.gateway_node.opts.is_consensus = False self.sut = OntRelayConnection( MockSocketConnection(1, node=self.gateway_node, ip_address=LOCALHOST, port=8001), self.gateway_node) self.node_conn = MockConnection( MockSocketConnection(1, self.gateway_node, ip_address=LOCALHOST, port=8002), self.gateway_node) self.gateway_node.connection_pool.add(1, LOCALHOST, 8002, self.node_conn) gateway_helpers.add_blockchain_peer(self.gateway_node, self.node_conn) self.node_conn_2 = MockConnection( MockSocketConnection(1, self.gateway_node, ip_address=LOCALHOST, port=8003), self.gateway_node) self.gateway_node.connection_pool.add(1, LOCALHOST, 8003, self.node_conn_2) gateway_helpers.add_blockchain_peer(self.gateway_node, self.node_conn_2) self.blockchain_connections = [self.node_conn, self.node_conn_2] self.gateway_node_sut = OntNodeConnectionProtocol( cast(OntNodeConnection, self.node_conn)) self.gateway_node.message_converter = converter_factory.create_ont_message_converter( 12345, self.gateway_node.opts) self.gateway_node.has_active_blockchain_peer = MagicMock( return_value=True) self.gateway_node.broadcast = MagicMock() self.node_conn.enqueue_msg = MagicMock() self.node_conn_2.enqueue_msg = MagicMock() self.sut.enqueue_msg = MagicMock() gateway_transaction_stats_service.set_node(self.gateway_node) def ont_transactions(self, block: BlockOntMessage = None) -> List[memoryview]: txs = block.txns() return [TxOntMessage(self.magic, self.version, tx[1:]) for tx in txs] def ont_block(self) -> BlockOntMessage: return self._get_sample_block() def bx_block(self, ont_block=None): if ont_block is None: ont_block = self.ont_block() return bytes( self.gateway_node.message_converter.block_to_bx_block( ont_block, self.gateway_node.get_tx_service(), True, self.gateway_node.network.min_tx_age_seconds)[0]) def bx_transactions(self, transactions=None, assign_short_ids=False): if transactions is None: block = self.ont_block() transactions = self.ont_transactions(block) bx_transactions = [] for i, transaction in enumerate(transactions): transaction = self.gateway_node.message_converter.tx_to_bx_txs( transaction, self.TEST_NETWORK_NUM)[0][0] if assign_short_ids: transaction._short_id = i + 1 # 0 is null SID bx_transactions.append(transaction) return bx_transactions @skip("Encryption block in Ontology") def test_msg_broadcast_wait_for_key(self): ont_block = self.ont_block() bx_block = self.bx_block(ont_block) key, ciphertext = symmetric_encrypt(bx_block) block_hash = crypto.double_sha256(ciphertext) broadcast_message = BroadcastMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "", BroadcastMessageType.BLOCK, True, ciphertext) self.sut.msg_broadcast(broadcast_message) # handle duplicate messages self.sut.msg_broadcast(broadcast_message) self.sut.msg_broadcast(broadcast_message) self.assertEqual(3, len(self.gateway_node.broadcast.call_args_list)) for call, conn_type in self.gateway_node.broadcast.call_args_list: msg, conn = call self.assertTrue(isinstance(msg, BlockReceivedMessage)) self.gateway_node.broadcast.reset_mock() self.assertEqual(1, len(self.gateway_node.in_progress_blocks)) key_message = KeyMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "", key) self.sut.msg_key(key_message) self._assert_block_sent(ont_block) def test_msg_broadcast_duplicate_block_with_different_short_id(self): # test scenario when first received block is compressed with unknown short ids, # but second received block is compressed with known short ids ont_block = self.ont_block() block_hash = ont_block.block_hash() transactions = self.bx_transactions() unknown_sid_transaction_service = ExtensionTransactionService( MockNode(gateway_helpers.get_gateway_opts(8999)), 0) for i, transaction in enumerate(transactions): transaction_key = unknown_sid_transaction_service.get_transaction_key( transaction.tx_hash()) unknown_sid_transaction_service.assign_short_id_by_key( transaction_key, i) unknown_sid_transaction_service.set_transaction_contents_by_key( transaction_key, transaction.tx_val()) unknown_short_id_block = bytes( self.gateway_node.message_converter.block_to_bx_block( ont_block, unknown_sid_transaction_service, True, self.gateway_node.network.min_tx_age_seconds)[0]) unknown_key, unknown_cipher = symmetric_encrypt(unknown_short_id_block) unknown_block_hash = crypto.double_sha256(unknown_cipher) unknown_message = BroadcastMessage(Sha256Hash(unknown_block_hash), self.TEST_NETWORK_NUM, "", BroadcastMessageType.BLOCK, False, bytearray(unknown_short_id_block)) unknown_key_message = KeyMessage(Sha256Hash(unknown_block_hash), self.TEST_NETWORK_NUM, "", unknown_key) local_transaction_service = self.gateway_node.get_tx_service() for i, transaction in enumerate(transactions): transaction_key = local_transaction_service.get_transaction_key( transaction.tx_hash()) local_transaction_service.assign_short_id_by_key( transaction_key, i + 20) local_transaction_service.set_transaction_contents_by_key( transaction_key, transaction.tx_val()) known_short_id_block = bytes( self.gateway_node.message_converter.block_to_bx_block( ont_block, local_transaction_service, True, self.gateway_node.network.min_tx_age_seconds)[0]) known_key, known_cipher = symmetric_encrypt(known_short_id_block) known_block_hash = crypto.double_sha256(known_cipher) known_message = BroadcastMessage(Sha256Hash(known_block_hash), self.TEST_NETWORK_NUM, "", BroadcastMessageType.BLOCK, False, bytearray(known_short_id_block)) known_key_message = KeyMessage(Sha256Hash(known_block_hash), self.TEST_NETWORK_NUM, "", known_key) self.sut.msg_broadcast(unknown_message) self.sut.msg_key(unknown_key_message) block_queuing_service = self.gateway_node.block_queuing_service_manager.get_block_queuing_service( self.node_conn) self.assertEqual(1, len(block_queuing_service)) self.assertEqual( True, block_queuing_service._blocks_waiting_for_recovery[block_hash]) self.assertEqual( 1, len(self.gateway_node.block_recovery_service. _block_hash_to_bx_block_hashes)) self.assertNotIn(block_hash, self.gateway_node.blocks_seen.contents) self.sut.msg_broadcast(known_message) self.sut.msg_key(known_key_message) self.gateway_node_sut.msg_get_data( GetDataOntMessage(self.magic, InventoryOntType.MSG_BLOCK.value, block_hash)) self.gateway_node.broadcast.assert_called() self.assertEqual(0, len(block_queuing_service)) self.assertEqual( 0, len(self.gateway_node.block_recovery_service. _block_hash_to_bx_block_hashes)) self.assertIn(block_hash, self.gateway_node.blocks_seen.contents) @skip("Encryption block in Ontology") def test_msg_key_wait_for_broadcast(self): ont_block = self.ont_block() bx_block = self.bx_block(ont_block) key, ciphertext = symmetric_encrypt(bx_block) block_hash = crypto.double_sha256(ciphertext) self.gateway_node.broadcast.assert_not_called() key_message = KeyMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "", key) self.sut.msg_key(key_message) # handle duplicate broadcasts self.sut.msg_key(key_message) self.sut.msg_key(key_message) self.assertEqual(1, len(self.gateway_node.in_progress_blocks)) broadcast_message = BroadcastMessage(Sha256Hash(block_hash), self.TEST_NETWORK_NUM, "", BroadcastMessageType.BLOCK, True, ciphertext) self.sut.msg_broadcast(broadcast_message) self._assert_block_sent(ont_block) def test_msg_tx(self): transactions = self.bx_transactions(assign_short_ids=True) for transaction in transactions: self.sut.msg_tx(transaction) for i, transaction in enumerate(transactions): transaction_hash = transaction.tx_hash() transaction_key = self.gateway_node.get_tx_service( ).get_transaction_key(transaction_hash) self.assertTrue(self.gateway_node.get_tx_service(). has_transaction_contents_by_key(transaction_key)) self.assertTrue(self.gateway_node.get_tx_service(). has_transaction_short_id_by_key(transaction_key)) self.assertEqual( i + 1, self.gateway_node.get_tx_service().get_short_id_by_key( transaction_key)) stored_hash, stored_content, _ = self.gateway_node.get_tx_service( ).get_transaction(i + 1) self.assertEqual(transaction_hash, stored_hash) self.assertEqual(transaction.tx_val(), stored_content) self.assertEqual(len(transactions), self.gateway_node.broadcast.call_count) def test_msg_tx_additional_sid(self): transactions = self.bx_transactions(assign_short_ids=True) for transaction in transactions: self.sut.msg_tx(transaction) # rebroadcast transactions with more sids for i, transaction in enumerate(transactions): transaction._short_id += 20 self.sut.msg_tx(transaction) for i, transaction in enumerate(transactions): transaction_hash = transaction.tx_hash() self.assertTrue( self.gateway_node.get_tx_service().has_transaction_contents( transaction_hash)) self.assertTrue( self.gateway_node.get_tx_service().has_transaction_short_id( transaction_hash)) stored_hash, stored_content, _ = self.gateway_node.get_tx_service( ).get_transaction(i + 1) self.assertEqual(transaction_hash, stored_hash) self.assertEqual(transaction.tx_val(), stored_content) stored_hash2, stored_content2, _ = self.gateway_node.get_tx_service( ).get_transaction(i + 21) self.assertEqual(transaction_hash, stored_hash2) self.assertEqual(transaction.tx_val(), stored_content2) # only 10 times even with rebroadcast SID self.assertEqual(len(transactions), self.gateway_node.broadcast.call_count) def test_msg_tx_duplicate_ignore(self): transactions = self.bx_transactions(assign_short_ids=True) for transaction in transactions: self.sut.msg_tx(transaction) for transaction in transactions: self.sut.msg_tx(transaction) self.assertEqual(len(transactions), self.gateway_node.broadcast.call_count) @skip("Encrpytion in Ontology is not developed yet") def test_get_txs_block_recovery_encrypted(self): block: BlockOntMessage = self.ont_block() transactions: List[TxOntMessage] = self.ont_transactions(block) # assign short ids that the local connection won't know about until it gets the txs message remote_transaction_service = ExtensionTransactionService( MockNode(gateway_helpers.get_gateway_opts(8999)), 0) short_id_mapping = {} for i, transaction in enumerate(transactions): tx_hash = transaction.tx_hash() transaction_key = remote_transaction_service.get_transaction_key( tx_hash) remote_transaction_service.assign_short_id_by_key( transaction_key, i + 1) remote_transaction_service.set_transaction_contents_by_key( transaction_key, transaction.rawbytes()) short_id_mapping[tx_hash] = TransactionInfo( tx_hash, transaction.rawbytes(), i + 1) bx_block = bytes( self.gateway_node.message_converter.block_to_bx_block( block, remote_transaction_service, True, self.gateway_node.network.min_tx_age_seconds)[0]) self.gateway_node.block_recovery_service.add_block = \ MagicMock(wraps=self.gateway_node.block_recovery_service.add_block) self.gateway_node.broadcast = MagicMock() key, ciphertext = symmetric_encrypt(bx_block) block_hash = crypto.double_sha256(ciphertext) key_message = KeyMessage(Sha256Hash(block_hash), DEFAULT_NETWORK_NUM, "", key) broadcast_message = BroadcastMessage(Sha256Hash(block_hash), DEFAULT_NETWORK_NUM, "", BroadcastMessageType.BLOCK, True, ciphertext) self.sut.msg_broadcast(broadcast_message) self.gateway_node.broadcast.reset_mock() self.sut.msg_key(key_message) self.gateway_node.block_recovery_service.add_block.assert_called_once() self.assertEqual(2, self.gateway_node.broadcast.call_count) recovery_broadcast = self.gateway_node.broadcast.call_args_list[0] ((gettxs_message, ), recovery_kwargs) = recovery_broadcast self.assertIsInstance(gettxs_message, GetTxsMessage) self.assertIn(ConnectionType.RELAY_TRANSACTION, recovery_kwargs["connection_types"]) key_broadcast = self.gateway_node.broadcast.call_args_list[1] ((key_message, _conn), recovery_kwargs) = key_broadcast self.assertIsInstance(key_message, KeyMessage) self.assertIn(ConnectionType.GATEWAY, recovery_kwargs["connection_types"]) txs = [tx for tx in short_id_mapping.values()] txs_message = TxsMessage(txs=txs) self.sut.msg_txs(txs_message) self._assert_block_sent(block) def test_get_txs_block_recovery(self): block: BlockOntMessage = self.ont_block() transactions: List[TxOntMessage] = self.ont_transactions(block) # assign short ids that the local connection won't know about until it gets the txs message remote_transaction_service = ExtensionTransactionService( MockNode(gateway_helpers.get_gateway_opts(8999)), 0) short_id_mapping = {} for i, transaction in enumerate(transactions): tx_hash = transaction.tx_hash() remote_transaction_service.assign_short_id(tx_hash, i + 1) remote_transaction_service.set_transaction_contents( tx_hash, transaction.rawbytes()) short_id_mapping[tx_hash] = TransactionInfo( tx_hash, transaction.rawbytes(), i + 1) bx_block = bytes( self.gateway_node.message_converter.block_to_bx_block( block, remote_transaction_service, True, self.gateway_node.network.min_tx_age_seconds)[0]) self.gateway_node.block_recovery_service.add_block = \ MagicMock(wraps=self.gateway_node.block_recovery_service.add_block) self.gateway_node.broadcast = MagicMock() broadcast_message = BroadcastMessage(block.block_hash(), DEFAULT_NETWORK_NUM, "", BroadcastMessageType.BLOCK, False, bytearray(bx_block)) self.sut.msg_broadcast(broadcast_message) self.gateway_node.block_recovery_service.add_block.assert_called_once() self.assertEqual(1, self.gateway_node.broadcast.call_count) recovery_broadcast = self.gateway_node.broadcast.call_args_list[0] ((gettxs_message, ), recovery_kwargs) = recovery_broadcast self.assertIsInstance(gettxs_message, GetTxsMessage) self.assertIn(ConnectionType.RELAY_TRANSACTION, recovery_kwargs["connection_types"]) txs = [tx for tx in short_id_mapping.values()] txs_message = TxsMessage(txs=txs) self.sut.msg_txs(txs_message) self._assert_block_sent(block) def test_get_txs_multiple_sid_assignments(self): block = self.ont_block() transactions = self.ont_transactions(block) # assign short ids that the local connection won't know about until it gets the txs message remote_transaction_service1 = ExtensionTransactionService( MockNode(gateway_helpers.get_gateway_opts(8999)), 0) short_id_mapping1 = {} for i, transaction in enumerate(transactions): remote_transaction_service1.assign_short_id( transaction.tx_hash(), i + 1) remote_transaction_service1.set_transaction_contents( transaction.tx_hash(), transaction.tx()) short_id_mapping1[transaction.tx_hash()] = TransactionInfo( transaction.tx_hash(), transaction.tx(), i + 1) txs_message_1 = TxsMessage([tx for tx in short_id_mapping1.values()]) self.sut.msg_txs(txs_message_1) for transaction_hash, tx_info in short_id_mapping1.items(): transaction_key = self.gateway_node.get_tx_service( ).get_transaction_key(transaction_hash) self.assertEqual( tx_info.short_id, self.gateway_node.get_tx_service().get_short_id_by_key( transaction_key)) stored_hash, stored_content, _ = self.gateway_node.get_tx_service( ).get_transaction(tx_info.short_id) self.assertEqual(transaction_hash, stored_hash) self.assertEqual(tx_info.contents, stored_content) remote_transaction_service2 = ExtensionTransactionService( MockNode(gateway_helpers.get_gateway_opts(8999)), 0) short_id_mapping2 = {} for i, transaction in enumerate(transactions): remote_transaction_service2.assign_short_id( transaction.tx_hash(), i + 101) remote_transaction_service2.set_transaction_contents( transaction.tx_hash(), transaction.tx()) short_id_mapping2[transaction.tx_hash()] = TransactionInfo( transaction.tx_hash(), transaction.tx(), i + 101) txs_message_2 = TxsMessage([tx for tx in short_id_mapping2.values()]) self.sut.msg_txs(txs_message_2) for transaction_hash, tx_info in short_id_mapping2.items(): stored_hash, stored_content, _ = self.gateway_node.get_tx_service( ).get_transaction(tx_info.short_id) self.assertEqual(transaction_hash, stored_hash) self.assertEqual(tx_info.contents, stored_content) def _assert_block_sent(self, ont_block): for node_conn in self.blockchain_connections: node_conn.enqueue_msg.assert_called() calls = node_conn.enqueue_msg.call_args_list sent_to_blockchain_calls = [] for call in calls: sent_to_blockchain_calls.append(call) self.assertEqual(1, len(sent_to_blockchain_calls)) ((sent_inv_msg, ), _) = sent_to_blockchain_calls[0] self.assertIsInstance(sent_inv_msg, InvOntMessage) sent_inv_msg = cast(InvOntMessage, sent_inv_msg) sent_inv_type, sent_inv_hash = sent_inv_msg.inv_type() self.assertEqual(InventoryOntType.MSG_BLOCK.value, sent_inv_type) self.assertEqual(ont_block.block_hash(), sent_inv_hash[0]) def _get_sample_block(self): root_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) with open(os.path.join(root_dir, "samples/ont_sample_block.txt")) as sample_file: ont_block = sample_file.read().strip("\n") buf = bytearray(convert.hex_to_bytes(ont_block)) parsed_block = BlockOntMessage(buf=buf) self.magic = parsed_block.magic() self.version = parsed_block.version() return parsed_block
def hash_stop(self) -> OntObjectHash: return OntObjectHash( buf=self.buf, offset=ont_constants.ONT_HDR_COMMON_OFF + self.payload_len() - ont_constants.ONT_HASH_LEN, length=ont_constants.ONT_HASH_LEN )