def __init__(self, data_dir, validator_id):
"""Initialize the consensus state store
Args:
data_dir (str): The directory where underlying database file will
be stored
validator_id (str): A unique ID for the validator for which the
consensus state store is being created
Returns:
None
"""
with ConsensusStateStore._lock:
# Create an underlying LMDB database file for the validator if
# there already isn't one. We will create the LMDB with the 'c'
# flag so that it will open if already exists.
self._store_db = ConsensusStateStore._store_dbs.get(validator_id)
if self._store_db is None:
db_file_name = \
os.path.join(
data_dir,
'poet_consensus_state-{}.lmdb'.format(
validator_id[:8]))
LOGGER.debug('Create consensus store: %s', db_file_name)
self._store_db = LMDBNoLockDatabase(db_file_name, 'c')
ConsensusStateStore._store_dbs[validator_id] = self._store_db
def __init__(self, network_endpoint, component_endpoint, peer_list):
db_filename = os.path.join(os.path.expanduser('~'), 'merkle.lmdb')
LOGGER.debug('database file is %s', db_filename)
lmdb = LMDBNoLockDatabase(db_filename, 'n')
context_manager = ContextManager(lmdb)
block_db_filename = os.path.join(os.path.expanduser('~'), 'block.lmdb')
LOGGER.debug('block store file is %s', block_db_filename)
block_store = LMDBNoLockDatabase(block_db_filename, 'n')
self._service = ValidatorService(component_endpoint)
# setup network
dispatcher = Dispatcher()
faux_network = FauxNetwork(dispatcher=dispatcher)
identity = "{}-{}".format(socket.gethostname(),
os.getpid()).encode('ascii')
self._network = Network(identity,
network_endpoint,
peer_list,
dispatcher=dispatcher)
# Create and configure journal
executor = TransactionExecutor(self._service, context_manager)
self._journal = Journal(
consensus=dev_mode_consensus,
block_store={}, # FIXME - block_store=block_store
# -- need to serialize blocks to dicts
send_message=faux_network.send_message,
transaction_executor=executor,
squash_handler=context_manager.get_squash_handler(),
first_state_root=context_manager.get_first_root())
dispatcher.on_batch_received = \
self._journal.on_batch_received
dispatcher.on_block_received = \
self._journal.on_block_received
dispatcher.on_block_request = \
self._journal.on_block_request
self._service.add_handler('default', DefaultHandler())
self._service.add_handler('state/getrequest',
state.GetHandler(context_manager))
self._service.add_handler('state/setrequest',
state.SetHandler(context_manager))
self._service.add_handler('tp/register',
ProcessorRegisterHandler(self._service))
self._service.add_handler('system/load',
SystemLoadHandler(faux_network))
def __init__(self, url):
db_filename = os.path.join(os.path.expanduser('~'), 'merkle.lmdb')
LOGGER.debug('database file is %s', db_filename)
lmdb = LMDBNoLockDatabase(db_filename, 'n')
context_manager = ContextManager(lmdb)
service = ValidatorService()
executor = TransactionExecutor(service, context_manager)
journal = FauxJournal(executor)
dispatcher = Dispatcher()
dispatcher.on_batch_received = journal.get_on_batch_received_handler()
network = FauxNetwork(dispatcher=dispatcher)
service.add_handler('default', DefaultHandler())
service.add_handler('state/getrequest',
state.GetHandler(context_manager))
service.add_handler('state/setrequest',
state.SetHandler(context_manager))
service.add_handler('tp/register', ProcessorRegisterHandler(service))
service.add_handler('tp/response', ResponseHandler(service))
service.add_handler('system/load', SystemLoadHandler(network))
self._server = grpc.server(ThreadPoolExecutor(max_workers=10))
validator_pb2.add_ValidatorServicer_to_server(service, self._server)
port_spec = url
LOGGER.debug('listening on port %s', port_spec)
self._server.add_insecure_port(port_spec)
def get_databases(bind_network, data_dir=None):
# Get the global state database to operate on
if data_dir is not None:
global_state_db_filename = os.path.join(
data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('verifying state in %s', global_state_db_filename)
global_state_db = LMDBNoLockDatabase(global_state_db_filename, 'c')
else:
global_state_db = DictDatabase()
# Get the blockstore
block_db_filename = os.path.join(data_dir,
'block-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('block store file is %s', block_db_filename)
block_db = IndexedDatabase(block_db_filename,
BlockStore.serialize_block,
BlockStore.deserialize_block,
flag='c',
indexes=BlockStore.create_index_configuration())
blockstore = BlockStore(block_db)
return global_state_db, blockstore
def __init__(self,
bind_network,
bind_component,
bind_consensus,
endpoint,
peering,
seeds_list,
peer_list,
data_dir,
config_dir,
identity_signer,
scheduler_type,
permissions,
minimum_peer_connectivity,
maximum_peer_connectivity,
state_pruning_block_depth,
network_public_key=None,
network_private_key=None,
roles=None):
"""Constructs a validator instance.
Args:
bind_network (str): the network endpoint
bind_component (str): the component endpoint
endpoint (str): the zmq-style URI of this validator's
publically reachable endpoint
peering (str): The type of peering approach. Either 'static'
or 'dynamic'. In 'static' mode, no attempted topology
buildout occurs -- the validator only attempts to initiate
peering connections with endpoints specified in the
peer_list. In 'dynamic' mode, the validator will first
attempt to initiate peering connections with endpoints
specified in the peer_list and then attempt to do a
topology buildout starting with peer lists obtained from
endpoints in the seeds_list. In either mode, the validator
will accept incoming peer requests up to max_peers.
seeds_list (list of str): a list of addresses to connect
to in order to perform the initial topology buildout
peer_list (list of str): a list of peer addresses
data_dir (str): path to the data directory
config_dir (str): path to the config directory
identity_signer (str): cryptographic signer the validator uses for
signing
"""
# -- Setup Global State Database and Factory -- #
global_state_db_filename = os.path.join(
data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('global state database file is %s',
global_state_db_filename)
global_state_db = NativeLmdbDatabase(
global_state_db_filename,
indexes=MerkleDatabase.create_index_configuration())
state_view_factory = StateViewFactory(global_state_db)
# -- Setup Receipt Store -- #
receipt_db_filename = os.path.join(
data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('txn receipt store file is %s', receipt_db_filename)
receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c')
receipt_store = TransactionReceiptStore(receipt_db)
# -- Setup Block Store -- #
block_db_filename = os.path.join(
data_dir, 'block-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('block store file is %s', block_db_filename)
block_db = IndexedDatabase(
block_db_filename,
BlockStore.serialize_block,
BlockStore.deserialize_block,
flag='c',
indexes=BlockStore.create_index_configuration())
block_store = BlockStore(block_db)
# The cache keep time for the journal's block cache must be greater
# than the cache keep time used by the completer.
base_keep_time = 1200
block_cache = BlockCache(block_store,
keep_time=int(base_keep_time * 9 / 8),
purge_frequency=30)
# -- Setup Thread Pools -- #
component_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=10, name='Component')
network_thread_pool = InstrumentedThreadPoolExecutor(max_workers=10,
name='Network')
client_thread_pool = InstrumentedThreadPoolExecutor(max_workers=5,
name='Client')
sig_pool = InstrumentedThreadPoolExecutor(max_workers=3,
name='Signature')
# -- Setup Dispatchers -- #
component_dispatcher = Dispatcher()
network_dispatcher = Dispatcher()
# -- Setup Services -- #
component_service = Interconnect(bind_component,
component_dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True,
max_future_callback_workers=10)
zmq_identity = hashlib.sha512(
time.time().hex().encode()).hexdigest()[:23]
secure = False
if network_public_key is not None and network_private_key is not None:
secure = True
network_service = Interconnect(bind_network,
dispatcher=network_dispatcher,
zmq_identity=zmq_identity,
secured=secure,
server_public_key=network_public_key,
server_private_key=network_private_key,
heartbeat=True,
public_endpoint=endpoint,
connection_timeout=120,
max_incoming_connections=100,
max_future_callback_workers=10,
authorize=True,
signer=identity_signer,
roles=roles)
# -- Setup Transaction Execution Platform -- #
context_manager = ContextManager(global_state_db)
batch_tracker = BatchTracker(block_store)
settings_cache = SettingsCache(
SettingsViewFactory(state_view_factory), )
transaction_executor = TransactionExecutor(
service=component_service,
context_manager=context_manager,
settings_view_factory=SettingsViewFactory(state_view_factory),
scheduler_type=scheduler_type,
invalid_observers=[batch_tracker])
component_service.set_check_connections(
transaction_executor.check_connections)
event_broadcaster = EventBroadcaster(component_service, block_store,
receipt_store)
# -- Setup P2P Networking -- #
gossip = Gossip(network_service,
settings_cache,
lambda: block_store.chain_head,
block_store.chain_head_state_root,
endpoint=endpoint,
peering_mode=peering,
initial_seed_endpoints=seeds_list,
initial_peer_endpoints=peer_list,
minimum_peer_connectivity=minimum_peer_connectivity,
maximum_peer_connectivity=maximum_peer_connectivity,
topology_check_frequency=1)
completer = Completer(block_store,
gossip,
cache_keep_time=base_keep_time,
cache_purge_frequency=30,
requested_keep_time=300)
block_sender = BroadcastBlockSender(completer, gossip)
batch_sender = BroadcastBatchSender(completer, gossip)
chain_id_manager = ChainIdManager(data_dir)
identity_view_factory = IdentityViewFactory(
StateViewFactory(global_state_db))
id_cache = IdentityCache(identity_view_factory)
# -- Setup Permissioning -- #
permission_verifier = PermissionVerifier(
permissions, block_store.chain_head_state_root, id_cache)
identity_observer = IdentityObserver(to_update=id_cache.invalidate,
forked=id_cache.forked)
settings_observer = SettingsObserver(
to_update=settings_cache.invalidate, forked=settings_cache.forked)
# -- Consensus Engine -- #
consensus_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=3, name='Consensus')
consensus_dispatcher = Dispatcher()
consensus_service = Interconnect(bind_consensus,
consensus_dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True,
max_future_callback_workers=10)
consensus_notifier = ConsensusNotifier(consensus_service)
# -- Setup Journal -- #
batch_injector_factory = DefaultBatchInjectorFactory(
block_cache=block_cache,
state_view_factory=state_view_factory,
signer=identity_signer)
block_publisher = BlockPublisher(
transaction_executor=transaction_executor,
block_cache=block_cache,
state_view_factory=state_view_factory,
settings_cache=settings_cache,
block_sender=block_sender,
batch_sender=batch_sender,
chain_head=block_store.chain_head,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
permission_verifier=permission_verifier,
check_publish_block_frequency=0.1,
batch_observers=[batch_tracker],
batch_injector_factory=batch_injector_factory)
block_publisher_batch_sender = block_publisher.batch_sender()
block_validator = BlockValidator(
block_cache=block_cache,
state_view_factory=state_view_factory,
transaction_executor=transaction_executor,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
permission_verifier=permission_verifier)
chain_controller = ChainController(
block_store=block_store,
block_cache=block_cache,
block_validator=block_validator,
state_database=global_state_db,
chain_head_lock=block_publisher.chain_head_lock,
state_pruning_block_depth=state_pruning_block_depth,
data_dir=data_dir,
observers=[
event_broadcaster, receipt_store, batch_tracker,
identity_observer, settings_observer
])
genesis_controller = GenesisController(
context_manager=context_manager,
transaction_executor=transaction_executor,
completer=completer,
block_store=block_store,
state_view_factory=state_view_factory,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
chain_id_manager=chain_id_manager,
batch_sender=batch_sender)
responder = Responder(completer)
completer.set_on_batch_received(block_publisher_batch_sender.send)
completer.set_on_block_received(chain_controller.queue_block)
completer.set_chain_has_block(chain_controller.has_block)
# -- Register Message Handler -- #
network_handlers.add(network_dispatcher, network_service, gossip,
completer, responder, network_thread_pool,
sig_pool, chain_controller.has_block,
block_publisher.has_batch, permission_verifier,
block_publisher, consensus_notifier)
component_handlers.add(component_dispatcher, gossip, context_manager,
transaction_executor, completer, block_store,
batch_tracker, global_state_db,
self.get_chain_head_state_root_hash,
receipt_store, event_broadcaster,
permission_verifier, component_thread_pool,
client_thread_pool, sig_pool, block_publisher)
# -- Store Object References -- #
self._component_dispatcher = component_dispatcher
self._component_service = component_service
self._component_thread_pool = component_thread_pool
self._network_dispatcher = network_dispatcher
self._network_service = network_service
self._network_thread_pool = network_thread_pool
consensus_proxy = ConsensusProxy(
block_cache=block_cache,
chain_controller=chain_controller,
block_publisher=block_publisher,
gossip=gossip,
identity_signer=identity_signer,
settings_view_factory=SettingsViewFactory(state_view_factory),
state_view_factory=state_view_factory)
consensus_handlers.add(consensus_dispatcher, consensus_thread_pool,
consensus_proxy)
self._consensus_dispatcher = consensus_dispatcher
self._consensus_service = consensus_service
self._consensus_thread_pool = consensus_thread_pool
self._client_thread_pool = client_thread_pool
self._sig_pool = sig_pool
self._context_manager = context_manager
self._transaction_executor = transaction_executor
self._genesis_controller = genesis_controller
self._gossip = gossip
self._block_publisher = block_publisher
self._chain_controller = chain_controller
self._block_validator = block_validator
def __init__(self, network_endpoint, component_endpoint, peer_list,
data_dir, identity_signing_key):
"""Constructs a validator instance.
Args:
network_endpoint (str): the network endpoint
component_endpoint (str): the component endpoint
peer_list (list of str): a list of peer addresses
data_dir (str): path to the data directory
key_dir (str): path to the key directory
"""
db_filename = os.path.join(
data_dir, 'merkle-{}.lmdb'.format(network_endpoint[-2:]))
LOGGER.debug('database file is %s', db_filename)
merkle_db = LMDBNoLockDatabase(db_filename, 'c')
context_manager = ContextManager(merkle_db)
self._context_manager = context_manager
state_view_factory = StateViewFactory(merkle_db)
block_db_filename = os.path.join(
data_dir, 'block-{}.lmdb'.format(network_endpoint[-2:]))
LOGGER.debug('block store file is %s', block_db_filename)
block_db = LMDBNoLockDatabase(block_db_filename, 'c')
block_store = BlockStore(block_db)
# setup network
self._dispatcher = Dispatcher()
thread_pool = ThreadPoolExecutor(max_workers=10)
process_pool = ProcessPoolExecutor(max_workers=3)
self._thread_pool = thread_pool
self._process_pool = process_pool
self._service = Interconnect(component_endpoint,
self._dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20)
executor = TransactionExecutor(service=self._service,
context_manager=context_manager,
config_view_factory=ConfigViewFactory(
StateViewFactory(merkle_db)))
self._executor = executor
zmq_identity = hashlib.sha512(
time.time().hex().encode()).hexdigest()[:23]
network_thread_pool = ThreadPoolExecutor(max_workers=10)
self._network_thread_pool = network_thread_pool
self._network_dispatcher = Dispatcher()
# Server public and private keys are hardcoded here due to
# the decision to avoid having separate identities for each
# validator's server socket. This is appropriate for a public
# network. For a permissioned network with requirements for
# server endpoint authentication at the network level, this can
# be augmented with a local lookup service for side-band provided
# endpoint, public_key pairs and a local configuration option
# for 'server' side private keys.
self._network = Interconnect(
network_endpoint,
dispatcher=self._network_dispatcher,
zmq_identity=zmq_identity,
secured=True,
server_public_key=b'wFMwoOt>yFqI/ek.G[tfMMILHWw#vXB[Sv}>l>i)',
server_private_key=b'r&oJ5aQDj4+V]p2:Lz70Eu0x#m%IwzBdP(}&hWM*',
heartbeat=True,
connection_timeout=30,
max_incoming_connections=100)
self._gossip = Gossip(self._network, initial_peer_endpoints=peer_list)
completer = Completer(block_store, self._gossip)
block_sender = BroadcastBlockSender(completer, self._gossip)
batch_sender = BroadcastBatchSender(completer, self._gossip)
chain_id_manager = ChainIdManager(data_dir)
# Create and configure journal
self._journal = Journal(
block_store=block_store,
state_view_factory=StateViewFactory(merkle_db),
block_sender=block_sender,
batch_sender=batch_sender,
transaction_executor=executor,
squash_handler=context_manager.get_squash_handler(),
identity_signing_key=identity_signing_key,
chain_id_manager=chain_id_manager,
data_dir=data_dir,
check_publish_block_frequency=0.1,
block_cache_purge_frequency=30,
block_cache_keep_time=300)
self._genesis_controller = GenesisController(
context_manager=context_manager,
transaction_executor=executor,
completer=completer,
block_store=block_store,
state_view_factory=state_view_factory,
identity_key=identity_signing_key,
data_dir=data_dir,
chain_id_manager=chain_id_manager,
batch_sender=batch_sender)
responder = Responder(completer)
completer.set_on_batch_received(self._journal.on_batch_received)
completer.set_on_block_received(self._journal.on_block_received)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_GET_REQUEST,
tp_state_handlers.TpStateGetHandler(context_manager), thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_SET_REQUEST,
tp_state_handlers.TpStateSetHandler(context_manager), thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_REGISTER_REQUEST,
processor_handlers.ProcessorRegisterHandler(executor.processors),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_UNREGISTER_REQUEST,
processor_handlers.ProcessorUnRegisterHandler(executor.processors),
thread_pool)
# Set up base network handlers
self._network_dispatcher.add_handler(
validator_pb2.Message.NETWORK_PING, PingHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.NETWORK_CONNECT,
ConnectHandler(network=self._network), network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.NETWORK_DISCONNECT,
DisconnectHandler(network=self._network), network_thread_pool)
# Set up gossip handlers
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_REGISTER,
PeerRegisterHandler(gossip=self._gossip), network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_UNREGISTER,
PeerUnregisterHandler(gossip=self._gossip), network_thread_pool)
# GOSSIP_MESSAGE 1) Sends acknowledgement to the sender
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(),
network_thread_pool)
# GOSSIP_MESSAGE 2) Verifies signature
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
signature_verifier.GossipMessageSignatureVerifier(), process_pool)
# GOSSIP_MESSAGE 3) Determines if we should broadcast the
# message to our peers. It is important that this occur prior
# to the sending of the message to the completer, as this step
# relies on whether the gossip message has previously been
# seen by the validator to determine whether or not forwarding
# should occur
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
GossipBroadcastHandler(gossip=self._gossip, completer=completer),
network_thread_pool)
# GOSSIP_MESSAGE 4) Send message to completer
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
CompleterGossipHandler(completer), network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_REQUEST,
BlockResponderHandler(responder, self._gossip),
network_thread_pool)
# GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
GossipBlockResponseHandler(), network_thread_pool)
# GOSSIP_BLOCK_RESPONSE 2) Verifies signature
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
signature_verifier.GossipBlockResponseSignatureVerifier(),
process_pool)
# GOSSIP_BLOCK_RESPONSE 3) Send message to completer
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
CompleterGossipBlockResponseHandler(completer),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST,
BatchByBatchIdResponderHandler(responder, self._gossip),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST,
BatchByTransactionIdResponderHandler(responder, self._gossip),
network_thread_pool)
# GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
GossipBatchResponseHandler(), network_thread_pool)
# GOSSIP_BATCH_RESPONSE 2) Verifies signature
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
signature_verifier.GossipBatchResponseSignatureVerifier(),
process_pool)
# GOSSIP_BATCH_RESPONSE 3) Send message to completer
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
CompleterGossipBatchResponseHandler(completer),
network_thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
signature_verifier.BatchListSignatureVerifier(), process_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
CompleterBatchListBroadcastHandler(completer, self._gossip),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
client_handlers.BatchSubmitFinisher(
self._journal.get_block_store(), completer.batch_cache),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST,
client_handlers.BatchStatusRequest(self._journal.get_block_store(),
completer.batch_cache),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_LIST_REQUEST,
client_handlers.StateListRequest(merkle_db,
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_GET_REQUEST,
client_handlers.StateGetRequest(merkle_db,
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST,
client_handlers.BlockListRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST,
client_handlers.BlockGetRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST,
client_handlers.BatchListRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_GET_REQUEST,
client_handlers.BatchGetRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST,
client_handlers.StateCurrentRequest(
self._journal.get_current_root), thread_pool)
def __init__(self, network_endpoint, component_endpoint, peer_list,
data_dir):
"""Constructs a validator instance.
Args:
network_endpoint (str): the network endpoint
component_endpoint (str): the component endpoint
peer_list (list of str): a list of peer addresses
data_dir (str): path to the data directory
"""
db_filename = os.path.join(
data_dir, 'merkle-{}.lmdb'.format(network_endpoint[-2:]))
LOGGER.debug('database file is %s', db_filename)
lmdb = LMDBNoLockDatabase(db_filename, 'n')
context_manager = ContextManager(lmdb)
block_db_filename = os.path.join(data_dir, 'block.lmdb')
LOGGER.debug('block store file is %s', block_db_filename)
# block_store = LMDBNoLockDatabase(block_db_filename, 'n')
block_store = {}
# this is not currently being used but will be something like this
# in the future, when Journal takes a block_store that isn't a dict
# setup network
self._dispatcher = Dispatcher()
completer = Completer(block_store)
thread_pool = ThreadPoolExecutor(max_workers=10)
process_pool = ProcessPoolExecutor(max_workers=3)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_GET_REQUEST,
tp_state_handlers.TpStateGetHandler(context_manager), thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_SET_REQUEST,
tp_state_handlers.TpStateSetHandler(context_manager), thread_pool)
self._service = Interconnect(component_endpoint,
self._dispatcher,
secured=False)
executor = TransactionExecutor(self._service, context_manager)
self._dispatcher.add_handler(
validator_pb2.Message.TP_REGISTER_REQUEST,
ProcessorRegisterHandler(executor.processors), thread_pool)
identity = hashlib.sha512(time.time().hex().encode()).hexdigest()[:23]
network_thread_pool = ThreadPoolExecutor(max_workers=10)
self._network_dispatcher = Dispatcher()
# Server public and private keys are hardcoded here due to
# the decision to avoid having separate identities for each
# validator's server socket. This is appropriate for a public
# network. For a permissioned network with requirements for
# server endpoint authentication at the network level, this can
# be augmented with a local lookup service for side-band provided
# endpoint, public_key pairs and a local configuration option
# for 'server' side private keys.
self._network = Interconnect(
network_endpoint,
dispatcher=self._network_dispatcher,
identity=identity,
peer_connections=peer_list,
secured=True,
server_public_key=b'wFMwoOt>yFqI/ek.G[tfMMILHWw#vXB[Sv}>l>i)',
server_private_key=b'r&oJ5aQDj4+V]p2:Lz70Eu0x#m%IwzBdP(}&hWM*')
self._gossip = Gossip(self._network)
block_sender = BroadcastBlockSender(completer, self._gossip)
self._network_dispatcher.add_handler(validator_pb2.Message.GOSSIP_PING,
PingHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
signature_verifier.GossipMessageSignatureVerifier(), process_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
GossipBroadcastHandler(gossip=self._gossip), network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
CompleterGossipHandler(completer), network_thread_pool)
# Create and configure journal
self._journal = Journal(
consensus=dev_mode_consensus,
block_store=block_store,
block_sender=block_sender,
transaction_executor=executor,
squash_handler=context_manager.get_squash_handler())
self._genesis_controller = GenesisController(
context_manager=context_manager,
transaction_executor=executor,
completer=completer,
block_store=block_store,
data_dir=data_dir)
completer.set_on_batch_received(self._journal.on_batch_received)
completer.set_on_block_received(self._journal.on_block_received)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
signature_verifier.BatchListSignatureVerifier(), process_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
CompleterBatchListBroadcastHandler(completer, self._gossip),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_LIST_REQUEST,
client_handlers.StateListRequest(lmdb,
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_GET_REQUEST,
client_handlers.StateGetRequest(lmdb,
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST,
client_handlers.BlockGetRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST,
client_handlers.BlockListRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST,
client_handlers.StateCurrentRequest(
self._journal.get_current_root), thread_pool)
def __init__(self, network_endpoint, component_endpoint, peer_list):
data_dir = os.path.expanduser('~')
db_filename = os.path.join(data_dir,
'merkle-{}.lmdb'.format(
network_endpoint[-2:]))
LOGGER.debug('database file is %s', db_filename)
lmdb = LMDBNoLockDatabase(db_filename, 'n')
context_manager = ContextManager(lmdb)
block_db_filename = os.path.join(data_dir, 'block.lmdb')
LOGGER.debug('block store file is %s', block_db_filename)
block_store = {}
# block_store = LMDBNoLockDatabase(block_db_filename, 'n')
# this is not currently being used but will be something like this
# in the future, when Journal takes a block_store that isn't a dict
# setup network
self._dispatcher = Dispatcher()
completer = Completer()
thread_pool = ThreadPoolExecutor(max_workers=10)
process_pool = ProcessPoolExecutor(max_workers=3)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_GET_REQUEST,
tp_state_handlers.TpStateGetHandler(context_manager),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_SET_REQUEST,
tp_state_handlers.TpStateSetHandler(context_manager),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_GET_REQUEST,
client_handlers.StateGetRequestHandler(lmdb),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_LIST_REQUEST,
client_handlers.StateListRequestHandler(lmdb),
thread_pool)
self._service = Interconnect(component_endpoint, self._dispatcher)
executor = TransactionExecutor(self._service, context_manager)
self._dispatcher.add_handler(
validator_pb2.Message.TP_REGISTER_REQUEST,
ProcessorRegisterHandler(executor.processors),
thread_pool)
identity = hashlib.sha512(
time.time().hex().encode()).hexdigest()[:23]
network_thread_pool = ThreadPoolExecutor(max_workers=10)
self._network_dispatcher = Dispatcher()
self._network = Interconnect(
network_endpoint,
dispatcher=self._network_dispatcher,
identity=identity,
peer_connections=peer_list)
self._gossip = Gossip(self._network)
block_sender = BroadcastBlockSender(completer, self._gossip)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_PING,
PingHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_REGISTER,
PeerRegisterHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_UNREGISTER,
PeerUnregisterHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
GossipMessageHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
signature_verifier.GossipMessageSignatureVerifier(),
process_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
GossipBroadcastHandler(
gossip=self._gossip),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
CompleterGossipHandler(
completer),
network_thread_pool)
# Create and configure journal
self._journal = Journal(
consensus=dev_mode_consensus,
block_store=block_store,
block_sender=block_sender,
transaction_executor=executor,
squash_handler=context_manager.get_squash_handler())
self._genesis_controller = GenesisController(
context_manager=context_manager,
transaction_executor=executor,
completer=completer,
block_store=block_store,
data_dir=data_dir
)
completer.set_on_batch_received(self._journal.on_batch_received)
completer.set_on_block_received(self._journal.on_block_received)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
signature_verifier.BatchListSignatureVerifier(),
process_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
CompleterBatchListBroadcastHandler(
completer, self._gossip),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_GET_REQUEST,
client_handlers.StateGetRequestHandler(lmdb),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_LIST_REQUEST,
client_handlers.StateListRequestHandler(lmdb),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST,
client_handlers.BlockGetRequestHandler(
self._journal.get_block_store()), thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST,
client_handlers.BlockListRequestHandler(
self._journal.get_block_store()), thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST,
client_handlers.StateCurrentRequestHandler(
self._journal.get_current_root), thread_pool)
def __init__(self, bind_network, bind_component, endpoint,
peering, seeds_list, peer_list, data_dir, config_dir,
identity_signing_key, scheduler_type,
network_public_key=None,
network_private_key=None):
"""Constructs a validator instance.
Args:
bind_network (str): the network endpoint
bind_component (str): the component endpoint
endpoint (str): the zmq-style URI of this validator's
publically reachable endpoint
peering (str): The type of peering approach. Either 'static'
or 'dynamic'. In 'static' mode, no attempted topology
buildout occurs -- the validator only attempts to initiate
peering connections with endpoints specified in the
peer_list. In 'dynamic' mode, the validator will first
attempt to initiate peering connections with endpoints
specified in the peer_list and then attempt to do a
topology buildout starting with peer lists obtained from
endpoints in the seeds_list. In either mode, the validator
will accept incoming peer requests up to max_peers.
seeds_list (list of str): a list of addresses to connect
to in order to perform the initial topology buildout
peer_list (list of str): a list of peer addresses
data_dir (str): path to the data directory
config_dir (str): path to the config directory
identity_signing_key (str): key validator uses for signing
"""
db_filename = os.path.join(data_dir,
'merkle-{}.lmdb'.format(
bind_network[-2:]))
LOGGER.debug('database file is %s', db_filename)
merkle_db = LMDBNoLockDatabase(db_filename, 'c')
delta_db_filename = os.path.join(data_dir,
'state-deltas-{}.lmdb'.format(
bind_network[-2:]))
LOGGER.debug('state delta store file is %s', delta_db_filename)
state_delta_db = LMDBNoLockDatabase(delta_db_filename, 'c')
state_delta_store = StateDeltaStore(state_delta_db)
context_manager = ContextManager(merkle_db, state_delta_store)
self._context_manager = context_manager
state_view_factory = StateViewFactory(merkle_db)
block_db_filename = os.path.join(data_dir, 'block-{}.lmdb'.format(
bind_network[-2:]))
LOGGER.debug('block store file is %s', block_db_filename)
block_db = LMDBNoLockDatabase(block_db_filename, 'c')
block_store = BlockStore(block_db)
batch_tracker = BatchTracker(block_store)
block_store.add_update_observer(batch_tracker)
# setup network
self._dispatcher = Dispatcher()
thread_pool = ThreadPoolExecutor(max_workers=10)
sig_pool = ThreadPoolExecutor(max_workers=3)
self._thread_pool = thread_pool
self._sig_pool = sig_pool
self._service = Interconnect(bind_component,
self._dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True)
config_file = os.path.join(config_dir, "validator.toml")
validator_config = {}
if os.path.exists(config_file):
with open(config_file) as fd:
raw_config = fd.read()
validator_config = toml.loads(raw_config)
if scheduler_type is None:
scheduler_type = validator_config.get("scheduler", "serial")
executor = TransactionExecutor(
service=self._service,
context_manager=context_manager,
settings_view_factory=SettingsViewFactory(
StateViewFactory(merkle_db)),
scheduler_type=scheduler_type,
invalid_observers=[batch_tracker])
self._executor = executor
self._service.set_check_connections(executor.check_connections)
state_delta_processor = StateDeltaProcessor(self._service,
state_delta_store,
block_store)
zmq_identity = hashlib.sha512(
time.time().hex().encode()).hexdigest()[:23]
network_thread_pool = ThreadPoolExecutor(max_workers=10)
self._network_thread_pool = network_thread_pool
self._network_dispatcher = Dispatcher()
secure = False
if network_public_key is not None and network_private_key is not None:
secure = True
self._network = Interconnect(
bind_network,
dispatcher=self._network_dispatcher,
zmq_identity=zmq_identity,
secured=secure,
server_public_key=network_public_key,
server_private_key=network_private_key,
heartbeat=True,
public_endpoint=endpoint,
connection_timeout=30,
max_incoming_connections=100)
self._gossip = Gossip(self._network,
endpoint=endpoint,
peering_mode=peering,
initial_seed_endpoints=seeds_list,
initial_peer_endpoints=peer_list,
minimum_peer_connectivity=3,
maximum_peer_connectivity=10,
topology_check_frequency=1)
completer = Completer(block_store, self._gossip)
block_sender = BroadcastBlockSender(completer, self._gossip)
batch_sender = BroadcastBatchSender(completer, self._gossip)
chain_id_manager = ChainIdManager(data_dir)
# Create and configure journal
self._journal = Journal(
block_store=block_store,
state_view_factory=StateViewFactory(merkle_db),
block_sender=block_sender,
batch_sender=batch_sender,
transaction_executor=executor,
squash_handler=context_manager.get_squash_handler(),
identity_signing_key=identity_signing_key,
chain_id_manager=chain_id_manager,
state_delta_processor=state_delta_processor,
data_dir=data_dir,
config_dir=config_dir,
check_publish_block_frequency=0.1,
block_cache_purge_frequency=30,
block_cache_keep_time=300,
batch_observers=[batch_tracker]
)
self._genesis_controller = GenesisController(
context_manager=context_manager,
transaction_executor=executor,
completer=completer,
block_store=block_store,
state_view_factory=state_view_factory,
identity_key=identity_signing_key,
data_dir=data_dir,
config_dir=config_dir,
chain_id_manager=chain_id_manager,
batch_sender=batch_sender
)
responder = Responder(completer)
completer.set_on_batch_received(self._journal.on_batch_received)
completer.set_on_block_received(self._journal.on_block_received)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_GET_REQUEST,
tp_state_handlers.TpStateGetHandler(context_manager),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_STATE_SET_REQUEST,
tp_state_handlers.TpStateSetHandler(context_manager),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_REGISTER_REQUEST,
processor_handlers.ProcessorRegisterHandler(executor.processors),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.TP_UNREGISTER_REQUEST,
processor_handlers.ProcessorUnRegisterHandler(executor.processors),
thread_pool)
# Set up base network handlers
self._network_dispatcher.add_handler(
validator_pb2.Message.NETWORK_PING,
PingHandler(),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.NETWORK_CONNECT,
ConnectHandler(network=self._network),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.NETWORK_DISCONNECT,
DisconnectHandler(network=self._network),
network_thread_pool)
# Set up gossip handlers
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_GET_PEERS_REQUEST,
GetPeersRequestHandler(gossip=self._gossip),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_GET_PEERS_RESPONSE,
GetPeersResponseHandler(gossip=self._gossip),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_REGISTER,
PeerRegisterHandler(gossip=self._gossip),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_UNREGISTER,
PeerUnregisterHandler(gossip=self._gossip),
network_thread_pool)
# GOSSIP_MESSAGE 1) Sends acknowledgement to the sender
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
GossipMessageHandler(),
network_thread_pool)
# GOSSIP_MESSAGE 2) Verifies signature
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
signature_verifier.GossipMessageSignatureVerifier(),
sig_pool)
# GOSSIP_MESSAGE 3) Verifies batch structure
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
structure_verifier.GossipHandlerStructureVerifier(),
network_thread_pool)
# GOSSIP_MESSAGE 4) Determines if we should broadcast the
# message to our peers. It is important that this occur prior
# to the sending of the message to the completer, as this step
# relies on whether the gossip message has previously been
# seen by the validator to determine whether or not forwarding
# should occur
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
GossipBroadcastHandler(
gossip=self._gossip,
completer=completer),
network_thread_pool)
# GOSSIP_MESSAGE 5) Send message to completer
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_MESSAGE,
CompleterGossipHandler(
completer),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_REQUEST,
BlockResponderHandler(responder, self._gossip),
network_thread_pool)
# GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
GossipBlockResponseHandler(),
network_thread_pool)
# GOSSIP_BLOCK_RESPONSE 2) Verifies signature
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
signature_verifier.GossipBlockResponseSignatureVerifier(),
sig_pool)
# GOSSIP_BLOCK_RESPONSE 3) Check batch structure
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
structure_verifier.GossipBlockResponseStructureVerifier(),
network_thread_pool)
# GOSSIP_BLOCK_RESPONSE 4) Send message to completer
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
CompleterGossipBlockResponseHandler(
completer),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BLOCK_RESPONSE,
ResponderBlockResponseHandler(responder, self._gossip),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST,
BatchByBatchIdResponderHandler(responder, self._gossip),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST,
BatchByTransactionIdResponderHandler(responder, self._gossip),
network_thread_pool)
# GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
GossipBatchResponseHandler(),
network_thread_pool)
# GOSSIP_BATCH_RESPONSE 2) Verifies signature
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
signature_verifier.GossipBatchResponseSignatureVerifier(),
sig_pool)
# GOSSIP_BATCH_RESPONSE 3) Check batch structure
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
structure_verifier.GossipBatchResponseStructureVerifier(),
network_thread_pool)
# GOSSIP_BATCH_RESPONSE 4) Send message to completer
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
CompleterGossipBatchResponseHandler(
completer),
network_thread_pool)
self._network_dispatcher.add_handler(
validator_pb2.Message.GOSSIP_BATCH_RESPONSE,
ResponderBatchResponseHandler(responder, self._gossip),
network_thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
BatchListPermissionVerifier(
settings_view_factory=SettingsViewFactory(
StateViewFactory(merkle_db)),
current_root_func=self._journal.get_current_root,
),
sig_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
signature_verifier.BatchListSignatureVerifier(),
sig_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
structure_verifier.BatchListStructureVerifier(),
network_thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
CompleterBatchListBroadcastHandler(
completer, self._gossip),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
client_handlers.BatchSubmitFinisher(batch_tracker),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST,
client_handlers.BatchStatusRequest(batch_tracker),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_LIST_REQUEST,
client_handlers.StateListRequest(
merkle_db,
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_GET_REQUEST,
client_handlers.StateGetRequest(
merkle_db,
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST,
client_handlers.BlockListRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST,
client_handlers.BlockGetRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST,
client_handlers.BatchListRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_BATCH_GET_REQUEST,
client_handlers.BatchGetRequest(self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_TRANSACTION_LIST_REQUEST,
client_handlers.TransactionListRequest(
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_TRANSACTION_GET_REQUEST,
client_handlers.TransactionGetRequest(
self._journal.get_block_store()),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST,
client_handlers.StateCurrentRequest(
self._journal.get_current_root), thread_pool)
# State Delta Subscription Handlers
self._dispatcher.add_handler(
validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST,
StateDeltaSubscriberValidationHandler(state_delta_processor),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST,
StateDeltaAddSubscriberHandler(state_delta_processor),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.STATE_DELTA_UNSUBSCRIBE_REQUEST,
StateDeltaUnsubscriberHandler(state_delta_processor),
thread_pool)
self._dispatcher.add_handler(
validator_pb2.Message.STATE_DELTA_GET_EVENTS_REQUEST,
StateDeltaGetEventsHandler(block_store, state_delta_store),
thread_pool)
class ConsensusStateStore(MutableMapping):
"""Manages access to the underlying database holding per-block consensus
state information. Note that because of the architectural model around
the consensus objects, all ConsensusStateStore objects actually reference
a single underlying database. Provides a dict-like interface to the
consensus state, mapping block IDs to their corresponding consensus state.
"""
_store_dbs = {}
_lock = threading.Lock()
def __init__(self, data_dir, validator_id):
"""Initialize the consensus state store
Args:
data_dir (str): The directory where underlying database file will
be stored
validator_id (str): A unique ID for the validator for which the
consensus state store is being created
Returns:
None
"""
with ConsensusStateStore._lock:
# Create an underlying LMDB database file for the validator if
# there already isn't one. We will create the LMDB with the 'c'
# flag so that it will open if already exists.
self._store_db = ConsensusStateStore._store_dbs.get(validator_id)
if self._store_db is None:
db_file_name = \
os.path.join(
data_dir,
'poet_consensus_state-{}.lmdb'.format(
validator_id[:8]))
LOGGER.debug('Create consensus store: %s', db_file_name)
self._store_db = LMDBNoLockDatabase(db_file_name, 'c')
ConsensusStateStore._store_dbs[validator_id] = self._store_db
def __setitem__(self, block_id, consensus_state):
"""Adds/updates an item in the consensus state store
Args:
block_id (str): The ID of the block that this consensus state
corresponds to
consensus_state (ConsensusState): The consensus state
Returns:
None
"""
self._store_db[block_id] = consensus_state.serialize_to_bytes()
def __getitem__(self, block_id):
"""Return the consensus state corresponding to the block ID
Args:
block_id (str): The ID of the block for which consensus state
is being requested
Returns:
ConsensusState object
Raises:
KeyError if the block ID is not in the store
"""
serialized_consensus_state = self._store_db[block_id]
if serialized_consensus_state is None:
raise KeyError('Block ID {} not found'.format(block_id))
try:
consensus_state = ConsensusState()
consensus_state.parse_from_bytes(
buffer=serialized_consensus_state)
return consensus_state
except ValueError as error:
raise \
KeyError(
'Cannot return block with ID {}: {}'.format(
block_id,
error))
def __delitem__(self, block_id):
del self._store_db[block_id]
def __contains__(self, block_id):
return block_id in self._store_db
def __iter__(self):
# Required by abstract base class, but implementing is non-trivial
raise NotImplementedError('ConsensusState is not iterable')
def __len__(self):
return len(self._store_db)
def __str__(self):
out = []
for block_id in self._store_db.keys():
try:
serialized_consensus_state = self._store_db[block_id]
consensus_state = ConsensusState()
consensus_state.parse_from_bytes(
buffer=serialized_consensus_state)
out.append(
'{}...{}: {{{}}}'.format(
block_id[:8],
block_id[-8:],
consensus_state))
except ValueError:
pass
return ', '.join(out)
# pylint: disable=arguments-differ
def get(self, block_id, default=None):
"""Return the consensus state corresponding to block ID or the default
value if none exists
Args:
block_id (str): The ID of the block for which consensus state
is being requested
default (ConsensusState): The default value to return if there
is no consensus state associated with the block ID
Returns:
ConsensusState object or default if no state for key
"""
try:
return self.__getitem__(block_id)
except KeyError:
pass
return default
import struct
from sawtooth_validator.database.lmdb_nolock_database import LMDBNoLockDatabase
def _serialize_tuple(tup):
return "{}-{}-{}".format(*tup).encode()
def _deserialize_tuple(bytestring):
(rec_id, name, data) = tuple(bytestring.decode().split('-'))
return (int(rec_id), name, data)
db = LMDBNoLockDatabase(
os.path.join('/tmp/sawtooth', 'nolock_db'),
flag='c')
def do_put():
db.put('1', (1, "foo", "bar"))
db.put('2', (2, "alice", "Alice's data"))
db.put('3', (3, "bob", "Bob's data"))
def test_get_multi():
"""Given a database with three records and an index, test that it can
return multiple values from a set of keys.
Show that:
- it returns all existing values in the list
def verify_state(bind_network, bind_component, scheduler_type, data_dir=None):
"""
Verify the state root hash of all blocks is in state and if not,
reconstruct the missing state. Assumes that there are no "holes" in
state, ie starting from genesis, state is present for all blocks up to some
point and then not at all. If persist is False, this recomputes state in
memory for all blocks in the blockstore and verifies the state root
hashes.
Raises:
InvalidChainError: The chain in the blockstore is not valid.
ExecutionError: An unrecoverable error was encountered during batch
execution.
"""
# Get the global state database to operate on
if data_dir is not None:
global_state_db_filename = os.path.join(
data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('verifying state in %s', global_state_db_filename)
global_state_db = LMDBNoLockDatabase(global_state_db_filename, 'c')
else:
global_state_db = DictDatabase()
state_view_factory = StateViewFactory(global_state_db)
# Get the blockstore
block_db_filename = os.path.join(data_dir,
'block-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('block store file is %s', block_db_filename)
block_db = IndexedDatabase(block_db_filename,
BlockStore.serialize_block,
BlockStore.deserialize_block,
flag='c',
indexes=BlockStore.create_index_configuration())
blockstore = BlockStore(block_db)
# Check if we should do state verification
start_block, prev_state_root = search_for_present_state_root(
blockstore, state_view_factory)
if start_block is None:
LOGGER.info(
"Skipping state verification: chain head's state root is present")
return
LOGGER.info("Recomputing missing state from block %s with %s scheduler",
start_block, scheduler_type)
component_thread_pool = InstrumentedThreadPoolExecutor(max_workers=10,
name='Component')
component_dispatcher = Dispatcher()
component_service = Interconnect(bind_component,
component_dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True,
max_future_callback_workers=10)
context_manager = ContextManager(global_state_db)
transaction_executor = TransactionExecutor(
service=component_service,
context_manager=context_manager,
settings_view_factory=SettingsViewFactory(state_view_factory),
scheduler_type=scheduler_type,
invalid_observers=[])
component_service.set_check_connections(
transaction_executor.check_connections)
component_dispatcher.add_handler(
validator_pb2.Message.TP_RECEIPT_ADD_DATA_REQUEST,
tp_state_handlers.TpReceiptAddDataHandler(context_manager),
component_thread_pool)
component_dispatcher.add_handler(
validator_pb2.Message.TP_EVENT_ADD_REQUEST,
tp_state_handlers.TpEventAddHandler(context_manager),
component_thread_pool)
component_dispatcher.add_handler(
validator_pb2.Message.TP_STATE_DELETE_REQUEST,
tp_state_handlers.TpStateDeleteHandler(context_manager),
component_thread_pool)
component_dispatcher.add_handler(
validator_pb2.Message.TP_STATE_GET_REQUEST,
tp_state_handlers.TpStateGetHandler(context_manager),
component_thread_pool)
component_dispatcher.add_handler(
validator_pb2.Message.TP_STATE_SET_REQUEST,
tp_state_handlers.TpStateSetHandler(context_manager),
component_thread_pool)
component_dispatcher.add_handler(
validator_pb2.Message.TP_REGISTER_REQUEST,
processor_handlers.ProcessorRegisterHandler(
transaction_executor.processors), component_thread_pool)
component_dispatcher.add_handler(
validator_pb2.Message.TP_UNREGISTER_REQUEST,
processor_handlers.ProcessorUnRegisterHandler(
transaction_executor.processors), component_thread_pool)
component_dispatcher.start()
component_service.start()
process_blocks(initial_state_root=prev_state_root,
blocks=blockstore.get_block_iter(start_block=start_block,
reverse=False),
transaction_executor=transaction_executor,
context_manager=context_manager,
state_view_factory=state_view_factory)
component_dispatcher.stop()
component_service.stop()
component_thread_pool.shutdown(wait=True)
transaction_executor.stop()
context_manager.stop()
def __init__(self, network_endpoint, component_endpoint, peer_list):
db_filename = os.path.join(os.path.expanduser('~'), 'merkle.lmdb')
LOGGER.debug('database file is %s', db_filename)
lmdb = LMDBNoLockDatabase(db_filename, 'n')
context_manager = ContextManager(lmdb)
block_db_filename = os.path.join(os.path.expanduser('~'), 'block.lmdb')
LOGGER.debug('block store file is %s', block_db_filename)
# block_store = LMDBNoLockDatabase(block_db_filename, 'n')
# this is not currently being used but will be something like this
# in the future, when Journal takes a block_store that isn't a dict
self._service = Interconnect(component_endpoint)
# setup network
dispatcher = Dispatcher()
faux_network = FauxNetwork(dispatcher=dispatcher)
identity = "{}-{}".format(socket.gethostname(),
os.getpid()).encode('ascii')
self._network = Network(identity,
network_endpoint,
peer_list,
dispatcher=dispatcher)
# Create and configure journal
executor = TransactionExecutor(self._service, context_manager)
self._journal = Journal(
consensus=dev_mode_consensus,
block_store={},
# -- need to serialize blocks to dicts
send_message=faux_network.send_message,
transaction_executor=executor,
squash_handler=context_manager.get_squash_handler(),
first_state_root=context_manager.get_first_root())
dispatcher.on_batch_received = \
self._journal.on_batch_received
dispatcher.on_block_received = \
self._journal.on_block_received
dispatcher.on_block_request = \
self._journal.on_block_request
self._service.add_handler(
validator_pb2.Message.DEFAULT,
DefaultHandler())
self._service.add_handler(
validator_pb2.Message.TP_STATE_GET_REQUEST,
state.GetHandler(context_manager))
self._service.add_handler(
validator_pb2.Message.TP_STATE_SET_REQUEST,
state.SetHandler(context_manager))
self._service.add_handler(
validator_pb2.Message.TP_REGISTER_REQUEST,
ProcessorRegisterHandler(self._service))
self._service.add_handler(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
SystemLoadHandler(faux_network))
self._service.add_handler(
validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST,
ClientStateCurrentRequestHandler(self._journal.get_current_root))
self._service.add_handler(
validator_pb2.Message.CLIENT_STATE_GET_REQUEST,
ClientStateGetRequestHandler(lmdb))
self._service.add_handler(
validator_pb2.Message.CLIENT_STATE_LIST_REQUEST,
ClientStateListRequestHandler(lmdb))
