def __init__(self, can_accept_fn, queue_info_fn, metrics_registry=None):
self._can_accept = can_accept_fn
self._queue_info = queue_info_fn
self._applying_backpressure = False
if metrics_registry:
self._batches_rejected_count = CounterWrapper(
metrics_registry.counter(
'backpressure_batches_rejected_count'))
self._batches_rejected_gauge = GaugeWrapper(
metrics_registry.gauge('backpressure_batches_rejected_gauge',
default=0))
else:
self._batches_rejected_count = CounterWrapper()
self._batches_rejected_gauge = GaugeWrapper()
def __init__(self,
max_workers=None,
name='',
trace=None,
metrics_registry=None):
if trace is None:
self._trace = 'SAWTOOTH_TRACE_LOGGING' in os.environ
else:
self._trace = trace
self._name = name
if name == '':
self._name = 'Instrumented'
LOGGER.debug('Creating thread pool executor %s', self._name)
self._workers_in_use = atomic.Counter()
self._max_workers = max_workers
if self._max_workers is None:
# This is the same default as ThreadPoolExecutor, but we want to
# know how many workers there are for logging
self._max_workers = multiprocessing.cpu_count() * 5
super().__init__(max_workers)
if metrics_registry:
# Tracks how many workers are already in use
self._workers_already_in_use_gauge = GaugeWrapper(
metrics_registry.gauge(
'{}-threadpool.workers_already_in_use'.format(self._name)))
# Tracks how long tasks take to run
self._task_run_timer = TimerWrapper(
metrics_registry.timer('{}-threadpool.task_run_time'.format(
self._name)))
# Tracks how long tasks wait in the queue
self._task_time_in_queue_timer = TimerWrapper(
metrics_registry.timer(
'{}-threadpool.task_time_in_queue'.format(self._name)))
else:
self._workers_already_in_use_gauge = GaugeWrapper()
self._task_run_timer = TimerWrapper()
self._task_time_in_queue_timer = TimerWrapper()
def __init__(self,
block_store,
gossip,
cache_keep_time=1200,
cache_purge_frequency=30,
requested_keep_time=300,
metrics_registry=None):
"""
:param block_store (dictionary) The block store shared with the journal
:param gossip (gossip.Gossip) Broadcasts block and batch request to
peers
:param cache_keep_time (float) Time in seconds to keep values in
TimedCaches.
:param cache_purge_frequency (float) Time between purging the
TimedCaches.
:param requested_keep_time (float) Time in seconds to keep the ids
of requested objects. WARNING this time should always be less than
cache_keep_time or the validator can get into a state where it
fails to make progress because it thinks it has already requested
something that it is missing.
"""
self.gossip = gossip
self.batch_cache = TimedCache(cache_keep_time, cache_purge_frequency)
self.block_cache = BlockCache(block_store, cache_keep_time,
cache_purge_frequency)
self._block_store = block_store
# avoid throwing away the genesis block
self.block_cache[NULL_BLOCK_IDENTIFIER] = None
self._seen_txns = TimedCache(cache_keep_time, cache_purge_frequency)
self._incomplete_batches = TimedCache(cache_keep_time,
cache_purge_frequency)
self._incomplete_blocks = TimedCache(cache_keep_time,
cache_purge_frequency)
self._requested = TimedCache(requested_keep_time,
cache_purge_frequency)
self._on_block_received = None
self._on_batch_received = None
self._has_block = None
self.lock = RLock()
if metrics_registry:
# Tracks how many times an unsatisfied dependency is found
self._unsatisfied_dependency_count = CounterWrapper(
metrics_registry.counter(
'completer.unsatisfied_dependency_count'))
# Tracks the length of the completer's _seen_txns
self._seen_txns_length = GaugeWrapper(
metrics_registry.gauge('completer.seen_txns_length'))
# Tracks the length of the completer's _incomplete_blocks
self._incomplete_blocks_length = GaugeWrapper(
metrics_registry.gauge('completer.incomplete_blocks_length'))
# Tracks the length of the completer's _incomplete_batches
self._incomplete_batches_length = GaugeWrapper(
metrics_registry.gauge('completer.incomplete_batches_length'))
else:
self._unsatisfied_dependency_count = CounterWrapper()
self._seen_txns_length = GaugeWrapper()
self._incomplete_blocks_length = GaugeWrapper()
self._incomplete_batches_length = GaugeWrapper()
def __init__(self,
transaction_executor,
block_cache,
state_view_factory,
settings_cache,
block_sender,
batch_sender,
squash_handler,
chain_head,
identity_signer,
data_dir,
config_dir,
permission_verifier,
check_publish_block_frequency,
batch_observers,
batch_injector_factory=None,
metrics_registry=None):
"""
Initialize the BlockPublisher object
Args:
transaction_executor (:obj:`TransactionExecutor`): A
TransactionExecutor instance.
block_cache (:obj:`BlockCache`): A BlockCache instance.
state_view_factory (:obj:`StateViewFactory`): StateViewFactory for
read-only state views.
block_sender (:obj:`BlockSender`): The BlockSender instance.
batch_sender (:obj:`BatchSender`): The BatchSender instance.
squash_handler (function): Squash handler function for merging
contexts.
chain_head (:obj:`BlockWrapper`): The initial chain head.
identity_signer (:obj:`Signer`): Cryptographic signer for signing
blocks
data_dir (str): path to location where persistent data for the
consensus module can be stored.
config_dir (str): path to location where configuration can be
found.
batch_injector_factory (:obj:`BatchInjectorFatctory`): A factory
for creating BatchInjectors.
metrics_registry (MetricsRegistry): Metrics registry used to
create pending batch gauge
"""
self._lock = RLock()
self._candidate_block = None # _CandidateBlock helper,
# the next block in potential chain
self._block_cache = block_cache
self._state_view_factory = state_view_factory
self._settings_cache = settings_cache
self._transaction_executor = transaction_executor
self._block_sender = block_sender
self._batch_publisher = BatchPublisher(identity_signer, batch_sender)
self._pending_batches = [] # batches we are waiting for validation,
# arranged in the order of batches received.
self._pending_batch_ids = []
self._publish_count_average = _RollingAverage(
NUM_PUBLISH_COUNT_SAMPLES, INITIAL_PUBLISH_COUNT)
self._chain_head = chain_head # block (BlockWrapper)
self._squash_handler = squash_handler
self._identity_signer = identity_signer
self._data_dir = data_dir
self._config_dir = config_dir
self._permission_verifier = permission_verifier
self._batch_injector_factory = batch_injector_factory
# For metric gathering
if metrics_registry:
self._pending_batch_gauge = GaugeWrapper(
metrics_registry.gauge('pending_batch_gauge'))
self._blocks_published_count = CounterWrapper(
metrics_registry.counter('blocks_published_count'))
else:
self._blocks_published_count = CounterWrapper()
self._pending_batch_gauge = GaugeWrapper()
self._batch_queue = queue.Queue()
self._queued_batch_ids = []
self._batch_observers = batch_observers
self._check_publish_block_frequency = check_publish_block_frequency
self._publisher_thread = None
# A series of states that allow us to check for condition changes.
# These can be used to log only at the boundary of condition changes.
self._logging_states = _PublisherLoggingStates()
def __init__(self,
block_cache,
block_sender,
state_view_factory,
transaction_executor,
chain_head_lock,
on_chain_updated,
squash_handler,
chain_id_manager,
identity_signer,
data_dir,
config_dir,
permission_verifier,
chain_observers,
thread_pool=None,
metrics_registry=None):
"""Initialize the ChainController
Args:
block_cache: The cache of all recent blocks and the processing
state associated with them.
block_sender: an interface object used to send blocks to the
network.
state_view_factory: The factory object to create
transaction_executor: The TransactionExecutor used to produce
schedulers for batch validation.
chain_head_lock: Lock to hold while the chain head is being
updated, this prevents other components that depend on the
chain head and the BlockStore from having the BlockStore change
under them. This lock is only for core Journal components
(BlockPublisher and ChainController), other components should
handle block not found errors from the BlockStore explicitly.
on_chain_updated: The callback to call to notify the rest of the
system the head block in the chain has been changed.
squash_handler: a parameter passed when creating transaction
schedulers.
chain_id_manager: The ChainIdManager instance.
identity_signer: Private key for signing blocks.
data_dir: path to location where persistent data for the
consensus module can be stored.
config_dir: path to location where config data for the
consensus module can be found.
chain_observers (list of :obj:`ChainObserver`): A list of chain
observers.
Returns:
None
"""
self._lock = RLock()
self._chain_head_lock = chain_head_lock
self._block_cache = block_cache
self._block_store = block_cache.block_store
self._state_view_factory = state_view_factory
self._block_sender = block_sender
self._transaction_executor = transaction_executor
self._notify_on_chain_updated = on_chain_updated
self._squash_handler = squash_handler
self._identity_signer = identity_signer
self._data_dir = data_dir
self._config_dir = config_dir
self._blocks_processing = {} # a set of blocks that are
# currently being processed.
self._blocks_pending = {} # set of blocks that the previous block
# is being processed. Once that completes this block will be
# scheduled for validation.
self._chain_id_manager = chain_id_manager
self._chain_head = None
self._permission_verifier = permission_verifier
self._chain_observers = chain_observers
if metrics_registry:
self._chain_head_gauge = GaugeWrapper(
metrics_registry.gauge('chain_head', default='no chain head'))
self._committed_transactions_count = CounterWrapper(
metrics_registry.counter('committed_transactions_count'))
self._block_num_gauge = GaugeWrapper(
metrics_registry.gauge('block_num'))
else:
self._chain_head_gauge = GaugeWrapper()
self._committed_transactions_count = CounterWrapper()
self._block_num_gauge = GaugeWrapper()
self._block_queue = queue.Queue()
self._thread_pool = \
InstrumentedThreadPoolExecutor(1) \
if thread_pool is None else thread_pool
self._chain_thread = None
# Only run this after all member variables have been bound
self._set_chain_head_from_block_store()
def __init__(self, block_cache, block_sender, state_view_factory, executor,
transaction_executor, chain_head_lock, on_chain_updated,
squash_handler, chain_id_manager, identity_signing_key,
data_dir, config_dir, permission_verifier, chain_observers,
metrics_registry):
"""Initialize the ChainController
Args:
block_cache: The cache of all recent blocks and the processing
state associated with them.
block_sender: an interface object used to send blocks to the
network.
state_view_factory: The factory object to create
executor: The thread pool to process block validations.
transaction_executor: The TransactionExecutor used to produce
schedulers for batch validation.
chain_head_lock: Lock to hold while the chain head is being
updated, this prevents other components that depend on the
chain head and the BlockStore from having the BlockStore change
under them. This lock is only for core Journal components
(BlockPublisher and ChainController), other components should
handle block not found errors from the BlockStore explicitly.
on_chain_updated: The callback to call to notify the rest of the
system the head block in the chain has been changed.
squash_handler: a parameter passed when creating transaction
schedulers.
chain_id_manager: The ChainIdManager instance.
identity_signing_key: Private key for signing blocks.
data_dir: path to location where persistent data for the
consensus module can be stored.
config_dir: path to location where config data for the
consensus module can be found.
chain_observers (list of :obj:`ChainObserver`): A list of chain
observers.
Returns:
None
"""
self._lock = RLock()
self._chain_head_lock = chain_head_lock
self._block_cache = block_cache
self._block_store = block_cache.block_store
self._state_view_factory = state_view_factory
self._block_sender = block_sender
self._executor = executor
self._transaction_executor = transaction_executor
self._notify_on_chain_updated = on_chain_updated
self._squash_handler = squash_handler
self._identity_signing_key = identity_signing_key
self._identity_public_key = \
signing.generate_pubkey(self._identity_signing_key)
self._data_dir = data_dir
self._config_dir = config_dir
self._blocks_processing = {} # a set of blocks that are
# currently being processed.
self._blocks_pending = {} # set of blocks that the previous block
# is being processed. Once that completes this block will be
# scheduled for validation.
self._chain_id_manager = chain_id_manager
try:
self._chain_head = self._block_store.chain_head
if self._chain_head is not None:
LOGGER.info("Chain controller initialized with chain head: %s",
self._chain_head)
except Exception as exc:
LOGGER.error("Invalid block store. Head of the block chain cannot "
"be determined")
LOGGER.exception(exc)
raise
self._notify_on_chain_updated(self._chain_head)
self._permission_verifier = permission_verifier
self._chain_observers = chain_observers
self._chain_head_gauge = \
metrics_registry.gauge('chain_head', default='no chain head') \
if metrics_registry else None
if metrics_registry:
self._committed_transactions_count = CounterWrapper(
metrics_registry.counter('committed_transactions_count'))
self._block_num_gauge = GaugeWrapper(
metrics_registry.gauge('block_num'))
else:
self._committed_transactions_count = CounterWrapper()
self._block_num_gauge = GaugeWrapper()
def __init__(self,
block_cache,
block_validator,
state_view_factory,
chain_head_lock,
on_chain_updated,
chain_id_manager,
data_dir,
config_dir,
chain_observers,
metrics_registry=None):
"""Initialize the ChainController
Args:
block_cache: The cache of all recent blocks and the processing
state associated with them.
block_validator: The object to use for submitting block validation
work.
state_view_factory: A factory that can be used to create read-
only views of state for a particular merkle root, in
particular the state as it existed when a particular block
was the chain head.
chain_head_lock: Lock to hold while the chain head is being
updated, this prevents other components that depend on the
chain head and the BlockStore from having the BlockStore change
under them. This lock is only for core Journal components
(BlockPublisher and ChainController), other components should
handle block not found errors from the BlockStore explicitly.
on_chain_updated: The callback to call to notify the rest of the
system the head block in the chain has been changed.
chain_id_manager: The ChainIdManager instance.
data_dir: path to location where persistent data for the
consensus module can be stored.
config_dir: path to location where config data for the
consensus module can be found.
chain_observers (list of :obj:`ChainObserver`): A list of chain
observers.
metrics_registry: (Optional) Pyformance metrics registry handle for
creating new metrics.
Returns:
None
"""
self._lock = RLock()
self._chain_head_lock = chain_head_lock
self._block_cache = block_cache
self._block_store = block_cache.block_store
self._state_view_factory = state_view_factory
self._notify_on_chain_updated = on_chain_updated
self._data_dir = data_dir
self._config_dir = config_dir
self._blocks_processing = {} # a set of blocks that are
# currently being processed.
self._blocks_pending = {} # set of blocks that the previous block
# is being processed. Once that completes this block will be
# scheduled for validation.
self._chain_id_manager = chain_id_manager
self._chain_head = None
self._chain_observers = chain_observers
self._metrics_registry = metrics_registry
if metrics_registry:
self._chain_head_gauge = GaugeWrapper(
metrics_registry.gauge('chain_head', default='no chain head'))
self._committed_transactions_count = CounterWrapper(
metrics_registry.counter('committed_transactions_count'))
self._block_num_gauge = GaugeWrapper(
metrics_registry.gauge('block_num'))
self._blocks_considered_count = CounterWrapper(
metrics_registry.counter('blocks_considered_count'))
else:
self._chain_head_gauge = GaugeWrapper()
self._committed_transactions_count = CounterWrapper()
self._block_num_gauge = GaugeWrapper()
self._blocks_considered_count = CounterWrapper()
self._block_queue = queue.Queue()
self._chain_thread = None
self._block_validator = block_validator
# Only run this after all member variables have been bound
self._set_chain_head_from_block_store()