class TransactionProcessor:
"""TransactionProcessor is a generic class for communicating with a
validator and routing transaction processing requests to a registered
handler. It uses ZMQ and channels to handle requests concurrently.
"""
def __init__(self, url):
"""
Args:
url (string): The URL of the validator
"""
self._stream = Stream(url)
self._url = url
self._handlers = []
@property
def zmq_id(self):
return self._stream.zmq_id
def add_handler(self, handler):
"""Adds a transaction family handler
Args:
handler (TransactionHandler): the handler to be added
"""
self._handlers.append(handler)
def _matches(self, handler, header):
return header.family_name == handler.family_name \
and header.family_version in handler.family_versions
def _find_handler(self, header):
"""Find a handler for a particular (family_name, family_versions)
:param header transaction_pb2.TransactionHeader:
:return: handler
"""
try:
return next(
handler for handler in self._handlers
if self._matches(handler, header))
except StopIteration:
LOGGER.debug("Missing handler for header: %s", header)
return None
def _register_requests(self):
"""Returns all of the TpRegisterRequests for handlers
:return (list): list of TpRegisterRequests
"""
return itertools.chain.from_iterable( # flattens the nested list
[
[TpRegisterRequest(
family=n,
version=v,
namespaces=h.namespaces)
for n, v in itertools.product(
[h.family_name],
h.family_versions,)] for h in self._handlers])
def _unregister_request(self):
"""Returns a single TP_UnregisterRequest that requests
that the validator stop sending transactions for previously
registered handlers.
:return (processor_pb2.TpUnregisterRequest):
"""
return TpUnregisterRequest()
def _process(self, msg):
if msg.message_type != Message.TP_PROCESS_REQUEST:
LOGGER.debug(
"Transaction Processor recieved invalid message type. "
"Message type should be TP_PROCESS_REQUEST,"
" but is %s", Message.MessageType.Name(msg.message_type))
return
request = TpProcessRequest()
request.ParseFromString(msg.content)
state = Context(self._stream, request.context_id)
header = request.header
try:
if not self._stream.is_ready():
raise ValidatorConnectionError()
handler = self._find_handler(header)
if handler is None:
return
handler.apply(request, state)
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.OK
).SerializeToString())
except InvalidTransaction as it:
LOGGER.warning("Invalid Transaction %s", it)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INVALID_TRANSACTION,
message=str(it),
extended_data=it.extended_data
).SerializeToString())
except ValidatorConnectionError as vce:
# TP_PROCESS_REQUEST has made it through the
# handler.apply and an INVALID_TRANSACTION would have been
# sent back but the validator has disconnected and so it
# doesn't care about the response.
LOGGER.warning("during invalid transaction response: %s", vce)
except InternalError as ie:
LOGGER.warning("internal error: %s", ie)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INTERNAL_ERROR,
message=str(ie),
extended_data=ie.extended_data
).SerializeToString())
except ValidatorConnectionError as vce:
# Same as the prior except block, but an internal error has
# happened, but because of the disconnect the validator
# probably doesn't care about the response.
LOGGER.warning("during internal error response: %s", vce)
except ValidatorConnectionError as vce:
# Somewhere within handler.apply a future resolved with an
# error status that the validator has disconnected. There is
# nothing left to do but reconnect.
LOGGER.warning("during handler.apply a future was resolved "
"with error status: %s", vce)
except AuthorizationException as ae:
LOGGER.warning("AuthorizationException: %s", ae)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INVALID_TRANSACTION,
message=str(ae),
).SerializeToString())
except ValidatorConnectionError as vce:
# TP_PROCESS_REQUEST has made it through the
# handler.apply and an INVALID_TRANSACTION would have been
# sent back but the validator has disconnected and so it
# doesn't care about the response.
LOGGER.warning("during invalid transaction response: %s", vce)
def _process_future(self, future, timeout=None, sigint=False):
try:
msg = future.result(timeout)
except CancelledError:
# This error is raised when Task.cancel is called on
# disconnect from the validator in stream.py, for
# this future.
return
if msg is RECONNECT_EVENT:
if sigint is False:
LOGGER.info("reregistering with validator")
self._stream.wait_for_ready()
self._register()
else:
LOGGER.debug(
'received message of type: %s',
Message.MessageType.Name(msg.message_type))
if msg.message_type == Message.PING_REQUEST:
self._stream.send_back(
message_type=Message.PING_RESPONSE,
correlation_id=msg.correlation_id,
content=PingResponse().SerializeToString())
return
self._process(msg)
def _register(self):
futures = []
for message in self._register_requests():
self._stream.wait_for_ready()
future = self._stream.send(
message_type=Message.TP_REGISTER_REQUEST,
content=message.SerializeToString())
futures.append(future)
for future in futures:
resp = TpRegisterResponse()
try:
resp.ParseFromString(future.result().content)
LOGGER.info("register attempt: %s",
TpRegisterResponse.Status.Name(resp.status))
except ValidatorConnectionError as vce:
LOGGER.info("during waiting for response on registration: %s",
vce)
def _unregister(self):
message = self._unregister_request()
self._stream.wait_for_ready()
future = self._stream.send(
message_type=Message.TP_UNREGISTER_REQUEST,
content=message.SerializeToString())
response = TpUnregisterResponse()
try:
response.ParseFromString(future.result(1).content)
LOGGER.info("unregister attempt: %s",
TpUnregisterResponse.Status.Name(response.status))
except ValidatorConnectionError as vce:
LOGGER.info("during waiting for response on unregistration: %s",
vce)
def start(self):
"""Connects the transaction processor to a validator and starts
listening for requests and routing them to an appropriate
transaction handler.
"""
fut = None
try:
self._register()
while True:
# During long running processing this
# is where the transaction processor will
# spend most of its time
fut = self._stream.receive()
self._process_future(fut)
except KeyboardInterrupt:
try:
# tell the validator to not send any more messages
self._unregister()
while True:
if fut is not None:
# process futures as long as the tp has them,
# if the TP_PROCESS_REQUEST doesn't come from
# zeromq->asyncio in 1 second raise a
# concurrent.futures.TimeOutError and be done.
self._process_future(fut, 1, sigint=True)
fut = self._stream.receive()
except concurrent.futures.TimeoutError:
# Where the tp will usually exit after
# a KeyboardInterrupt. Caused by the 1 second
# timeout in _process_future.
pass
except FutureTimeoutError:
# If the validator is not able to respond to the
# unregister request, exit.
pass
def stop(self):
"""Closes the connection between the TransactionProcessor and the
validator.
"""
self._stream.close()
class TransactionProcessor(object):
def __init__(self, url):
self._stream = Stream(url)
self._url = url
self._handlers = []
@property
def zmq_id(self):
return self._stream.zmq_id
def add_handler(self, handler):
"""Add a transaction family handler
:param handler:
"""
self._handlers.append(handler)
def _matches(self, handler, header):
return header.family_name == handler.family_name \
and header.family_version in handler.family_versions \
and header.payload_encoding in handler.encodings
def _find_handler(self, header):
"""Find a handler for a particular (family_name,
family_versions, payload_encoding)
:param header transaction_pb2.TransactionHeader:
:return: handler
"""
try:
return next(handler for handler in self._handlers
if self._matches(handler, header))
except StopIteration:
LOGGER.debug("Missing handler for header: %s", header)
return None
def _register_requests(self):
"""Returns all of the TpRegisterRequests for handlers
:return (list): list of TpRegisterRequests
"""
return itertools.chain.from_iterable( # flattens the nested list
[[
TpRegisterRequest(family=n,
version=v,
encoding=e,
namespaces=h.namespaces)
for n, v, e in itertools.product(
[h.family_name], h.family_versions, h.encodings)
] for h in self._handlers])
def _unregister_request(self):
"""Returns a single TP_UnregisterRequest that requests
that the validator stop sending transactions for previously
registered handlers.
:return (processor_pb2.TpUnregisterRequest):
"""
return TpUnregisterRequest()
def _process(self, msg):
if msg.message_type != Message.TP_PROCESS_REQUEST:
LOGGER.debug(
"Transaction Processor recieved invalid message type. "
"Message type should be TP_PROCESS_REQUEST,"
" but is %s", Message.MessageType.Name(msg.message_type))
return
request = TpProcessRequest()
request.ParseFromString(msg.content)
state = State(self._stream, request.context_id)
header = TransactionHeader()
header.ParseFromString(request.header)
try:
if not self._stream.is_ready():
raise ValidatorConnectionError()
handler = self._find_handler(header)
if handler is None:
return
handler.apply(request, state)
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.OK).SerializeToString())
except InvalidTransaction as it:
LOGGER.warning("Invalid Transaction %s", it)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INVALID_TRANSACTION,
message=str(it),
extended_data=it.extended_data).SerializeToString())
except ValidatorConnectionError as vce:
# TP_PROCESS_REQUEST has made it through the
# handler.apply and an INVALID_TRANSACTION would have been
# sent back but the validator has disconnected and so it
# doesn't care about the response.
LOGGER.warning("during invalid transaction response: %s", vce)
except InternalError as ie:
LOGGER.warning("internal error: %s", ie)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INTERNAL_ERROR,
message=str(ie),
extended_data=ie.extended_data).SerializeToString())
except ValidatorConnectionError as vce:
# Same as the prior except block, but an internal error has
# happened, but because of the disconnect the validator
# probably doesn't care about the response.
LOGGER.warning("during internal error response: %s", vce)
except ValidatorConnectionError as vce:
# Somewhere within handler.apply a future resolved with an
# error status that the validator has disconnected. There is
# nothing left to do but reconnect.
LOGGER.warning(
"during handler.apply a future was resolved "
"with error status: %s", vce)
def _process_future(self, future, timeout=None, sigint=False):
try:
msg = future.result(timeout)
except CancelledError:
# This error is raised when Task.cancel is called on
# disconnect from the validator in stream.py, for
# this future.
return
if msg is RECONNECT_EVENT:
if sigint is False:
LOGGER.info("reregistering with validator")
self._stream.wait_for_ready()
self._register()
else:
LOGGER.debug('received message of type: %s',
Message.MessageType.Name(msg.message_type))
if msg.message_type == Message.TP_PING:
self._stream.send_back(
message_type=Message.TP_PING_RESPONSE,
correlation_id=msg.correlation_id,
content=TpPingResponse(
status=TpPingResponse.OK).SerializeToString())
return
self._process(msg)
def _register(self):
futures = []
for message in self._register_requests():
self._stream.wait_for_ready()
future = self._stream.send(
message_type=Message.TP_REGISTER_REQUEST,
content=message.SerializeToString())
futures.append(future)
for future in futures:
resp = TpRegisterResponse()
try:
resp.ParseFromString(future.result().content)
LOGGER.info("register attempt: %s",
TpRegisterResponse.Status.Name(resp.status))
except ValidatorConnectionError as vce:
LOGGER.info("during waiting for response on registration: %s",
vce)
def _unregister(self):
message = self._unregister_request()
self._stream.wait_for_ready()
future = self._stream.send(message_type=Message.TP_UNREGISTER_REQUEST,
content=message.SerializeToString())
response = TpUnregisterResponse()
try:
response.ParseFromString(future.result(1).content)
LOGGER.info("unregister attempt: %s",
TpUnregisterResponse.Status.Name(response.status))
except ValidatorConnectionError as vce:
LOGGER.info("during waiting for response on unregistration: %s",
vce)
def start(self):
fut = None
try:
self._register()
while True:
# During long running processing this
# is where the transaction processor will
# spend most of its time
fut = self._stream.receive()
self._process_future(fut)
except KeyboardInterrupt:
try:
# tell the validator to not send any more messages
self._unregister()
while True:
if fut is not None:
# process futures as long as the tp has them,
# if the TP_PROCESS_REQUEST doesn't come from
# zeromq->asyncio in 1 second raise a
# concurrent.futures.TimeOutError and be done.
self._process_future(fut, 1, sigint=True)
fut = self._stream.receive()
except concurrent.futures.TimeoutError:
# Where the tp will usually exit after
# a KeyboardInterrupt. Caused by the 1 second
# timeout in _process_future.
pass
except FutureTimeoutError:
# If the validator is not able to respond to the
# unregister request, exit.
pass
def stop(self):
self._stream.close()
class TransactionProcessor:
"""TransactionProcessor is a generic class for communicating with a
validator and routing transaction processing requests to a registered
handler. It uses ZMQ and channels to handle requests concurrently.
"""
# This is the version used by SDK to match if validator supports feature
# it requested during registration. It should only be incremented when
# there are changes in TpRegisterRequest. Remember to sync this
# information in validator if changed.
# Note: SDK_PROTOCOL_VERSION is the highest version the SDK supports
class _FeatureVersion(Enum):
FEATURE_UNUSED = 0
FEATURE_CUSTOM_HEADER_STYLE = 1
SDK_PROTOCOL_VERSION = 1
def __init__(self, url):
"""
Args:
url (string): The URL of the validator
"""
self._stream = Stream(url)
self._url = url
self._handlers = []
self._highest_sdk_feature_requested = \
self._FeatureVersion.FEATURE_UNUSED
self._header_style = TpRegisterRequest.HEADER_STYLE_UNSET
@property
def zmq_id(self):
return self._stream.zmq_id
def add_handler(self, handler):
"""Adds a transaction family handler
Args:
handler (TransactionHandler): the handler to be added
"""
self._handlers.append(handler)
def set_header_style(self, style):
"""Sets a flag to request the validator for custom transaction header
style in TpProcessRequest.
Args:
style (TpProcessRequestHeaderStyle): enum value to set header style
"""
if self._FeatureVersion.FEATURE_CUSTOM_HEADER_STYLE.value > \
self._highest_sdk_feature_requested.value:
self._highest_sdk_feature_requested = \
self._FeatureVersion.FEATURE_CUSTOM_HEADER_STYLE
self._header_style = style
def _matches(self, handler, header):
return header.family_name == handler.family_name \
and header.family_version in handler.family_versions
def _find_handler(self, header):
"""Find a handler for a particular (family_name, family_versions)
:param header transaction_pb2.TransactionHeader:
:return: handler
"""
try:
return next(handler for handler in self._handlers
if self._matches(handler, header))
except StopIteration:
LOGGER.debug("Missing handler for header: %s", header)
return None
def _register_requests(self):
"""Returns all of the TpRegisterRequests for handlers
:return (list): list of TpRegisterRequests
"""
return itertools.chain.from_iterable( # flattens the nested list
[[
TpRegisterRequest(
family=n,
version=v,
namespaces=h.namespaces,
protocol_version=self._highest_sdk_feature_requested.value,
request_header_style=self._header_style)
for n, v in itertools.product(
[h.family_name],
h.family_versions,
)
] for h in self._handlers])
def _unregister_request(self):
"""Returns a single TP_UnregisterRequest that requests
that the validator stop sending transactions for previously
registered handlers.
:return (processor_pb2.TpUnregisterRequest):
"""
return TpUnregisterRequest()
def _process(self, msg):
if msg.message_type != Message.TP_PROCESS_REQUEST:
LOGGER.debug(
"Transaction Processor recieved invalid message type. "
"Message type should be TP_PROCESS_REQUEST,"
" but is %s", Message.MessageType.Name(msg.message_type))
return
request = TpProcessRequest()
request.ParseFromString(msg.content)
state = Context(self._stream, request.context_id)
if self._header_style == TpRegisterRequest.RAW:
header = TransactionHeader()
header.ParseFromString(request.header_bytes)
else:
header = request.header
try:
if not self._stream.is_ready():
raise ValidatorConnectionError()
handler = self._find_handler(header)
if handler is None:
return
handler.apply(request, state)
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.OK).SerializeToString())
except InvalidTransaction as it:
LOGGER.warning("Invalid Transaction %s", it)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INVALID_TRANSACTION,
message=str(it),
extended_data=it.extended_data).SerializeToString())
except ValidatorConnectionError as vce:
# TP_PROCESS_REQUEST has made it through the
# handler.apply and an INVALID_TRANSACTION would have been
# sent back but the validator has disconnected and so it
# doesn't care about the response.
LOGGER.warning("during invalid transaction response: %s", vce)
except InternalError as ie:
LOGGER.warning("internal error: %s", ie)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INTERNAL_ERROR,
message=str(ie),
extended_data=ie.extended_data).SerializeToString())
except ValidatorConnectionError as vce:
# Same as the prior except block, but an internal error has
# happened, but because of the disconnect the validator
# probably doesn't care about the response.
LOGGER.warning("during internal error response: %s", vce)
except ValidatorConnectionError as vce:
# Somewhere within handler.apply a future resolved with an
# error status that the validator has disconnected. There is
# nothing left to do but reconnect.
LOGGER.warning(
"during handler.apply a future was resolved "
"with error status: %s", vce)
except AuthorizationException as ae:
LOGGER.warning("AuthorizationException: %s", ae)
try:
self._stream.send_back(
message_type=Message.TP_PROCESS_RESPONSE,
correlation_id=msg.correlation_id,
content=TpProcessResponse(
status=TpProcessResponse.INVALID_TRANSACTION,
message=str(ae),
).SerializeToString())
except ValidatorConnectionError as vce:
# TP_PROCESS_REQUEST has made it through the
# handler.apply and an INVALID_TRANSACTION would have been
# sent back but the validator has disconnected and so it
# doesn't care about the response.
LOGGER.warning("during invalid transaction response: %s", vce)
def _process_future(self, future, timeout=None, sigint=False):
try:
msg = future.result(timeout)
except CancelledError:
# This error is raised when Task.cancel is called on
# disconnect from the validator in stream.py, for
# this future.
return
if msg is RECONNECT_EVENT:
if sigint is False:
LOGGER.info("reregistering with validator")
self._stream.wait_for_ready()
self._register()
else:
LOGGER.debug('received message of type: %s',
Message.MessageType.Name(msg.message_type))
if msg.message_type == Message.PING_REQUEST:
self._stream.send_back(
message_type=Message.PING_RESPONSE,
correlation_id=msg.correlation_id,
content=PingResponse().SerializeToString())
return
self._process(msg)
def _register(self):
futures = []
for message in self._register_requests():
self._stream.wait_for_ready()
future = self._stream.send(
message_type=Message.TP_REGISTER_REQUEST,
content=message.SerializeToString())
futures.append(future)
for future in futures:
resp = TpRegisterResponse()
try:
resp.ParseFromString(future.result().content)
if resp.protocol_version != \
self._highest_sdk_feature_requested.value:
LOGGER.error(
"Validator version %s does not support "
"requested feature by SDK version %s. "
"Unregistering with the validator.",
str(resp.protocol_version),
str(self._highest_sdk_feature_requested.value))
raise ValidatorVersionError()
LOGGER.info("register attempt: %s",
TpRegisterResponse.Status.Name(resp.status))
if resp.status == TpRegisterResponse.ERROR:
raise RuntimeError("Transaction processor registration "
"failed")
except ValidatorConnectionError as vce:
LOGGER.info("during waiting for response on registration: %s",
vce)
def _unregister(self):
message = self._unregister_request()
self._stream.wait_for_ready()
future = self._stream.send(message_type=Message.TP_UNREGISTER_REQUEST,
content=message.SerializeToString())
response = TpUnregisterResponse()
try:
response.ParseFromString(future.result(1).content)
LOGGER.info("unregister attempt: %s",
TpUnregisterResponse.Status.Name(response.status))
except ValidatorConnectionError as vce:
LOGGER.info("during waiting for response on unregistration: %s",
vce)
def start(self):
"""Connects the transaction processor to a validator and starts
listening for requests and routing them to an appropriate
transaction handler.
"""
fut = None
try:
self._register()
while True:
# During long running processing this
# is where the transaction processor will
# spend most of its time
fut = self._stream.receive()
self._process_future(fut)
except (KeyboardInterrupt, ValidatorVersionError):
try:
# tell the validator to not send any more messages
self._unregister()
while True:
if fut is not None:
# process futures as long as the tp has them,
# if the TP_PROCESS_REQUEST doesn't come from
# zeromq->asyncio in 1 second raise a
# concurrent.futures.TimeOutError and be done.
self._process_future(fut, 1, sigint=True)
fut = self._stream.receive()
except concurrent.futures.TimeoutError:
# Where the tp will usually exit after
# a KeyboardInterrupt. Caused by the 1 second
# timeout in _process_future.
pass
except FutureTimeoutError:
# If the validator is not able to respond to the
# unregister request, exit.
pass
except RuntimeError as e:
LOGGER.error("Error: %s", e)
self.stop()
def stop(self):
"""Closes the connection between the TransactionProcessor and the
validator.
"""
self._stream.close()
