async def propose_and_commit(self, txn: dict) -> bool:
# Act as transaction Leader
self.ledger.add_transaction(txn)
# Stage-1: Propose
co = sirius_sdk.CoProtocolThreadedTheirs(
thid='consensus1-txn-' + uuid.uuid4().hex,
theirs=self.microledger
)
results = await co.switch(
message=Message({
'@type': TYPE_BFT_CONSENSUS_PROPOSE,
'txn': txn,
# According to algorithm Merkle-Proofs used for validation by participants
'uncommitted_root_hash': self.ledger.uncommitted_root_hash
})
)
unreachable = [pairwise.their.did for pairwise, (ok, _) in results.items() if not ok]
errored = [pairwise.their.did for pairwise, (ok, msg) in results.items() if ok and msg['@type'] != TYPE_BFT_CONSENSUS_PRE_COMMIT]
if unreachable or errored:
# Some error occur. Exit with participants notification
await co.send(
message=Message({
'@type': TYPE_BFT_CONSENSUS_PROBLEM,
'problem-code': 'some-code',
'explain': 'Some error occur in participants: ' + ','.join([p.their.did for p in unreachable + errored])
})
)
self.ledger.reset_uncommitted()
return False
# Allocate PreCommits. Assumed every participant signed self copy of PreCommit so
# all others may check consistency among all network
pre_commits = [msg for _, (_, msg) in results.items()]
# Stage-2: Pre-Commit
results = await co.switch(
message=Message({
'@type': TYPE_BFT_CONSENSUS_COMMIT,
'pre_commits': pre_commits,
})
)
unreachable = [pairwise.their.did for pairwise, (ok, _) in results.items() if not ok]
errored = [pairwise.their.did for pairwise, (ok, msg) in results.items() if
ok and msg['@type'] != TYPE_BFT_CONSENSUS_COMMIT]
# Stage-3: check commits
if unreachable or errored:
# Some error occur. Exit with participants notification
await co.send(
message=Message({
'@type': TYPE_BFT_CONSENSUS_PROBLEM,
'problem-code': 'some-code',
'explain': 'Some error occur in participants: ' + ','.join(
[p.their.did for p in unreachable + errored])
})
)
self.ledger.reset_uncommitted()
return False
# Commit to local storage
self.ledger.commit()
return True
async def receive(self, timeout: int = None) -> Message:
"""
Read message.
Tunnel allows to receive non-encrypted messages, high-level logic may control message encryption flag
via context.encrypted field
:param timeout:timeout in seconds
:return: received packet
"""
payload = await self.__input.read(timeout)
if not isinstance(payload, bytes) and not isinstance(payload, dict):
raise TypeError('Expected bytes or dict, got {}'.format(
type(payload)))
if isinstance(payload, bytes):
try:
payload = json.loads(payload)
except Exception as e:
raise SiriusInvalidPayloadStructure(
"Invalid packed message") from e
if 'protected' in payload:
unpacked = self.__p2p.unpack(payload)
self.__context.encrypted = True
return Message(unpacked)
else:
self.__context.encrypted = False
return Message(payload)
async def _setup(self, context: Message):
# Extract proxy info
proxies = context.get('~proxy', [])
channel_rpc = None
channel_sub_protocol = None
for proxy in proxies:
if proxy['id'] == 'reverse':
channel_rpc = proxy['data']['json']['address']
elif proxy['id'] == 'sub-protocol':
channel_sub_protocol = proxy['data']['json']['address']
if channel_rpc is None:
raise RuntimeError('rpc channel is empty')
if channel_sub_protocol is None:
raise RuntimeError('sub-protocol channel is empty')
self.__tunnel_rpc = AddressedTunnel(address=channel_rpc,
input_=self._connector,
output_=self._connector,
p2p=self._p2p)
self.__tunnel_coprotocols = AddressedTunnel(
address=channel_sub_protocol,
input_=self._connector,
output_=self._connector,
p2p=self._p2p)
# Extract active endpoints
endpoints = context.get('~endpoints', [])
endpoint_collection = []
for endpoint in endpoints:
body = endpoint['data']['json']
address = body['address']
frontend_key = body.get('frontend_routing_key', None)
if frontend_key:
for routing_key in body.get('routing_keys', []):
is_default = routing_key['is_default']
key = routing_key['routing_key']
endpoint_collection.append(
Endpoint(address=address,
routing_keys=[key, frontend_key],
is_default=is_default))
else:
endpoint_collection.append(
Endpoint(address=address,
routing_keys=[],
is_default=False))
if not endpoint_collection:
raise RuntimeError('Endpoints are empty')
self.__endpoints = endpoint_collection
# Extract Networks
self.__networks = context.get('~networks', [])
async def accept_transaction(self, leader: sirius_sdk.Pairwise, txn_propose: Message) -> bool:
# Act as transaction acceptor
self.ledger.add_transaction(txn_propose['txn'])
assert txn_propose['@type'] == TYPE_BFT_CONSENSUS_PROPOSE
co = sirius_sdk.CoProtocolThreadedP2P(
thid=txn_propose['~thread']['thid'],
to=leader
)
# Stage-1: Check local ledger is in consistent state with leader
if self.ledger.uncommitted_root_hash != txn_propose['uncommitted_root_hash']:
await co.send(message=Message({
'@type': TYPE_BFT_CONSENSUS_PROBLEM,
'problem-code': 'some-code',
'explain': 'non consistent ledger states'
}))
self.ledger.reset_uncommitted()
return False
# stage-2: send pre-commit response and wait commits from all participants
# (assumed in production commits will be signed)
ok, response = await co.switch(message=Message({
'@type': TYPE_BFT_CONSENSUS_PRE_COMMIT,
'uncommitted_root_hash': self.ledger.uncommitted_root_hash
}))
if ok:
assert response['@type'] == TYPE_BFT_CONSENSUS_COMMIT
pre_commits = response['pre_commits']
# Here developers may check signatures and consistent
for pre_commit in pre_commits:
if pre_commit['uncommitted_root_hash'] != self.ledger.uncommitted_root_hash:
await co.send(message=Message({
'@type': TYPE_BFT_CONSENSUS_PROBLEM,
'problem-code': 'some-code',
'explain': 'non consistent ledger states'
}))
self.ledger.reset_uncommitted()
return False
# Ack commit
await co.send(message=Message({
'@type': TYPE_BFT_CONSENSUS_COMMIT,
}))
self.ledger.commit()
return True
else:
# Timeout occur or something else
self.ledger.reset_uncommitted()
return False
async def test_send_message_via_transport_via_websocket(
agent1: Agent, agent2: Agent):
await agent1.open()
await agent2.open()
try:
a2b = await get_pairwise(agent1, agent2)
b2a = await get_pairwise(agent2, agent1)
thread_id = 'thread-' + uuid.uuid4().hex
a2b.their.endpoint = a2b.their.endpoint.replace('http://', 'ws://')
transport_for_a = await agent1.spawn(thread_id, a2b)
await transport_for_a.start()
transport_for_b = await agent2.spawn(thread_id, b2a)
await transport_for_b.start()
print('\n>START')
stamp1 = datetime.now()
for n in range(TEST_ITERATIONS):
msg = Message({
'@id': 'message-id-' + uuid.uuid4().hex,
'@type':
'did:sov:BzCbsNYhMrjHiqZDTUASHg;spec/test/1.0/message',
"comment": "Hi. Are you listening?",
"response_requested": True
})
await transport_for_a.send(msg)
message, sender_vk, recip_vk = await transport_for_b.get_one()
assert message['@id'] == msg['@id']
print('\n>STOP')
stamp2 = datetime.now()
delta = stamp2 - stamp1
print(f'>timeout: {delta.seconds}')
finally:
await agent1.close()
await agent2.close()
async def __detect_current_environment(self) -> Environment:
async with sirius_sdk.context(**BAY_DOOR):
# Open communication channel to transmit requests and await events from participants
communication = sirius_sdk.CoProtocolThreadedTheirs(
thid='request-id-' + uuid.uuid4().hex,
theirs=self.airlocks,
time_to_live=5)
log('Bay Door: check environment')
# SWITCH method suspend runtime thread until events will be accumulated or error occur
results = await communication.switch(
message=Message({'@type': TYPE_STATE_REQUEST}))
has_error = any(
[ok is False for airlock, (ok, _) in results.items()])
if has_error:
ret = Environment.HOSTILE # if almost one airlock unreachable environment is hostile
else:
# Parse responses
airlock_statuses = [
response['status']
for airlock, (_, response) in results.items()
]
if all([s == State.CLOSED.value for s in airlock_statuses]):
ret = Environment.FRIENDLY # All airlocks should be closed
else:
ret = Environment.HOSTILE
log(f'Bay Door: current environment: {ret}')
return ret
async def test_send_message(agent1: Agent, agent2: Agent):
await agent1.open()
await agent2.open()
try:
a2b = await get_pairwise(agent1, agent2)
b2a = await get_pairwise(agent2, agent1)
listener = await agent2.subscribe()
print('\n>START')
stamp1 = datetime.now()
for n in range(TEST_ITERATIONS):
msg = Message({
'@id': 'message-id-' + uuid.uuid4().hex,
'@type':
'did:sov:BzCbsNYhMrjHiqZDTUASHg;spec/test/1.0/message',
"comment": "Hi. Are you listening?",
"response_requested": True
})
await agent1.send_to(msg, a2b)
resp = await listener.get_one()
assert resp.message['@id'] == msg['@id']
print('\n>STOP')
stamp2 = datetime.now()
delta = stamp2 - stamp1
print(f'>timeout: {delta.seconds}')
finally:
await agent1.close()
await agent2.close()
async def test_sane(p2p: dict):
agent_to_sdk = p2p['agent']['tunnel']
sdk_to_agent = p2p['sdk']['tunnel']
future = Future(tunnel=sdk_to_agent)
with pytest.raises(SiriusPendingOperation):
future.get_value()
expected = 'Test OK'
promise_msg = Message({
'@type': MSG_TYPE_FUTURE,
'@id': 'promise-message-id',
'is_tuple': False,
'is_bytes': False,
'value': expected,
'exception': None,
'~thread': {
'thid': future.promise['id']
}
})
ok = await future.wait(5)
assert ok is False
await agent_to_sdk.post(message=promise_msg)
ok = await future.wait(5)
assert ok is True
actual = future.get_value()
assert actual == expected
async def test_agents_communications(test_suite: ServerTestSuite):
agent1_params = test_suite.get_agent_params('agent1')
agent2_params = test_suite.get_agent_params('agent2')
entity1 = list(agent1_params['entities'].items())[0][1]
entity2 = list(agent2_params['entities'].items())[0][1]
agent1 = Agent(
server_address=agent1_params['server_address'],
credentials=agent1_params['credentials'],
p2p=agent1_params['p2p'],
timeout=5,
)
agent2 = Agent(
server_address=agent2_params['server_address'],
credentials=agent2_params['credentials'],
p2p=agent2_params['p2p'],
timeout=5,
)
await agent1.open()
await agent2.open()
try:
# Get endpoints
agent2_endpoint = [
e for e in agent2.endpoints if e.routing_keys == []
][0].address
agent2_listener = await agent2.subscribe()
# Exchange Pairwise
await agent1.wallet.did.store_their_did(entity2['did'],
entity2['verkey'])
if not await agent1.wallet.pairwise.is_pairwise_exists(entity2['did']):
print('#1')
await agent1.wallet.pairwise.create_pairwise(
their_did=entity2['did'], my_did=entity1['did'])
await agent2.wallet.did.store_their_did(entity1['did'],
entity1['verkey'])
if not await agent2.wallet.pairwise.is_pairwise_exists(entity1['did']):
print('#2')
await agent2.wallet.pairwise.create_pairwise(
their_did=entity1['did'], my_did=entity2['did'])
# Prepare message
trust_ping = Message({
'@id': 'trust-ping-message-' + uuid.uuid4().hex,
'@type': 'did:sov:BzCbsNYhMrjHiqZDTUASHg;spec/trust_ping/1.0/ping',
"comment": "Hi. Are you listening?",
"response_requested": True
})
await agent1.send_message(message=trust_ping,
their_vk=entity2['verkey'],
endpoint=agent2_endpoint,
my_vk=entity1['verkey'],
routing_keys=[])
event = await agent2_listener.get_one(timeout=5)
msg = event['message']
assert msg[
'@type'] == 'did:sov:BzCbsNYhMrjHiqZDTUASHg;spec/trust_ping/1.0/ping'
assert msg['@id'] == trust_ping.id
finally:
await agent1.close()
await agent2.close()
def __setup(self, message: Message, please_ack: bool = True):
if please_ack:
if PLEASE_ACK_DECORATOR not in message:
message[PLEASE_ACK_DECORATOR] = {'message_id': message.id}
if self.__thread_id:
thread = message.get(THREAD_DECORATOR, {})
if 'thid' not in thread:
thread['thid'] = self.__thread_id
message[THREAD_DECORATOR] = thread
async def __producer(t: AbstractCoProtocolTransport):
for n in range(TEST_ITERATIONS // 2):
msg = Message({
'@id': 'message-id-' + uuid.uuid4().hex,
'@type':
'did:sov:BzCbsNYhMrjHiqZDTUASHg;spec/test/1.0/message',
"comment": "RUN",
"response_requested": True
})
ok, resp = await t.switch(msg)
assert ok is True
msg = Message({
'@id': 'message-id-' + uuid.uuid4().hex,
'@type':
'did:sov:BzCbsNYhMrjHiqZDTUASHg;spec/test/1.0/message',
"comment": "STOP",
"response_requested": True
})
await t.send(msg)
def build_response(packet: Message):
if packet.get('@type') == MSG_TYPE_FUTURE:
if packet.get('~thread', None) is not None:
parsed = {'exception': None, 'value': None}
exception = packet['exception']
if exception:
parsed['exception'] = exception
else:
value = packet['value']
if packet['is_tuple']:
parsed['value'] = tuple(value)
elif packet['is_bytes']:
parsed['value'] = base64.b64decode(value.encode('ascii'))
else:
parsed['value'] = value
return parsed
else:
raise SiriusInvalidPayloadStructure('Expect ~thread decorator')
else:
raise SiriusInvalidType('Expect message type "%s"' % MSG_TYPE_FUTURE)
async def pull(self, timeout: int = None) -> Message:
if not self._connector.is_open:
raise SiriusConnectionClosed('Open agent connection at first')
data = None
for n in range(self.RECONNECT_TRY_COUNT):
try:
data = await self._connector.read(timeout=timeout)
break
except SiriusConnectionClosed:
await self._reopen()
if data is None:
SiriusConnectionClosed('agent unreachable')
try:
payload = json.loads(data.decode(self._connector.ENC))
except json.JSONDecodeError:
raise SiriusInvalidPayloadStructure()
if 'protected' in payload:
message = self._p2p.unpack(payload)
return Message(message)
else:
return Message(payload)
async def post(self, message: Message, encrypt: bool = True) -> bool:
"""Write message
:param message: message to send
:param encrypt: do encryption
:return: operation success
"""
if encrypt:
payload = self.__p2p.pack(message)
else:
payload = message.serialize().encode(self.ENC)
return await self.__output.write(payload)
async def broadcast_for_all_participants(txn: dict):
async with get_connection() as agent:
entity = settings.AGENT['entity']
participants_dids = [did for did in settings.PARTICIPANTS_META.keys() if did != entity]
for did in participants_dids:
to = await agent.pairwise_list.load_for_did(did)
if to:
msg = Message(txn)
print(f'============ SEND message to DID: {did} =======')
print(json.dumps(msg, indent=2, sort_keys=True))
print('================================================')
await agent.send_to(msg, to)
else:
print('Empty pairwise for DID: ' + did)
async def get_one(self, timeout: int = None) -> Event:
event = await self.__source.pull(timeout)
if 'message' in event:
ok, message = restore_message_instance(event['message'])
if ok:
event['message'] = message
else:
event['message'] = Message(event['message'])
their_verkey = event.get('sender_verkey', None)
if self.__pairwise_resolver and their_verkey:
pairwise = await self.__pairwise_resolver.load_for_verkey(
their_verkey)
else:
pairwise = None
return Event(pairwise=pairwise, **event)
async def routine_for_pinger(agent: Agent, p: Pairwise, thread_id: str):
transport = await agent.spawn(thread_id, p)
await transport.start()
try:
for n in range(TEST_ITERATIONS):
ping = Message({
'@id': 'message-id-' + uuid.uuid4().hex,
'@type': 'did:sov:BzCbsNYhMrjHiqZDTUASHg;spec/test/1.0/ping',
"comment": "Hi",
})
ok, pong = await transport.switch(ping)
assert ok
assert pong['@id'] == ping['@id']
finally:
await transport.stop()
async def listen(self):
# Bay door may acts as reactor: respond other devices with self status, etc. according to events protocol
# So, Sirius SDK provide building blocks to implement reactive nature of the Entity
async with sirius_sdk.context(**BAY_DOOR):
listener = await sirius_sdk.subscribe()
async for event in listener:
if event.message[
'@type'] == TYPE_STATE_REQUEST and event.pairwise is not None:
# Open communication channel from income event context
communication = await sirius_sdk.open_communication(event)
await communication.send(
message=Message({
'@type': TYPE_STATE_RESPONSE,
'status': self.state.value
}))
def build_request(msg_type: str, future: Future, params: dict) -> Message:
"""
:param msg_type: Aries RFCs attribute
https://github.com/hyperledger/aries-rfcs/tree/master/concepts/0020-message-types
:param future: Future to check response routine is completed
:param params: RPC call params
:return: RPC service packet
"""
typ = Type.from_str(msg_type)
if typ.protocol not in ['sirius_rpc', 'admin', 'microledgers', 'microledgers-batched']:
raise SiriusInvalidType('Expect sirius_rpc protocol')
return Message({
'@type': msg_type,
'@promise': future.promise,
'params': {k: incapsulate_param(v) for k, v in params.items()}
})
async def listen(self):
log(f'{self.name} listener started')
# AirLock acts as reactor: respond other devices with self status, etc. according to events protocol
# So, Sirius SDK provide building blocks to implement reactive nature of the Entity
async with sirius_sdk.context(**self.hub_credentials):
listener = await sirius_sdk.subscribe()
async for event in listener:
if event.message[
'@type'] == TYPE_STATE_REQUEST and event.pairwise is not None:
log(f'{self.name}: \tprocess state request')
# Open communication channel from income event context
communication = await sirius_sdk.open_communication(event)
await communication.send(
message=Message({
'@type': TYPE_STATE_RESPONSE,
'status': self.state.value
}))
async def test_set_indy_error(p2p: dict):
agent_to_sdk = p2p['agent']['tunnel']
sdk_to_agent = p2p['sdk']['tunnel']
future = Future(tunnel=sdk_to_agent)
exc = WalletItemAlreadyExists(error_code=ErrorCode.WalletItemAlreadyExists,
error_details=dict(
message='test error message',
indy_backtrace=''))
promise_msg = Message({
'@type': MSG_TYPE_FUTURE,
'@id': 'promise-message-id',
'is_tuple': False,
'is_bytes': False,
'value': None,
'exception': {
'indy': {
'error_code': exc.error_code,
'message': exc.message
},
'class_name': exc.__class__.__name__,
'printable': str(exc)
},
'~thread': {
'thid': future.promise['id']
}
})
await agent_to_sdk.post(message=promise_msg)
ok = await future.wait(5)
assert ok is True
has_exc = future.has_exception()
assert has_exc is True
fut_exc = None
try:
future.raise_exception()
except WalletItemAlreadyExists as exc:
fut_exc = exc
assert fut_exc is not None
assert isinstance(fut_exc, WalletItemAlreadyExists)
assert fut_exc.message == 'test error message'
async def create(cls,
server_address: str,
credentials: bytes,
p2p: P2PConnection,
timeout: int = IO_TIMEOUT,
loop: asyncio.AbstractEventLoop = None):
instance = cls(server_address, credentials, p2p, timeout, loop)
await instance._connector.open()
payload = await instance._connector.read(timeout=timeout)
context = Message.deserialize(payload.decode())
msg_type = context.get('@type', None)
if msg_type is None:
raise RuntimeError('message @type is empty')
elif msg_type != cls.MSG_TYPE_CONTEXT:
raise RuntimeError('message @type is empty')
else:
await instance._setup(context)
return instance
async def test_set_non_indy_error(p2p: dict):
agent_to_sdk = p2p['agent']['tunnel']
sdk_to_agent = p2p['sdk']['tunnel']
future = Future(tunnel=sdk_to_agent)
exc = RuntimeError('test error message')
promise_msg = Message({
'@type': MSG_TYPE_FUTURE,
'@id': 'promise-message-id',
'is_tuple': False,
'is_bytes': False,
'value': None,
'exception': {
'indy': None,
'class_name': exc.__class__.__name__,
'printable': str(exc)
},
'~thread': {
'thid': future.promise['id']
}
})
await agent_to_sdk.post(message=promise_msg)
ok = await future.wait(5)
assert ok is True
has_exc = future.has_exception()
assert has_exc is True
fut_exc = None
try:
future.raise_exception()
except SiriusPromiseContextException as exc:
fut_exc = exc
assert fut_exc is not None
assert isinstance(fut_exc, SiriusPromiseContextException)
assert fut_exc.printable == 'test error message'
assert fut_exc.class_name == 'RuntimeError'
async def __detect_current_environment(self) -> Environment:
async with sirius_sdk.context(**self.hub_credentials):
# Open communication channel to transmit requests and await events from participants
communication = sirius_sdk.CoProtocolThreadedP2P(
thid='request-id-' + uuid.uuid4().hex,
to=self.baydoor,
time_to_live=5)
log(f'AirLock[{self.index}]: check environment')
# SWITCH method suspend runtime thread until participant will respond or error/timeout occur
ok, response = await communication.switch(
message=Message({'@type': TYPE_STATE_REQUEST}))
if ok:
if response['status'] == State.CLOSED.value:
ret = Environment.FRIENDLY # Bay door should be closed for Friendly environment
else:
ret = Environment.HOSTILE
else:
# Timeout occur
ret = Environment.HOSTILE
log(f'AitLock[{self.index}]: current environment: {ret}')
return ret
def from_url(cls, url: str) -> ConnProtocolMessage:
matches = re.match("(.+)?c_i=(.+)", url)
if not matches:
raise SiriusInvalidMessage("Invite string is improperly formatted")
msg = Message.deserialize(
base64.urlsafe_b64decode(matches.group(2)).decode('utf-8'))
if msg.protocol != cls.PROTOCOL:
raise SiriusInvalidMessage('Unexpected protocol "%s"' %
msg.type.protocol)
if msg.name != cls.NAME:
raise SiriusInvalidMessage('Unexpected protocol name "%s"' %
msg.type.name)
label = msg.pop('label')
if label is None:
raise SiriusInvalidMessage('label attribute missing')
recipient_keys = msg.pop('recipientKeys')
if recipient_keys is None:
raise SiriusInvalidMessage('recipientKeys attribute missing')
endpoint = msg.pop('serviceEndpoint')
if endpoint is None:
raise SiriusInvalidMessage('serviceEndpoint attribute missing')
routing_keys = msg.pop('routingKeys', [])
return Invitation(label, recipient_keys, endpoint, routing_keys, **msg)
async def test_bytes_value(p2p: dict):
agent_to_sdk = p2p['agent']['tunnel']
sdk_to_agent = p2p['sdk']['tunnel']
future = Future(tunnel=sdk_to_agent)
expected = b'Hello!'
promise_msg = Message({
'@type': MSG_TYPE_FUTURE,
'@id': 'promise-message-id',
'is_tuple': False,
'is_bytes': True,
'value': base64.b64encode(expected).decode('ascii'),
'exception': None,
'~thread': {
'thid': future.promise['id']
}
})
await agent_to_sdk.post(message=promise_msg)
ok = await future.wait(3)
assert ok is True
actual = future.get_value()
assert expected == actual
async def _setup(self, context: Message):
# Extract load balancing info
balancing = context.get('~balancing', [])
for balance in balancing:
if balance['id'] == 'kafka':
self.__balancing_group = balance['data']['json']['group_id']
async def _reopen(self):
await self._connector.reopen()
payload = await self._connector.read(timeout=1)
context = Message.deserialize(payload.decode())
await self._setup(context)