def _oidc_token_url():
# see _get_cached_cognito_jwt descr about the design concerns
token_endpoint = env("IGL_OAUTH_TOKEN_ENDPOINT", default=None) or ""
if token_endpoint:
# we have token endpoint provided directly to save us requests
return token_endpoint
wnurl = env("IGL_OAUTH_WELLKNOWN_URL")
if not wnurl:
raise Exception(
"HttpApiChannel uses Cognito/JWT auth but you haven't configured "
"env variables correctly, and they are required to issue the token"
)
wellknown_content = requests.get(wnurl)
assert wellknown_content.status_code == 200
return wellknown_content.json().get("token_endpoint")
def _prepare_use_cases(self):
self.uc = ProcessMessageUseCase(
country=env('IGL_COUNTRY', default='AU'),
bc_inbox_repo=self.bc_inbox_repo,
message_lake_repo=self.message_lake_repo,
object_acl_repo=self.object_acl_repo,
object_retreval_repo=self.object_retrieval_repo,
notifications_repo=self.notifications_repo,
blockchain_outbox_repo=self.blockchain_outbox_repo,
)
def _prepare_use_cases(self):
self.uc = ProcessMessageUseCase(
jurisdiction=env('IGL_JURISDICTION', default='AU'),
bc_inbox_repo=self.bc_inbox_repo,
message_lake_repo=self.message_lake_repo,
object_acl_repo=self.object_acl_repo,
object_retreval_repo=self.object_retrieval_repo,
notifications_repo=self.notifications_repo,
blockchain_outbox_repo=self.blockchain_outbox_repo,
)
class Config(object):
"""Base configuration."""
DEBUG = env_bool('IGL_DEBUG', default=True)
TESTING = env_bool('IGL_TESTING', default=True)
MESSAGE_LAKE_CONN = env_s3_config('MSGAPI_MESSAGE_LAKE')
BC_INBOX_CONF = env_queue_config('MSG_RX_API_BC_INBOX')
PUBLISH_NOTIFICATIONS_REPO_CONN = env_queue_config('MSG_RX_API_OUTBOX_REPO')
SENTRY_DSN = env("SENTRY_DSN", default=None)
def _issue_cognito_jwt(self):
# see _get_cached_cognito_jwt descr about the design concerns
new_style_details = self.CONFIG.get("ChannelAuthDetails")
if new_style_details:
OAUTH_CLIENT_ID = new_style_details["client_id"]
OAUTH_CLIENT_SECRET = new_style_details["client_secret"]
OAUTH_SCOPES = new_style_details["scope"]
TOKEN_URL = new_style_details["token_endpoint"]
else:
OAUTH_CLIENT_ID = env("IGL_OAUTH_CLIENT_ID", default=None)
OAUTH_CLIENT_SECRET = env("IGL_OAUTH_CLIENT_SECRET", default=None)
OAUTH_SCOPES = env("IGL_OAUTH_SCOPES", default=None)
TOKEN_URL = _oidc_token_url()
if not OAUTH_CLIENT_ID or not OAUTH_CLIENT_SECRET:
raise Exception("Improperly configured: Using Cognito/JWT auth "
"without any way to retrieve the token")
cognito_auth = base64.b64encode(
f"{OAUTH_CLIENT_ID}:{OAUTH_CLIENT_SECRET}".encode("utf-8")).decode(
"utf-8")
token_resp = requests.post(TOKEN_URL,
data={
"grant_type": "client_credentials",
"client_id": OAUTH_CLIENT_ID,
"scope": OAUTH_SCOPES,
},
headers={
'Authorization':
f'Basic {cognito_auth}',
})
assert token_resp.status_code == 200, token_resp.json()
json_resp = token_resp.json()
logger.info("Retrieved new JWT for %s; ends in %s",
self.CONFIG["Name"], json_resp['expires_in'])
return json_resp["access_token"], json_resp['expires_in']
from intergov.repos.notifications import NotificationsRepo
from intergov.repos.object_acl import ObjectACLRepo
from intergov.repos.object_retrieval import ObjectRetrievalRepo
from tests.unit.domain.wire_protocols.test_generic_message import (
_generate_msg_object)
MESSAGE_LAKE_REPO_CONF = env_s3_config('TEST_1')
OBJECT_ACL_REPO_CONF = env_s3_config('TEST_2')
BC_INBOX_REPO_CONF = env_queue_config('TEST_1')
OBJECT_RETRIEVAL_REPO_CONF = env_queue_config('TEST_2')
NOTIFICATIONS_REPO_CONF = env_queue_config('TEST_3')
BLOCKCHAIN_OUTBOX_REPO_CONF = env_postgres_config('TEST')
OUR_JRD = env("IGL_COUNTRY", default="AU")
def test():
# creating testing versions of all required repos
message_lake_repo = MessageLakeRepo(MESSAGE_LAKE_REPO_CONF)
object_acl_repo = ObjectACLRepo(OBJECT_ACL_REPO_CONF)
bc_inbox_repo = BCInboxRepo(BC_INBOX_REPO_CONF)
object_retrieval_repo = ObjectRetrievalRepo(OBJECT_RETRIEVAL_REPO_CONF)
notifications_repo = NotificationsRepo(NOTIFICATIONS_REPO_CONF)
blockchain_outbox_repo = ApiOutboxRepo(BLOCKCHAIN_OUTBOX_REPO_CONF)
def clear():
# clearing repos
def _prepare_use_case(self):
self.use_case = RetrieveAndStoreForeignDocumentsUseCase(
jurisdiction=Jurisdiction(env("IGL_JURISDICTION", default='AU')),
**self.repos)
from intergov.conf import env
MESSAGE_PATCH_API_ENDPOINT = env(
'IGL_PROC_BCH_MESSAGE_API_ENDPOINT',
default='http://message_api:5101/message/{sender}:{sender_ref}')
MESSAGE_RX_API_ENDPOINT = env('IGL_PROC_BCH_MESSAGE_RX_API_URL',
default='http://message_rx_api:5100/messages')
import os
import logging
from logging.config import dictConfig
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
from intergov.conf import env_bool, env
SENTRY_DSN = env('SENTRY_DSN', default=None)
if SENTRY_DSN: # pragma: no cover
sentry_sdk.init(
dsn=SENTRY_DSN,
integrations=[LoggingIntegration(),
AwsLambdaIntegration()])
with sentry_sdk.configure_scope() as scope:
scope.set_tag("service", "intergov")
scope.set_tag("jurisdiction", env("IGL_APP_JURISDICTION", default=""))
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(asctime)-15s %(levelname)s [%(name)s] %(message)s'
}
},
'handlers': {
class Config(object):
"""Base configuration."""
DEBUG = env_bool('IGL_DEBUG', default=True)
TESTING = env_bool('IGL_TESTING', default=True)
SUBSCR_REPO_CONF = env_s3_config('SUBSCR_API_REPO')
SENTRY_DSN = env("SENTRY_DSN", default=None)
from events_pb2 import EventSubscription, EventFilter, EventList
from client_event_pb2 import ClientEventsSubscribeRequest, ClientEventsSubscribeResponse
from network_pb2 import PingRequest
from validator_pb2 import Message
import transaction_receipt_pb2
from intergov.domain.wire_protocols import generic_discrete as gd
from intergov.conf import env
from intergov.loggers import logging
from intergov.processors.common.env import (
MESSAGE_RX_API_ENDPOINT, )
logger = logging.getLogger('bch_observer')
DEFAULT_IGL_COUNTRY = 'AU'
IGL_COUNTRY = env('IGL_COUNTRY', default=None)
if IGL_COUNTRY is None:
logger.warning(f'IGL_COUNTRY is undefined using {DEFAULT_IGL_COUNTRY}')
IGL_COUNTRY = DEFAULT_IGL_COUNTRY
CONNECT_FQDN = env('CONNECT_FQDN', default="memory-validator-default")
# TODO: this is the creation of the subscription request, we probably want to create
# a list of subscriptions based on some configuration (per channel). things that vary
# would be the event_type and the match string (the match string is transaction processor specific)
TP_NAMESPACE = hashlib.sha512(
'generic-discrete-message'.encode("utf-8")).hexdigest()[0:6]
SUBSCRIPTION = EventSubscription(event_type="memory/state-delta",
filters=[
EventFilter(
import logging
from logging.config import dictConfig
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
from intergov.conf import env_bool, env
SENTRY_DSN = env('SENTRY_DSN', default=None)
if SENTRY_DSN: # pragma: no cover
sentry_logging = LoggingIntegration(
level=logging.INFO, # Capture info and above as breadcrumbs
event_level=logging.ERROR # Send errors as events
)
sentry_sdk.init(
dsn=SENTRY_DSN,
integrations=[sentry_logging]
)
LOGGING = {
'version': 1,
'formatters': {
'verbose': {
'format': '%(levelname)s[%(name)s] %(message)s'
}
},
'handlers': {
'console': {
'level': 'DEBUG',
class LoopbackBlockchainWorker(object):
REJECT_EACH = int(
env('IGL_PROC_LOOPBACK_BCH_WORKER_REJECT_EACH', default=0))
MESSAGE_PATCH_API_ENDPOINT = env(
'IGL_PROC_BCH_MESSAGE_API_ENDPOINT',
default='http://message_api:5101/message/{sender}:{sender_ref}')
MESSAGE_RX_API_ENDPOINT = env(
'IGL_PROC_BCH_MESSAGE_RX_API_URL',
default='http://message_rx_api:5100/messages')
# default for all instances
rejected_each_counter = 0
def __init__(self,
blockchain_outbox_repo_conf=None,
rejected_messages_repo_conf=None):
self._prepare_blockchain_repo(blockchain_outbox_repo_conf)
self._prepare_rejected_messages_repo(rejected_messages_repo_conf)
def _prepare_blockchain_repo(self, config=None):
blockchain_outbox_repo_conf = env_postgres_config(
'PROC_BCH_OUTBOX_REPO')
if config:
blockchain_outbox_repo_conf.update(config)
self.blockchain_outbox_repo = ApiOutboxRepo(
blockchain_outbox_repo_conf)
def _prepare_rejected_messages_repo(self, config=None):
rejected_messages_repo_conf = env_queue_config(
'PROC_REJECTED_MESSAGES_REPO')
if config:
rejected_messages_repo_conf.update(config)
self.rejected_messages_repo = RejectedMessagesRepo(
rejected_messages_repo_conf)
def __iter__(self):
logger.info("Starting the loopback blockchain worker")
return self
def __next__(self):
try:
result = self._process_next_message()
except Exception as e:
logger.exception(e)
result = None
return result
def _create_message_payload(self, msg):
return {
'sender': msg.sender,
'receiver': msg.receiver,
'subject': msg.subject,
'obj': msg.obj,
'sender_ref': msg.sender_ref,
'predicate': msg.predicate,
}
def _patch_message_status(self, msg, status):
url = self.MESSAGE_PATCH_API_ENDPOINT.format(sender=msg.sender,
sender_ref=msg.sender_ref)
logger.info("Patching message status to %s by url %s", status, url)
resp = requests.patch(url, json={'status': status})
if resp.status_code not in (200, 201):
raise RuntimeError(
"Unable to patch message status, resp code {} body {}".format(
resp.status_code, resp.content))
def _post_message_rx(self, payload):
resp = requests.post(self.MESSAGE_RX_API_ENDPOINT, json=payload)
if resp.status_code not in (200, 201):
raise RuntimeError(
"Unable to post message, code {}, resp {}".format(
resp.status_code, resp.content))
return resp
def _process_next_message(self):
msg = self.blockchain_outbox_repo.get_next_pending_message()
if msg is None:
return None
# ensure message got to the repo as expected, won't be a problem for prod
# (remove it there)
# time.sleep(3)
logger.info("[Loopback] Processing message %s (%s) to the blockchain",
msg, msg.id)
# dummy reject controllable from env variable "IGL_PROC_LOOPBACK_BCH_WORKER_REJECT_EACH"
if self.REJECT_EACH > 0:
self.rejected_each_counter += 1
if self.rejected_each_counter == self.REJECT_EACH:
logger.info(
"Rejecting the message (because we reject one of %s, and this is %s)",
self.REJECT_EACH, self.rejected_each_counter)
self.rejected_each_counter = 0
self.blockchain_outbox_repo.patch(msg.id,
{'status': 'rejected'})
self.rejected_messages_repo.post(msg)
return True
self.blockchain_outbox_repo.patch(msg.id, {'status': 'sending'})
# time.sleep(1) # for the realistic debug
# please note that logically we don't forward message
# but send a message about the message which is equal to the message
# so we don't re-send the same object,
# but we send another object which (wow!) has all fields the same
# but it's not the same, because we logically got it from the
# blockchain, where it was encrypted/compressed and abused by other methos
message_to_be_sent = self._create_message_payload(msg)
# it's a silly situation when importer app in the same intergov setup
# gets both messages, but in the real situation remote importer_app
# will get only the blockchain one.
logger.info("message_to_be_sent %s", message_to_be_sent)
# we behave like this message has been received from the remote party
self._post_message_rx(message_to_be_sent)
self.blockchain_outbox_repo.patch(msg.id, {'status': 'accepted'})
# now we update the original message status
self._patch_message_status(msg, 'accepted')
logger.info("[Loopback] The message has been sent to blockchain and "
"immediately retrieved from it as a received")
return True
from intergov.repos.notifications import NotificationsRepo
from intergov.repos.object_acl import ObjectACLRepo
from intergov.repos.object_retrieval import ObjectRetrievalRepo
from tests.unit.domain.wire_protocols.test_generic_message import (
_generate_msg_object)
MESSAGE_LAKE_REPO_CONF = env_s3_config('TEST_1')
OBJECT_ACL_REPO_CONF = env_s3_config('TEST_2')
BC_INBOX_REPO_CONF = env_queue_config('TEST_1')
OBJECT_RETRIEVAL_REPO_CONF = env_queue_config('TEST_2')
NOTIFICATIONS_REPO_CONF = env_queue_config('TEST_3')
BLOCKCHAIN_OUTBOX_REPO_CONF = env_postgres_config('TEST')
OUR_JRD = env("IGL_JURISDICTION", default="AU")
def test():
# creating testing versions of all required repos
message_lake_repo = MessageLakeRepo(MESSAGE_LAKE_REPO_CONF)
object_acl_repo = ObjectACLRepo(OBJECT_ACL_REPO_CONF)
bc_inbox_repo = BCInboxRepo(BC_INBOX_REPO_CONF)
object_retrieval_repo = ObjectRetrievalRepo(OBJECT_RETRIEVAL_REPO_CONF)
notifications_repo = NotificationsRepo(NOTIFICATIONS_REPO_CONF)
blockchain_outbox_repo = ApiOutboxRepo(BLOCKCHAIN_OUTBOX_REPO_CONF)
def clear():
# clearing repos
from intergov.apis.common.errors.api.message import (
MessageDataEmptyError,
MessageDeserializationError,
MessageValidationError,
UnableWriteToInboxError
)
from .conf import Config
from .exceptions import (
MessageNotFoundError,
UnexpectedMessageStatusError
)
blueprint = Blueprint('messages', __name__)
logger = logging.getLogger(__name__)
IGL_COUNTRY = env('IGL_COUNTRY', default=None)
@statsd_timer("api.message.endpoint.message_retrieve")
@blueprint.route('/message/<reference>', methods=['GET'])
def message_retrieve(reference):
# TODO: auth
repo = MessageLakeRepo(Config.MESSAGE_LAKE_CONN)
use_case = GetMessageBySenderRefUseCase(repo)
try:
message = use_case.execute(reference)
except Exception as e:
if e.__class__.__name__ == 'NoSuchKey':
message = None
import os
import logging
from logging.config import dictConfig
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
from intergov.conf import env_bool, env
SENTRY_DSN = env('SENTRY_DSN', default=None)
if SENTRY_DSN: # pragma: no cover
sentry_sdk.init(
dsn=SENTRY_DSN,
integrations=[LoggingIntegration(),
AwsLambdaIntegration()])
with sentry_sdk.configure_scope() as scope:
scope.set_tag("service", "intergov")
scope.set_tag("country", env("ICL_APP_COUNTRY", default=""))
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(asctime)-15s %(levelname)s [%(name)s] %(message)s'
}
},
'handlers': {
def _prepare_use_case(self):
self.use_case = RetrieveAndStoreForeignDocumentsUseCase(
country=Country(env("IGL_COUNTRY", default='AU')), **self.repos)
EnqueueMessageUseCase,
PatchMessageMetadataUseCase,
)
from intergov.use_cases.common.errors import (UseCaseError)
from intergov.apis.common.utils import routing
from intergov.apis.common.errors import (InternalServerError)
from intergov.apis.common.errors.api.message import (
MessageDataEmptyError, MessageDeserializationError, MessageValidationError,
UnableWriteToInboxError)
from .conf import Config
from .exceptions import (MessageNotFoundError, UnexpectedMessageStatusError)
blueprint = Blueprint('messages', __name__)
logger = logging.getLogger(__name__)
IGL_JURISDICTION = env('IGL_JURISDICTION', default=None)
class MessageSchema(Schema):
sender = fields.String(required=True)
receiver = fields.String(required=True)
subject = fields.String(required=True)
obj = fields.String(required=True)
predicate = fields.String(required=True)
channel_id = fields.String()
channel_txn_id = fields.String()
status = fields.String()
sender_ref = fields.String()
@statsd_timer("api.message.endpoint.message_retrieve")