def _oidc_token_url(): # see _get_cached_cognito_jwt descr about the design concerns token_endpoint = env("IGL_OAUTH_TOKEN_ENDPOINT", default=None) or "" if token_endpoint: # we have token endpoint provided directly to save us requests return token_endpoint wnurl = env("IGL_OAUTH_WELLKNOWN_URL") if not wnurl: raise Exception( "HttpApiChannel uses Cognito/JWT auth but you haven't configured " "env variables correctly, and they are required to issue the token" ) wellknown_content = requests.get(wnurl) assert wellknown_content.status_code == 200 return wellknown_content.json().get("token_endpoint")
def _prepare_use_cases(self): self.uc = ProcessMessageUseCase( country=env('IGL_COUNTRY', default='AU'), bc_inbox_repo=self.bc_inbox_repo, message_lake_repo=self.message_lake_repo, object_acl_repo=self.object_acl_repo, object_retreval_repo=self.object_retrieval_repo, notifications_repo=self.notifications_repo, blockchain_outbox_repo=self.blockchain_outbox_repo, )
def _prepare_use_cases(self): self.uc = ProcessMessageUseCase( jurisdiction=env('IGL_JURISDICTION', default='AU'), bc_inbox_repo=self.bc_inbox_repo, message_lake_repo=self.message_lake_repo, object_acl_repo=self.object_acl_repo, object_retreval_repo=self.object_retrieval_repo, notifications_repo=self.notifications_repo, blockchain_outbox_repo=self.blockchain_outbox_repo, )
class Config(object): """Base configuration.""" DEBUG = env_bool('IGL_DEBUG', default=True) TESTING = env_bool('IGL_TESTING', default=True) MESSAGE_LAKE_CONN = env_s3_config('MSGAPI_MESSAGE_LAKE') BC_INBOX_CONF = env_queue_config('MSG_RX_API_BC_INBOX') PUBLISH_NOTIFICATIONS_REPO_CONN = env_queue_config('MSG_RX_API_OUTBOX_REPO') SENTRY_DSN = env("SENTRY_DSN", default=None)
def _issue_cognito_jwt(self): # see _get_cached_cognito_jwt descr about the design concerns new_style_details = self.CONFIG.get("ChannelAuthDetails") if new_style_details: OAUTH_CLIENT_ID = new_style_details["client_id"] OAUTH_CLIENT_SECRET = new_style_details["client_secret"] OAUTH_SCOPES = new_style_details["scope"] TOKEN_URL = new_style_details["token_endpoint"] else: OAUTH_CLIENT_ID = env("IGL_OAUTH_CLIENT_ID", default=None) OAUTH_CLIENT_SECRET = env("IGL_OAUTH_CLIENT_SECRET", default=None) OAUTH_SCOPES = env("IGL_OAUTH_SCOPES", default=None) TOKEN_URL = _oidc_token_url() if not OAUTH_CLIENT_ID or not OAUTH_CLIENT_SECRET: raise Exception("Improperly configured: Using Cognito/JWT auth " "without any way to retrieve the token") cognito_auth = base64.b64encode( f"{OAUTH_CLIENT_ID}:{OAUTH_CLIENT_SECRET}".encode("utf-8")).decode( "utf-8") token_resp = requests.post(TOKEN_URL, data={ "grant_type": "client_credentials", "client_id": OAUTH_CLIENT_ID, "scope": OAUTH_SCOPES, }, headers={ 'Authorization': f'Basic {cognito_auth}', }) assert token_resp.status_code == 200, token_resp.json() json_resp = token_resp.json() logger.info("Retrieved new JWT for %s; ends in %s", self.CONFIG["Name"], json_resp['expires_in']) return json_resp["access_token"], json_resp['expires_in']
from intergov.repos.notifications import NotificationsRepo from intergov.repos.object_acl import ObjectACLRepo from intergov.repos.object_retrieval import ObjectRetrievalRepo from tests.unit.domain.wire_protocols.test_generic_message import ( _generate_msg_object) MESSAGE_LAKE_REPO_CONF = env_s3_config('TEST_1') OBJECT_ACL_REPO_CONF = env_s3_config('TEST_2') BC_INBOX_REPO_CONF = env_queue_config('TEST_1') OBJECT_RETRIEVAL_REPO_CONF = env_queue_config('TEST_2') NOTIFICATIONS_REPO_CONF = env_queue_config('TEST_3') BLOCKCHAIN_OUTBOX_REPO_CONF = env_postgres_config('TEST') OUR_JRD = env("IGL_COUNTRY", default="AU") def test(): # creating testing versions of all required repos message_lake_repo = MessageLakeRepo(MESSAGE_LAKE_REPO_CONF) object_acl_repo = ObjectACLRepo(OBJECT_ACL_REPO_CONF) bc_inbox_repo = BCInboxRepo(BC_INBOX_REPO_CONF) object_retrieval_repo = ObjectRetrievalRepo(OBJECT_RETRIEVAL_REPO_CONF) notifications_repo = NotificationsRepo(NOTIFICATIONS_REPO_CONF) blockchain_outbox_repo = ApiOutboxRepo(BLOCKCHAIN_OUTBOX_REPO_CONF) def clear(): # clearing repos
def _prepare_use_case(self): self.use_case = RetrieveAndStoreForeignDocumentsUseCase( jurisdiction=Jurisdiction(env("IGL_JURISDICTION", default='AU')), **self.repos)
from intergov.conf import env MESSAGE_PATCH_API_ENDPOINT = env( 'IGL_PROC_BCH_MESSAGE_API_ENDPOINT', default='http://message_api:5101/message/{sender}:{sender_ref}') MESSAGE_RX_API_ENDPOINT = env('IGL_PROC_BCH_MESSAGE_RX_API_URL', default='http://message_rx_api:5100/messages')
import os import logging from logging.config import dictConfig import sentry_sdk from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration from intergov.conf import env_bool, env SENTRY_DSN = env('SENTRY_DSN', default=None) if SENTRY_DSN: # pragma: no cover sentry_sdk.init( dsn=SENTRY_DSN, integrations=[LoggingIntegration(), AwsLambdaIntegration()]) with sentry_sdk.configure_scope() as scope: scope.set_tag("service", "intergov") scope.set_tag("jurisdiction", env("IGL_APP_JURISDICTION", default="")) LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'verbose': { 'format': '%(asctime)-15s %(levelname)s [%(name)s] %(message)s' } }, 'handlers': {
class Config(object): """Base configuration.""" DEBUG = env_bool('IGL_DEBUG', default=True) TESTING = env_bool('IGL_TESTING', default=True) SUBSCR_REPO_CONF = env_s3_config('SUBSCR_API_REPO') SENTRY_DSN = env("SENTRY_DSN", default=None)
from events_pb2 import EventSubscription, EventFilter, EventList from client_event_pb2 import ClientEventsSubscribeRequest, ClientEventsSubscribeResponse from network_pb2 import PingRequest from validator_pb2 import Message import transaction_receipt_pb2 from intergov.domain.wire_protocols import generic_discrete as gd from intergov.conf import env from intergov.loggers import logging from intergov.processors.common.env import ( MESSAGE_RX_API_ENDPOINT, ) logger = logging.getLogger('bch_observer') DEFAULT_IGL_COUNTRY = 'AU' IGL_COUNTRY = env('IGL_COUNTRY', default=None) if IGL_COUNTRY is None: logger.warning(f'IGL_COUNTRY is undefined using {DEFAULT_IGL_COUNTRY}') IGL_COUNTRY = DEFAULT_IGL_COUNTRY CONNECT_FQDN = env('CONNECT_FQDN', default="memory-validator-default") # TODO: this is the creation of the subscription request, we probably want to create # a list of subscriptions based on some configuration (per channel). things that vary # would be the event_type and the match string (the match string is transaction processor specific) TP_NAMESPACE = hashlib.sha512( 'generic-discrete-message'.encode("utf-8")).hexdigest()[0:6] SUBSCRIPTION = EventSubscription(event_type="memory/state-delta", filters=[ EventFilter(
import logging from logging.config import dictConfig import sentry_sdk from sentry_sdk.integrations.logging import LoggingIntegration from intergov.conf import env_bool, env SENTRY_DSN = env('SENTRY_DSN', default=None) if SENTRY_DSN: # pragma: no cover sentry_logging = LoggingIntegration( level=logging.INFO, # Capture info and above as breadcrumbs event_level=logging.ERROR # Send errors as events ) sentry_sdk.init( dsn=SENTRY_DSN, integrations=[sentry_logging] ) LOGGING = { 'version': 1, 'formatters': { 'verbose': { 'format': '%(levelname)s[%(name)s] %(message)s' } }, 'handlers': { 'console': { 'level': 'DEBUG',
class LoopbackBlockchainWorker(object): REJECT_EACH = int( env('IGL_PROC_LOOPBACK_BCH_WORKER_REJECT_EACH', default=0)) MESSAGE_PATCH_API_ENDPOINT = env( 'IGL_PROC_BCH_MESSAGE_API_ENDPOINT', default='http://message_api:5101/message/{sender}:{sender_ref}') MESSAGE_RX_API_ENDPOINT = env( 'IGL_PROC_BCH_MESSAGE_RX_API_URL', default='http://message_rx_api:5100/messages') # default for all instances rejected_each_counter = 0 def __init__(self, blockchain_outbox_repo_conf=None, rejected_messages_repo_conf=None): self._prepare_blockchain_repo(blockchain_outbox_repo_conf) self._prepare_rejected_messages_repo(rejected_messages_repo_conf) def _prepare_blockchain_repo(self, config=None): blockchain_outbox_repo_conf = env_postgres_config( 'PROC_BCH_OUTBOX_REPO') if config: blockchain_outbox_repo_conf.update(config) self.blockchain_outbox_repo = ApiOutboxRepo( blockchain_outbox_repo_conf) def _prepare_rejected_messages_repo(self, config=None): rejected_messages_repo_conf = env_queue_config( 'PROC_REJECTED_MESSAGES_REPO') if config: rejected_messages_repo_conf.update(config) self.rejected_messages_repo = RejectedMessagesRepo( rejected_messages_repo_conf) def __iter__(self): logger.info("Starting the loopback blockchain worker") return self def __next__(self): try: result = self._process_next_message() except Exception as e: logger.exception(e) result = None return result def _create_message_payload(self, msg): return { 'sender': msg.sender, 'receiver': msg.receiver, 'subject': msg.subject, 'obj': msg.obj, 'sender_ref': msg.sender_ref, 'predicate': msg.predicate, } def _patch_message_status(self, msg, status): url = self.MESSAGE_PATCH_API_ENDPOINT.format(sender=msg.sender, sender_ref=msg.sender_ref) logger.info("Patching message status to %s by url %s", status, url) resp = requests.patch(url, json={'status': status}) if resp.status_code not in (200, 201): raise RuntimeError( "Unable to patch message status, resp code {} body {}".format( resp.status_code, resp.content)) def _post_message_rx(self, payload): resp = requests.post(self.MESSAGE_RX_API_ENDPOINT, json=payload) if resp.status_code not in (200, 201): raise RuntimeError( "Unable to post message, code {}, resp {}".format( resp.status_code, resp.content)) return resp def _process_next_message(self): msg = self.blockchain_outbox_repo.get_next_pending_message() if msg is None: return None # ensure message got to the repo as expected, won't be a problem for prod # (remove it there) # time.sleep(3) logger.info("[Loopback] Processing message %s (%s) to the blockchain", msg, msg.id) # dummy reject controllable from env variable "IGL_PROC_LOOPBACK_BCH_WORKER_REJECT_EACH" if self.REJECT_EACH > 0: self.rejected_each_counter += 1 if self.rejected_each_counter == self.REJECT_EACH: logger.info( "Rejecting the message (because we reject one of %s, and this is %s)", self.REJECT_EACH, self.rejected_each_counter) self.rejected_each_counter = 0 self.blockchain_outbox_repo.patch(msg.id, {'status': 'rejected'}) self.rejected_messages_repo.post(msg) return True self.blockchain_outbox_repo.patch(msg.id, {'status': 'sending'}) # time.sleep(1) # for the realistic debug # please note that logically we don't forward message # but send a message about the message which is equal to the message # so we don't re-send the same object, # but we send another object which (wow!) has all fields the same # but it's not the same, because we logically got it from the # blockchain, where it was encrypted/compressed and abused by other methos message_to_be_sent = self._create_message_payload(msg) # it's a silly situation when importer app in the same intergov setup # gets both messages, but in the real situation remote importer_app # will get only the blockchain one. logger.info("message_to_be_sent %s", message_to_be_sent) # we behave like this message has been received from the remote party self._post_message_rx(message_to_be_sent) self.blockchain_outbox_repo.patch(msg.id, {'status': 'accepted'}) # now we update the original message status self._patch_message_status(msg, 'accepted') logger.info("[Loopback] The message has been sent to blockchain and " "immediately retrieved from it as a received") return True
from intergov.repos.notifications import NotificationsRepo from intergov.repos.object_acl import ObjectACLRepo from intergov.repos.object_retrieval import ObjectRetrievalRepo from tests.unit.domain.wire_protocols.test_generic_message import ( _generate_msg_object) MESSAGE_LAKE_REPO_CONF = env_s3_config('TEST_1') OBJECT_ACL_REPO_CONF = env_s3_config('TEST_2') BC_INBOX_REPO_CONF = env_queue_config('TEST_1') OBJECT_RETRIEVAL_REPO_CONF = env_queue_config('TEST_2') NOTIFICATIONS_REPO_CONF = env_queue_config('TEST_3') BLOCKCHAIN_OUTBOX_REPO_CONF = env_postgres_config('TEST') OUR_JRD = env("IGL_JURISDICTION", default="AU") def test(): # creating testing versions of all required repos message_lake_repo = MessageLakeRepo(MESSAGE_LAKE_REPO_CONF) object_acl_repo = ObjectACLRepo(OBJECT_ACL_REPO_CONF) bc_inbox_repo = BCInboxRepo(BC_INBOX_REPO_CONF) object_retrieval_repo = ObjectRetrievalRepo(OBJECT_RETRIEVAL_REPO_CONF) notifications_repo = NotificationsRepo(NOTIFICATIONS_REPO_CONF) blockchain_outbox_repo = ApiOutboxRepo(BLOCKCHAIN_OUTBOX_REPO_CONF) def clear(): # clearing repos
from intergov.apis.common.errors.api.message import ( MessageDataEmptyError, MessageDeserializationError, MessageValidationError, UnableWriteToInboxError ) from .conf import Config from .exceptions import ( MessageNotFoundError, UnexpectedMessageStatusError ) blueprint = Blueprint('messages', __name__) logger = logging.getLogger(__name__) IGL_COUNTRY = env('IGL_COUNTRY', default=None) @statsd_timer("api.message.endpoint.message_retrieve") @blueprint.route('/message/<reference>', methods=['GET']) def message_retrieve(reference): # TODO: auth repo = MessageLakeRepo(Config.MESSAGE_LAKE_CONN) use_case = GetMessageBySenderRefUseCase(repo) try: message = use_case.execute(reference) except Exception as e: if e.__class__.__name__ == 'NoSuchKey': message = None
import os import logging from logging.config import dictConfig import sentry_sdk from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration from intergov.conf import env_bool, env SENTRY_DSN = env('SENTRY_DSN', default=None) if SENTRY_DSN: # pragma: no cover sentry_sdk.init( dsn=SENTRY_DSN, integrations=[LoggingIntegration(), AwsLambdaIntegration()]) with sentry_sdk.configure_scope() as scope: scope.set_tag("service", "intergov") scope.set_tag("country", env("ICL_APP_COUNTRY", default="")) LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'verbose': { 'format': '%(asctime)-15s %(levelname)s [%(name)s] %(message)s' } }, 'handlers': {
def _prepare_use_case(self): self.use_case = RetrieveAndStoreForeignDocumentsUseCase( country=Country(env("IGL_COUNTRY", default='AU')), **self.repos)
EnqueueMessageUseCase, PatchMessageMetadataUseCase, ) from intergov.use_cases.common.errors import (UseCaseError) from intergov.apis.common.utils import routing from intergov.apis.common.errors import (InternalServerError) from intergov.apis.common.errors.api.message import ( MessageDataEmptyError, MessageDeserializationError, MessageValidationError, UnableWriteToInboxError) from .conf import Config from .exceptions import (MessageNotFoundError, UnexpectedMessageStatusError) blueprint = Blueprint('messages', __name__) logger = logging.getLogger(__name__) IGL_JURISDICTION = env('IGL_JURISDICTION', default=None) class MessageSchema(Schema): sender = fields.String(required=True) receiver = fields.String(required=True) subject = fields.String(required=True) obj = fields.String(required=True) predicate = fields.String(required=True) channel_id = fields.String() channel_txn_id = fields.String() status = fields.String() sender_ref = fields.String() @statsd_timer("api.message.endpoint.message_retrieve")