def _prepare_message_updates_repo(self, conf):
# This repo used to talk to the message updater microservice,
# which just changes statuses in the message lake
repo_conf = env_queue_config('MCHR_MESSAGE_UPDATES_REPO',
use_default=False)
if not repo_conf:
repo_conf = env_queue_config('BCH_MESSAGE_UPDATES')
if conf:
repo_conf.update(conf)
self.message_updates_repo = MessageUpdatesRepo(repo_conf)
class Config(object):
"""Base configuration."""
DEBUG = env_bool('IGL_DEBUG', default=True)
TESTING = env_bool('IGL_TESTING', default=True)
MESSAGE_LAKE_CONN = env_s3_config('MSGAPI_MESSAGE_LAKE')
BC_INBOX_CONF = env_queue_config('MSG_RX_API_BC_INBOX')
PUBLISH_NOTIFICATIONS_REPO_CONN = env_queue_config(
'MSG_RX_API_OUTBOX_REPO')
def _prepare_channel_pending_message_repo(self, conf):
channel_pending_message_repo_conf = env_queue_config(
'PROC_BCH_CHANNEL_PENDING_MESSAGE')
if conf:
channel_pending_message_repo_conf.update(conf)
self.channel_pending_message_repo = ChannelPendingMessageRepo(
channel_pending_message_repo_conf)
def _prepare_delivery_outbox_repo(self, conf):
delivery_outbox_repo_conf = env_queue_config(
'PROC_DELIVERY_OUTBOX_REPO')
if conf:
delivery_outbox_repo_conf.update(conf)
self.delivery_outbox_repo = DeliveryOutboxRepo(
delivery_outbox_repo_conf)
def _prepare_rejected_messages_repo(self, config=None):
rejected_messages_repo_conf = env_queue_config(
'PROC_REJECTED_MESSAGES_REPO')
if config:
rejected_messages_repo_conf.update(config)
self.rejected_messages_repo = RejectedMessagesRepo(
rejected_messages_repo_conf)
def _prepare_repos_confs(self):
self.repo_conf = {
'object_lake': env_s3_config('PROC_OBJ_SPIDER_OBJ_LAKE'),
'object_retrieval':
env_queue_config('PROC_OBJ_SPIDER_OBJ_RETRIEVAL'),
'object_acl': env_s3_config('PROC_OBJ_SPIDER_OBJ_ACL'),
}
def test():
s3 = env_s3_config('TEST')
queue = env_queue_config('TEST')
postgres = env_postgres_config('TEST')
assert s3['bucket'] == 'test-bucket-1'
assert queue['queue_name'] == 'test-queue-1'
assert postgres['dbname'] == 'postgres_test'
s3 = env_s3_config('TEST_1')
queue = env_queue_config('TEST_1')
assert s3['bucket'] == 'test-bucket-1'
assert queue['queue_name'] == 'test-queue-1'
s3 = env_s3_config('TEST_2')
queue = env_queue_config('TEST_2')
assert s3['bucket'] == 'test-bucket-2'
assert queue['queue_name'] == 'test-queue-2'
def test_complex():
assert igl_value('default_sqs', 'qname') == 'IGL_DEFAULT_SQS_QNAME'
assert igl_value('default_s3', 'bucket') == 'IGL_DEFAULT_S3_BUCKET'
assert igl_value('default_postgres',
'dbname') == 'IGL_DEFAULT_POSTGRES_DBNAME'
for key, value in TEST_ENV.items():
assert os.environ.get(key) == value
default_s3_conf = conf.env_s3_config(TEST_NON_EXISTING_CONF_NAME)
default_sqs_conf = conf.env_queue_config(TEST_NON_EXISTING_CONF_NAME)
default_postgres_conf = conf.env_postgres_config(
TEST_NON_EXISTING_CONF_NAME)
assert to_str_dict(default_s3_conf) == TEST_S3_DEFAULT_CONF
assert to_str_dict(default_sqs_conf) == TEST_SQS_DEFAULT_CONF
assert to_str_dict(default_postgres_conf) == TEST_POSTGRES_DEFAULT_CONF
# bool values
assert isinstance(default_s3_conf['use_ssl'], bool)
assert isinstance(default_sqs_conf['use_ssl'], bool)
# nullable values
del os.environ['IGL_DEFAULT_SQS_QNAME']
del os.environ['IGL_DEFAULT_S3_BUCKET']
del os.environ['IGL_DEFAULT_POSTGRES_DBNAME']
default_s3_conf = conf.env_s3_config(TEST_NON_EXISTING_CONF_NAME)
default_sqs_conf = conf.env_queue_config(TEST_NON_EXISTING_CONF_NAME)
default_postgres_conf = conf.env_postgres_config(
TEST_NON_EXISTING_CONF_NAME)
assert default_s3_conf['bucket'] is None
assert default_sqs_conf['queue_name'] is None
assert default_postgres_conf['dbname'] is None
# testing fully custom configs
custom_s3_conf = conf.env_s3_config(CUSTOM_S3_CONF_NAME)
custom_sqs_conf = conf.env_queue_config(CUSTOM_SQS_CONF_NAME)
custom_postgres_conf = conf.env_postgres_config(CUSTOM_POSTGRES_CONF_NAME)
assert custom_s3_conf == CUSTOM_S3_CONF, os.environ
assert custom_sqs_conf == CUSTOM_SQS_CONF, os.environ
assert custom_postgres_conf == CUSTOM_POSTGRES_CONF, os.environ
def _prepare_repos_confs(self,
rejected_message_repo_conf=None,
message_lake_repo_conf=None):
self.rejected_messages_repo_conf = env_queue_config(
'PROC_REJECTED_MESSAGES_REPO')
self.message_lake_repo_conf = env_s3_config('PROC_MESSAGE_LAKE_REPO')
if message_lake_repo_conf:
self.message_lake_repo_conf.update(message_lake_repo_conf)
if rejected_message_repo_conf:
self.rejected_messages_repo_conf.update(rejected_message_repo_conf)
def _prepare_object_retrieval_repo(self, conf):
object_retrieval_repo_conf = env_queue_config('PROC_OBJ_RETR_REPO')
if conf:
object_retrieval_repo_conf.update(conf)
self.object_retrieval_repo = ObjectRetrievalRepo(object_retrieval_repo_conf)
class Config(object):
"""Base configuration."""
DEBUG = env_bool('IGL_DEBUG', default=True)
TESTING = env_bool('IGL_TESTING', default=True)
BC_INBOX_CONF = env_queue_config('MSG_RX_API_BC_INBOX')
Message)
from tests.unit.domain.wire_protocols.test_generic_message import (
_generate_msg_dict)
MESSAGES_DATA = [
('AU', 'aaaa'),
('US', 'bbbb'),
('DE', 'cccc'),
('FR', 'dddd'),
('GB', 'eeee'),
('UA', 'ffff'),
]
# required to not share repos with running workers
MESSAGE_LAKE_REPO_CONF = env_s3_config('TEST')
REJECTED_MESSAGES_REPO_CONF = env_queue_config('TEST')
def test(docker_setup):
message_lake_repo = MessageLakeRepo(MESSAGE_LAKE_REPO_CONF)
rejected_message_repo = RejectedMessagesRepo(REJECTED_MESSAGES_REPO_CONF)
# ensuring that repos are empty
message_lake_repo._unsafe_clear_for_test()
rejected_message_repo._unsafe_clear_for_test()
updater = RejectedStatusUpdater(
rejected_message_repo_conf=REJECTED_MESSAGES_REPO_CONF,
message_lake_repo_conf=MESSAGE_LAKE_REPO_CONF)
from intergov.conf import env_s3_config, env_queue_config
from intergov.repos.delivery_outbox import DeliveryOutboxRepo
from intergov.repos.notifications import NotificationsRepo
from intergov.repos.subscriptions import SubscriptionsRepo
from intergov.processors.callbacks_spreader import CallbacksSpreaderProcessor
from tests.unit.domain.wire_protocols.test_generic_message import (
_generate_msg_object)
DELIVERY_OUTBOX_REPO_CONF = env_queue_config('TEST_1')
NOTIFICATIONS_REPO_CONF = env_queue_config('TEST_2')
SUBSCRIPTIONS_REPO_CONF = env_s3_config('TEST')
CALLBACK_URL = "http://dummy-test-helper-server:5001/response/200/{}"
DEFAULT_EXPIRATION = 3600 * 2
# predicates dont have common prefixes
SUBSCRIPTIONS = {
'aaa.bbb.ccc.ddd': 2,
'eee.fff.jjj': 3,
'hhh.iii': 4,
'ggg': 1
}
SUBSCRIPTIONS_WITH_COMMON_PREFIXES = {
'ooo': {
'ooo.aaa.bbb': 2,
'ooo.aaa': 3,
},
'zzz': {
'zzz.aaa.bbb': 5,
from intergov.conf import env_s3_config, env_queue_config, env_postgres_config, env
from intergov.processors.message_processor import InboundMessageProcessor
from intergov.repos.api_outbox import ApiOutboxRepo
from intergov.repos.bc_inbox.elasticmq.elasticmqrepo import BCInboxRepo
from intergov.repos.message_lake import MessageLakeRepo
from intergov.repos.notifications import NotificationsRepo
from intergov.repos.object_acl import ObjectACLRepo
from intergov.repos.object_retrieval import ObjectRetrievalRepo
from tests.unit.domain.wire_protocols.test_generic_message import (
_generate_msg_object)
MESSAGE_LAKE_REPO_CONF = env_s3_config('TEST_1')
OBJECT_ACL_REPO_CONF = env_s3_config('TEST_2')
BC_INBOX_REPO_CONF = env_queue_config('TEST_1')
OBJECT_RETRIEVAL_REPO_CONF = env_queue_config('TEST_2')
NOTIFICATIONS_REPO_CONF = env_queue_config('TEST_3')
BLOCKCHAIN_OUTBOX_REPO_CONF = env_postgres_config('TEST')
OUR_JRD = env("IGL_COUNTRY", default="AU")
def test():
# creating testing versions of all required repos
message_lake_repo = MessageLakeRepo(MESSAGE_LAKE_REPO_CONF)
object_acl_repo = ObjectACLRepo(OBJECT_ACL_REPO_CONF)
bc_inbox_repo = BCInboxRepo(BC_INBOX_REPO_CONF)
object_retrieval_repo = ObjectRetrievalRepo(OBJECT_RETRIEVAL_REPO_CONF)
notifications_repo = NotificationsRepo(NOTIFICATIONS_REPO_CONF)
def _prepare_message_updates_repo(self, conf):
message_updates_repo_conf = env_queue_config('BCH_MESSAGE_UPDATES')
if conf:
message_updates_repo_conf.update(conf)
self.message_updates_repo = MessageUpdatesRepo(
message_updates_repo_conf)
def _prepare_notifications_repo(self, conf):
notifications_repo_conf = env_queue_config('PROC_OBJ_OUTBOX_REPO')
if conf:
notifications_repo_conf.update(conf)
self.notifications_repo = NotificationsRepo(notifications_repo_conf)
def _prepare_bc_inbox_repo(self, conf):
bc_inbox_repo_conf = env_queue_config('PROC_BC_INBOX')
if conf:
bc_inbox_repo_conf.update(conf)
self.bc_inbox_repo = BCInboxRepo(bc_inbox_repo_conf)
from unittest import mock
from intergov.conf import env_queue_config
# from intergov.domain.wire_protocols.generic_discrete import (
# Message
# )
from intergov.processors.callback_deliver import CallbacksDeliveryProcessor
from intergov.use_cases import DeliverCallbackUseCase
from intergov.repos.delivery_outbox import DeliveryOutboxRepo
from tests.unit.domain.wire_protocols.test_generic_message import (
_generate_msg_dict
)
DELIVERY_OUTBOX_REPO_CONF = env_queue_config('TEST')
POST_SUCCESS_URL = "http://dummy-test-helper-server:5001/response/200/success"
POST_ERROR_URL = "http://dummy-test-helper-server:5001/response/500/internal-error"
def _generate_job(url, payload=None):
if not payload:
payload = _generate_msg_dict()
return {
's': url,
'payload': payload
}
def _test_retries(processor, max_attempts):
retries = 0
