def reprocess(tx_id):
logger.debug('Reprocess function')
publisher = QueuePublisher(
settings.RABBIT_URLS,
'sdx-survey-notification-durable'
)
publisher.publish_message(tx_id, headers={'tx_id': tx_id})
def send_payload(payload, tx_id, no_of_submissions=1):
logger.debug(" [x] Sending encrypted Payload")
publisher = QueuePublisher(settings.RABBIT_URLS, settings.RABBIT_QUEUE)
for _ in range(no_of_submissions):
publisher.publish_message(payload, headers={'tx_id': tx_id})
logger.debug(" [x] Sent Payload to rabbitmq!")
def send_payload(payload, tx_id, no_of_submissions=1):
logger.info("Sending encrypted Payload", tx_id=tx_id)
publisher = QueuePublisher(settings.RABBIT_URLS, settings.RABBIT_QUEUE)
for _ in range(no_of_submissions):
publisher.publish_message(payload, headers={'tx_id': tx_id})
logger.info("Sent Payload to rabbitmq", tx_id=tx_id)
def __init__(self, logger=logger):
self.logger = logger
self.tx_id = None
self.rrm_publisher = PrivatePublisher(
settings.RABBIT_URLS, settings.RABBIT_RRM_RECEIPT_QUEUE)
self.notifications = QueuePublisher(settings.RABBIT_URLS,
settings.RABBIT_SURVEY_QUEUE)
self.dap = QueuePublisher(settings.RABBIT_URLS,
settings.RABBIT_DAP_QUEUE)
def send_payload(payload, tx_id):
logger.info("About to send SEFT payload", tx_id=tx_id)
publisher = QueuePublisher(settings.RABBIT_URLS, settings.SEFT_CONSUMER_RABBIT_QUEUE)
try:
publisher.publish_message(payload, headers={'tx_id': tx_id})
except PublishMessageError:
logger.exception("Failed to put SEFT payload on queue", tx_id=tx_id)
logger.info("SEFT payload successfully placed onto rabbit queue", tx_id=tx_id)
def test_queue_connect_loops_correctly(self):
this_publisher = QueuePublisher(loop_urls, queue_name)
self.assertEqual(this_publisher._urls, loop_urls)
self.assertEqual(this_publisher._queue, queue_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertTrue(this_publisher._connect())
def send_payload(payload, tx_id):
logger.info("About to send SEFT payload", tx_id=tx_id)
publisher = QueuePublisher(settings.RABBIT_URLS,
settings.SEFT_CONSUMER_RABBIT_QUEUE)
try:
publisher.publish_message(payload, headers={'tx_id': tx_id})
except PublishMessageError:
logger.exception("Failed to put SEFT payload on queue", tx_id=tx_id)
logger.info("SEFT payload successfully placed onto rabbit queue",
tx_id=tx_id)
def __init__(self):
"""Initialise a Bridge object."""
self._eq_queue_hosts = [
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST,
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST2
]
self._eq_queue_port = settings.SDX_GATEWAY_EQ_RABBIT_PORT
self._eq_queue_user = settings.SDX_GATEWAY_EQ_RABBITMQ_USER
self._eq_queue_password = settings.SDX_GATEWAY_EQ_RABBITMQ_PASSWORD
self._sdx_queue_port = settings.SDX_GATEWAY_SDX_RABBITMQ_PORT
self._sdx_queue_user = settings.SDX_GATEWAY_SDX_RABBITMQ_USER
self._sdx_queue_password = settings.SDX_GATEWAY_SDX_RABBITMQ_PASSWORD
self._sdx_queue_host = settings.SDX_GATEWAY_SDX_RABBITMQ_HOST
self._eq_queue_urls = [
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[0],
self._eq_queue_port),
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[1],
self._eq_queue_port),
]
self._sdx_queue_url = [
'amqp://{}:{}@{}:{}/%2f'.format(self._sdx_queue_user,
self._sdx_queue_password,
self._sdx_queue_host,
self._sdx_queue_port)
]
self.publisher = QueuePublisher(
self._sdx_queue_url,
settings.COLLECT_QUEUE,
)
self.quarantine_publisher = QueuePublisher(
urls=self._sdx_queue_url,
queue=settings.QUARANTINE_QUEUE,
)
self.consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.EQ_QUEUE,
rabbit_urls=self._eq_queue_urls,
quarantine_publisher=self.quarantine_publisher,
process=self.process,
)
def test_end_to_end(self):
'''
End to end test - spins up a consumer and FTP server. Encrypts a message including a encoded spread sheet and takes the
decrypted and reassemble file are the same files.
This test requires a rabbit mq server to be running locally with the default settings
It also requires a valid OPSWAT API
NOTE this test should only be run manually due to it duration
'''
consumer_thread = ConsumerThread(self.sdx_keys)
consumer_thread.start()
ftp_thread = FTPThread()
ftp_thread.start()
files = [
f for f in listdir(TEST_FILES_PATH)
if isfile(join(TEST_FILES_PATH, f)) and (
f.endswith(".xls") or f.endswith(".xlsx"))
]
for file in files:
with open(join(TEST_FILES_PATH, file), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"' + file + '", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34","survey_id": "221"}'
payload_as_json = json.loads(payload)
jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
with open("./encrypted_files/" + file, "w") as encrypted_file:
encrypted_file.write(jwt)
queue_publisher = QueuePublisher(settings.RABBIT_URLS,
settings.RABBIT_QUEUE)
headers = {'tx_id': str(uuid.uuid4())}
queue_publisher.publish_message(jwt, headers=headers)
# wait 30 seconds for opswat to process
time.sleep(30)
time.sleep(1)
consumer_thread.stop()
ftp_thread.stop()
for file in files:
if 'infected' not in file:
self.assertTrue(
filecmp.cmp(join(TEST_FILES_PATH, file),
join(EndToEndTest.TARGET_PATH, file)))
def run():
logging.basicConfig(format=app.settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=app.settings.LOGGING_LEVEL)
logging.getLogger('sdc.rabbit').setLevel(logging.INFO)
message_processor = MessageProcessor()
quarantine_publisher = QueuePublisher(
urls=app.settings.RABBIT_URLS,
queue=app.settings.RABBIT_QUARANTINE_QUEUE
)
message_consumer = MessageConsumer(
durable_queue=False,
exchange=app.settings.RABBIT_EXCHANGE,
exchange_type='topic',
rabbit_queue=app.settings.RABBIT_QUEUE,
rabbit_urls=app.settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=message_processor.process
)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def run(): # pragma: no cover
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG)
logger.info("Starting SDX Collect", version=__version__)
response_processor = ResponseProcessor()
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=response_processor.process)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def main():
# Set tornado to listen to healthcheck endpoint
app = make_app()
server = tornado.httpserver.HTTPServer(app)
server.bind(os.getenv('PORT', '8080'))
server.start(1)
rp = ResponseProcessor("outbound")
quarantine_publisher = QueuePublisher(
[Config.RABBIT_URL],
os.getenv('RABBIT_QUARANTINE_QUEUE', 'QUARANTINE_TEST'),
)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=os.getenv('RABBIT_EXCHANGE', 'test'),
exchange_type=os.getenv('EXCHANGE_TYPE', 'topic'),
rabbit_queue=os.getenv('RABBIT_QUEUE', 'test'),
rabbit_urls=[Config.RABBIT_URL],
quarantine_publisher=quarantine_publisher,
process=rp.process,
check_tx_id=False,
)
message_consumer.run()
return 0
def run():
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG)
logger.info("Starting SDX Receipt RRM", version=__version__)
response_processor = ResponseProcessor(logger)
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=response_processor.process)
try:
logger.info("Starting consumer")
if settings.SDX_RECEIPT_RRM_SECRET is None:
logger.error("No SDX_RECEIPT_RRM_SECRET env var supplied")
sys.exit(1)
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def run():
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger('sdc.rabbit').setLevel(logging.INFO)
# These structlog settings allow bound fields to persist between classes
structlog.configure(logger_factory=LoggerFactory(), context_class=wrap_dict(dict))
logger = structlog.getLogger()
logger.info('Starting SDX Downstream', version=__version__)
message_processor = MessageProcessor()
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS,
queue=settings.RABBIT_QUARANTINE_QUEUE
)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type='topic',
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=message_processor.process
)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def test_queue_init(self):
this_publisher = QueuePublisher(good_urls, queue_name)
self.assertEqual(this_publisher._urls, good_urls)
self.assertEqual(this_publisher._queue, queue_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertEqual(this_publisher._durable_queue, True)
def setUp(self):
self.amqp_url = 'amqp://*****:*****@0.0.0.0:5672'
self.quarantine_publisher = QueuePublisher([self.amqp_url],
'test_quarantine')
self.consumer = MessageConsumer(True, 'test', 'topic', 'test',
[self.amqp_url],
self.quarantine_publisher,
lambda x, y: True)
self.props = DotDict({'headers': {'tx_id': 'test'}})
self.props_no_tx_id = DotDict({'headers': {}})
self.props_no_headers = DotDict({})
self.props_no_x_delivery_count = DotDict(
{'headers': {
'tx_id': 'test'
}})
self.basic_deliver = DotDict({'delivery_tag': 'test'})
self.body = json.loads('"{test message}"')
def reprocess(tx_id):
logger.info('Reprocessing submission', tx_id=tx_id)
publisher = QueuePublisher(settings.RABBIT_URLS, 'sdx_downstream')
publisher.publish_message(tx_id, headers={'tx_id': tx_id})
logger.info('Successfully reprocessed submission', tx_id=tx_id)
import sys
import os
parent_dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(parent_dir_path)
from sdc.rabbit import QueuePublisher
from sdc.rabbit.exceptions import PublishMessageError
from app import settings
publisher = QueuePublisher(settings.RABBIT_URLS,
'sdx-survey-notification-durable')
if __name__ == "__main__":
with open('tx_ids', 'r') as fp:
lines = list(fp)
if not lines:
sys.exit("No tx_ids in file, exiting script")
for tx_id in lines:
# Remove newline character at the end of the tx_id (if present)
tx_id = tx_id.rstrip()
print("About to put {} on the queue".format(tx_id))
try:
publisher.publish_message(tx_id, headers={'tx_id': tx_id})
except PublishMessageError as e:
print(e)
raise
class ResponseProcessor:
@staticmethod
def options():
rv = {}
try:
rv["secret"] = os.getenv("SDX_COLLECT_SECRET").encode("ascii")
except AttributeError:
# No secret in env
pass
return rv
def __init__(self, logger=logger):
self.logger = logger
self.tx_id = None
self.rrm_publisher = PrivatePublisher(
settings.RABBIT_URLS, settings.RABBIT_RRM_RECEIPT_QUEUE)
self.notifications = QueuePublisher(settings.RABBIT_URLS,
settings.RABBIT_SURVEY_QUEUE)
self.dap = QueuePublisher(settings.RABBIT_URLS,
settings.RABBIT_DAP_QUEUE)
def service_name(self, url=None):
try:
parts = url.split('/')
if 'responses' in parts:
return 'SDX-STORE'
elif 'decrypt' in parts:
return 'SDX-DECRYPT'
elif 'validate' in parts:
return 'SDX-VALIDATE'
except AttributeError:
self.logger.exception("No valid service name")
def process(self, msg, tx_id=None, decrypt=True):
# Bind the tx_id from the rabbit message header as we don't have access to the one in the survey yet.
self.logger = self.logger.bind(tx_id=tx_id)
if decrypt:
decrypted_json = self.decrypt_survey(msg)
else:
decrypted_json = msg
metadata = decrypted_json.get('metadata', {})
self.logger = self.logger.bind(user_id=metadata.get('user_id'),
ru_ref=metadata.get('ru_ref'))
if not tx_id:
self.tx_id = decrypted_json.get('tx_id')
elif tx_id != decrypted_json.get('tx_id'):
self.logger.info(
'tx_ids from decrypted_json and message header do not match. Rejecting message',
decrypted_tx_id=decrypted_json.get('tx_id'),
message_tx_id=self.tx_id)
raise QuarantinableError
else:
self.tx_id = tx_id
valid = self.validate_survey(decrypted_json)
if not valid:
self.logger.info(
"Invalid survey data, skipping receipting and downstream processing"
)
decrypted_json['invalid'] = True
store_response_json = self.store_survey(decrypted_json)
self.logger.info("Saved data to the database", id=store_response_json)
if valid and self._requires_receipting(decrypted_json):
self.send_receipt(decrypted_json)
if valid and self._requires_downstream_processing(decrypted_json):
self.send_notification(store_response_json)
if valid and self._requires_dap_processing(decrypted_json):
self.send_to_dap_queue(decrypted_json)
# If we don't unbind these fields, their current value will be retained for the next
# submission. This leads to incorrect values being logged out in the bound fields.
self.logger = self.logger.unbind("user_id", "ru_ref", "tx_id")
def decrypt_survey(self, encrypted_survey):
self.logger.info("Decrypting survey")
response = self.remote_call(settings.SDX_DECRYPT_URL,
data=encrypted_survey)
try:
self.response_ok(response)
except ClientError:
self.logger.error(
"Survey decryption unsuccessful. Quarantining Survey.")
raise QuarantinableError
self.logger.info("Survey decryption successful")
return response.json()
def validate_survey(self, decrypted_json):
self.logger.info("Validating survey")
try:
self.response_ok(
self.remote_call(settings.SDX_VALIDATE_URL,
json=decrypted_json))
except ClientError:
# If the validation fails, the message is to be marked "invalid"
# and then stored. We don't then want to stop processing at this point.
return False
self.logger.info("Survey validation successful")
return True
def store_survey(self, decrypted_json):
self.logger.info("Storing survey")
response = self.remote_call(settings.SDX_RESPONSES_URL,
json=decrypted_json)
try:
self.response_ok(response)
except ClientError:
self.logger.error(
"Survey storage unsuccessful. Quarantining Survey.")
raise QuarantinableError
self.logger.info("Survey storage successful")
return response.json()
def _requires_receipting(self, decrypted_json):
if self._is_feedback_survey(decrypted_json):
self.logger.info("Feedback survey, skipping receipting")
return False
return True
def make_receipt(self, decrypted_json):
try:
receipt_json = {
'case_id': decrypted_json['case_id'],
'tx_id': decrypted_json['tx_id'],
'collection': {
'exercise_sid':
decrypted_json['collection']['exercise_sid']
},
'metadata': {
'ru_ref': decrypted_json['metadata']['ru_ref'],
'user_id': decrypted_json['metadata']['user_id']
}
}
except KeyError:
self.logger.exception("Unsuccessful publish, missing key values")
raise QuarantinableError
return receipt_json
def _requires_dap_processing(self, decrypted_json):
if self._is_feedback_survey(decrypted_json):
self.logger.info("Feedback survey, skipping sending to DAP")
return False
if decrypted_json.get("survey_id") in [
"007", "023", "134", "147", "281", "283", "lms", "census"
]:
# low carbon, RSI, MWSS, EPE, Dtrades
self.logger.info("Sending to DAP",
survey_id=decrypted_json.get("survey_id"))
return True
return False
def make_dap_data(self, decrypted_json):
"""Creates the json payload required by minifi to send the submission to dap"""
self.logger.info("Creating dap data")
response = self.remote_call(
f"{settings.SDX_RESPONSES_URL}/{decrypted_json['tx_id']}")
try:
self.response_ok(response)
except ClientError:
self.logger.error("Survey retrieval failed. Quarantining Survey.")
raise QuarantinableError
try:
description = "{} survey response for period {} sample unit {}".format(
decrypted_json['survey_id'],
decrypted_json['collection']['period'],
decrypted_json['metadata']['ru_ref'])
dap_json = {
'version':
'1',
'files': [{
'name': f"{decrypted_json['tx_id']}.json",
'URL':
f"{settings.SDX_RESPONSES_URL}/{decrypted_json['tx_id']}",
'sizeBytes': response.headers['Content-Length'],
'md5sum': response.headers['Content-MD5']
}],
'sensitivity':
'High',
'sourceName':
settings.DAP_SOURCE_NAME,
'manifestCreated':
self._get_formatted_current_utc(),
'description':
description,
'iterationL1':
decrypted_json['collection']['period'],
'dataset':
decrypted_json['survey_id'],
'schemaversion':
'1'
}
except KeyError:
self.logger.exception("Unsuccesful publish, missing key values")
raise QuarantinableError
self.logger.info("Created dap data")
return dap_json
def _get_formatted_current_utc(self):
"""
Returns a formatted utc date with only 3 milliseconds as opposed to the ususal 6 that python provides.
Additionally, we provide the Zulu time indicator (Z) at the end to indicate it being UTC time. This is
done for consistency with timestamps provided in other languages.
The format the time is returned is YYYY-mm-ddTHH:MM:SS.fffZ (e.g., 2018-10-10T08:42:24.737Z)
"""
date_time = datetime.utcnow()
milliseconds = date_time.strftime("%f")[:3]
return f"{date_time.strftime('%Y-%m-%dT%H:%M:%S')}.{milliseconds}Z"
def _requires_downstream_processing(self, decrypted_json):
if decrypted_json.get("version") == "0.0.2":
survey_id = decrypted_json.get("survey_id")
self.logger.info("Skipping downstream processing",
survey_id=survey_id)
return False
elif decrypted_json.get("survey_id") == "283":
if self._is_feedback_survey(decrypted_json):
return True
else:
self.logger.info(
"Covid-19 survey, skipping downstream processing")
return False
return True
@staticmethod
def _is_feedback_survey(decrypted_json):
response_type = str(decrypted_json.get("type"))
return response_type.find("feedback") != -1
def send_receipt(self, decrypted_json):
if not decrypted_json.get("survey_id"):
self.logger.error("No survey id")
raise QuarantinableError
self.logger.info("Receipting survey")
receipt = self.make_receipt(decrypted_json)
try:
self.logger.info("About to publish receipt into rrm queue")
self.logger.debug(str(receipt))
self.rrm_publisher.publish(
dumps(receipt),
headers={'tx_id': decrypted_json['tx_id']},
secret=settings.SDX_COLLECT_SECRET)
self.logger.info("Receipt published")
except PublishMessageError:
self.logger.exception("Unsuccesful publish")
raise RetryableError
def send_notification(self, store_response_json):
self.logger.info("Sending to downstream")
try:
self.logger.info("About to publish notification to queue")
self.notifications.publish_message(json.dumps(store_response_json),
headers={'tx_id': self.tx_id})
except PublishMessageError:
self.logger.exception("Unable to queue response notification")
raise RetryableError
def send_to_dap_queue(self, decrypted_json):
self.logger.info("Sending data to dap queue")
message = self.make_dap_data(decrypted_json)
try:
self.logger.info("Publishing data to dap queue")
self.dap.publish_message(dumps(message),
headers={'tx_id': self.tx_id})
except PublishMessageError:
self.logger.exception("Failed to publish to dap queue")
raise RetryableError
self.logger.info("Successfully published to dap queue")
def remote_call(self, request_url, json=None, data=None):
service = self.service_name(request_url)
try:
self.logger.info("Calling service",
request_url=request_url,
service=service)
if json:
return session.post(request_url, json=json, verify=True)
if data:
return session.post(request_url, data=data, verify=True)
return session.get(request_url, verify=True)
except MaxRetryError:
self.logger.error("Max retries exceeded (5)",
request_url=request_url)
raise RetryableError
except ConnectionError:
self.logger.error("Connection error occurred. Retrying")
raise RetryableError
def response_ok(self, res):
request_url = res.url
service = self.service_name(request_url)
res_logger = self.logger.bind(request_url=res.url,
status=res.status_code)
if res.status_code == 200 or res.status_code == 201:
res_logger.info("Returned from service",
response="ok",
service=service)
return
elif 400 <= res.status_code < 500:
if res.json().get('contains_invalid_character'):
logger.error(
"Invalid character found in payload, quarantining submission"
)
raise QuarantinableError
res_logger.error("Returned from service",
response="client error",
service=service)
raise ClientError
else:
res_logger.error("Returned from service",
response="service error",
service=service)
raise RetryableError
class Bridge:
"""A Bridge object that takes from one queue and publishes to another."""
def __init__(self):
"""Initialise a Bridge object."""
self._eq_queue_hosts = [
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST,
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST2
]
self._eq_queue_port = settings.SDX_GATEWAY_EQ_RABBIT_PORT
self._eq_queue_user = settings.SDX_GATEWAY_EQ_RABBITMQ_USER
self._eq_queue_password = settings.SDX_GATEWAY_EQ_RABBITMQ_PASSWORD
self._sdx_queue_port = settings.SDX_GATEWAY_SDX_RABBITMQ_PORT
self._sdx_queue_user = settings.SDX_GATEWAY_SDX_RABBITMQ_USER
self._sdx_queue_password = settings.SDX_GATEWAY_SDX_RABBITMQ_PASSWORD
self._sdx_queue_host = settings.SDX_GATEWAY_SDX_RABBITMQ_HOST
self._eq_queue_urls = [
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[0],
self._eq_queue_port),
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[1],
self._eq_queue_port),
]
self._sdx_queue_url = [
'amqp://{}:{}@{}:{}/%2f'.format(self._sdx_queue_user,
self._sdx_queue_password,
self._sdx_queue_host,
self._sdx_queue_port)
]
self.publisher = QueuePublisher(
self._sdx_queue_url,
settings.COLLECT_QUEUE,
)
self.quarantine_publisher = QueuePublisher(
urls=self._sdx_queue_url,
queue=settings.QUARANTINE_QUEUE,
)
self.consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.EQ_QUEUE,
rabbit_urls=self._eq_queue_urls,
quarantine_publisher=self.quarantine_publisher,
process=self.process,
)
def process(self, message, tx_id=None):
try:
self.publisher.publish_message(message, headers={'tx_id': tx_id})
except PublishMessageError:
logger.exception('Unsuccessful publish.', tx_id=tx_id)
raise RetryableError
except:
logger.exception(
'Unknown exception occurred during publish. Retrying.',
tx_id=tx_id)
raise RetryableError
def run(self):
"""Run this object's MessageConsumer. Stops on KeyboardInterrupt."""
logger.info("Starting consumer")
self.consumer.run()
def stop(self):
logger.info("Stopping consumer")
self.consumer.stop()
class ResponseProcessor:
@staticmethod
def options():
rv = {}
try:
rv["secret"] = os.getenv("SDX_COLLECT_SECRET").encode("ascii")
except AttributeError:
# No secret in env
pass
return rv
def __init__(self, logger=logger):
self.logger = logger
self.tx_id = None
self.rrm_publisher = PrivatePublisher(settings.RABBIT_URLS, settings.RABBIT_RRM_RECEIPT_QUEUE)
self.notifications = QueuePublisher(settings.RABBIT_URLS, settings.RABBIT_SURVEY_QUEUE)
self.dap = QueuePublisher(settings.RABBIT_URLS, settings.RABBIT_DAP_QUEUE)
def service_name(self, url=None):
try:
parts = url.split('/')
if 'responses' in parts:
return 'SDX-STORE'
elif 'decrypt' in parts:
return 'SDX-DECRYPT'
elif 'validate' in parts:
return 'SDX-VALIDATE'
except AttributeError:
self.logger.exception("No valid service name")
def process(self, msg, tx_id=None):
decrypted_json = self.decrypt_survey(msg)
metadata = decrypted_json.get('metadata', {})
self.logger = self.logger.bind(
user_id=metadata.get('user_id'), ru_ref=metadata.get('ru_ref'))
if not tx_id:
self.tx_id = decrypted_json.get('tx_id')
elif tx_id != decrypted_json.get('tx_id'):
self.logger.info(
'tx_ids from decrypted_json and message header do not match. Rejecting message',
decrypted_tx_id=decrypted_json.get('tx_id'),
message_tx_id=self.tx_id)
raise QuarantinableError
else:
self.tx_id = tx_id
self.logger = self.logger.bind(tx_id=self.tx_id)
valid = self.validate_survey(decrypted_json)
if not valid:
self.logger.info("Invalid survey data, skipping receipting and downstream processing")
decrypted_json['invalid'] = True
self.store_survey(decrypted_json)
if valid and self._requires_receipting(decrypted_json):
self.send_receipt(decrypted_json)
if valid and self._requires_downstream_processing(decrypted_json):
self.send_notification()
if valid and self._requires_dap_processing(decrypted_json):
self.send_to_dap_queue(decrypted_json)
self.logger.unbind("user_id", "ru_ref", "tx_id")
def decrypt_survey(self, encrypted_survey):
self.logger.info("Decrypting survey")
response = self.remote_call(settings.SDX_DECRYPT_URL, data=encrypted_survey)
try:
self.response_ok(response)
except ClientError:
self.logger.error("Survey decryption unsuccessful. Quarantining Survey.")
raise QuarantinableError
self.logger.info("Survey decryption successful")
return response.json()
def validate_survey(self, decrypted_json):
self.logger.info("Validating survey")
try:
self.response_ok(self.remote_call(settings.SDX_VALIDATE_URL, json=decrypted_json))
except ClientError:
# If the validation fails, the message is to be marked "invalid"
# and then stored. We don't then want to stop processing at this point.
return False
self.logger.info("Survey validation successful")
return True
def store_survey(self, decrypted_json):
self.logger.info("Storing survey")
response = self.remote_call(settings.SDX_RESPONSES_URL, json=decrypted_json)
try:
self.response_ok(response)
except ClientError:
self.logger.error("Survey storage unsuccessful. Quarantining Survey.")
raise QuarantinableError
self.logger.info("Survey storage successful")
return response
def _requires_receipting(self, decrypted_json):
if self._is_feedback_survey(decrypted_json):
self.logger.info("Feedback survey, skipping receipting")
return False
return True
def make_receipt(self, decrypted_json):
try:
receipt_json = {
'case_id': decrypted_json['case_id'],
'tx_id': decrypted_json['tx_id'],
'collection': {
'exercise_sid':
decrypted_json['collection']['exercise_sid']
},
'metadata': {
'ru_ref': decrypted_json['metadata']['ru_ref'],
'user_id': decrypted_json['metadata']['user_id']
}
}
except KeyError:
self.logger.exception("Unsuccesful publish, missing key values")
raise QuarantinableError
return receipt_json
def _requires_dap_processing(self, decrypted_json):
if self._is_feedback_survey(decrypted_json):
self.logger.info("Feedback survey, skipping sending to DAP")
return False
if decrypted_json.get("survey_id") in ["023", "281", "lms", "census"]: # RSI, Dtrades
self.logger.info("Sending to DAP", survey_id=decrypted_json.get("survey_id"))
return True
return False
def make_dap_data(self, decrypted_json):
self.logger.info("Creating dap data")
response = self.remote_call('{}/{}'.format(settings.SDX_RESPONSES_URL, decrypted_json['tx_id']))
try:
self.response_ok(response)
except ClientError:
self.logger.error("Survey retrieval failed. Quarantining Survey.")
raise QuarantinableError
try:
description = "{} survey response for period {} sample unit {}".format(
decrypted_json['survey_id'],
decrypted_json['collection']['period'],
decrypted_json['metadata']['ru_ref'])
dap_json = {
'version': '1',
'files': [{
'name': '{}.json'.format(decrypted_json['tx_id']),
'URL': '{}/{}'.format(settings.SDX_RESPONSES_URL, decrypted_json['tx_id']),
'sizeBytes': response.headers['Content-Length'],
'md5sum': response.headers['Content-MD5']
}],
'sensitivity': 'High',
'sourceName': settings.DAP_SOURCE_NAME,
'manifestCreated': self._get_formatted_current_utc(),
'description': description,
'iterationL1': decrypted_json['collection']['period'],
'dataset': decrypted_json['survey_id'],
'schemaversion': '1'
}
except KeyError:
self.logger.exception("Unsuccesful publish, missing key values")
raise QuarantinableError
self.logger.info("Created dap data")
return dap_json
def _get_formatted_current_utc(self):
"""
Returns a formatted utc date with only 3 milliseconds as opposed to the ususal 6 that python provides.
Additionally, we provide the Zulu time indicator (Z) at the end to indicate it being UTC time. This is
done for consistency with timestamps provided in other languages.
The format the time is returned is YYYY-mm-ddTHH:MM:SS.fffZ (e.g., 2018-10-10T08:42:24.737Z)
"""
date_time = datetime.utcnow()
milliseconds = date_time.strftime("%f")[:3]
return '{}.{}Z'.format(date_time.strftime("%Y-%m-%dT%H:%M:%S"), milliseconds)
def _requires_downstream_processing(self, decrypted_json):
if self._is_feedback_survey(decrypted_json):
self.logger.info("Feedback survey, skipping downstream processing")
return False
elif decrypted_json.get("version") == "0.0.2":
survey_id = decrypted_json.get("survey_id")
self.logger.info("Skipping downstream processing", survey_id=survey_id)
return False
return True
@staticmethod
def _is_feedback_survey(decrypted_json):
response_type = str(decrypted_json.get("type"))
return response_type.find("feedback") != -1
def send_receipt(self, decrypted_json):
if not decrypted_json.get("survey_id"):
self.logger.error("No survey id")
raise QuarantinableError
self.logger.info("Receipting survey")
receipt = self.make_receipt(decrypted_json)
try:
self.logger.info("About to publish receipt into rrm queue")
self.logger.debug(str(receipt))
self.rrm_publisher.publish(
dumps(receipt),
headers={'tx_id': decrypted_json['tx_id']},
secret=settings.SDX_COLLECT_SECRET)
self.logger.info("Receipt published")
except PublishMessageError:
self.logger.exception("Unsuccesful publish")
raise RetryableError
def send_notification(self):
self.logger.info("Sending to downstream")
try:
self.logger.info("About to publish notification to queue")
self.notifications.publish_message(
self.tx_id, headers={
'tx_id': self.tx_id
})
except PublishMessageError as e:
self.logger.error("Unable to queue response notification", error=e)
raise RetryableError
def send_to_dap_queue(self, decrypted_json):
self.logger.info("Sending data to dap queue")
message = self.make_dap_data(decrypted_json)
try:
self.logger.info("Publishing data to dap queue")
self.dap.publish_message(
dumps(message),
headers={'tx_id': self.tx_id})
except PublishMessageError:
self.logger.exception("Failed to publish to dap queue")
raise RetryableError
self.logger.info("Successfully published to dap queue")
def remote_call(self,
request_url,
json=None,
data=None,
headers=None,
verify=True,
auth=None):
service = self.service_name(request_url)
try:
self.logger.info("Calling service", request_url=request_url, service=service)
r = None
if json:
r = session.post(
request_url,
json=json,
headers=headers,
verify=verify,
auth=auth)
elif data:
r = session.post(
request_url,
data=data,
headers=headers,
verify=verify,
auth=auth)
else:
r = session.get(request_url, headers=headers, verify=verify, auth=auth)
return r
except MaxRetryError:
self.logger.error("Max retries exceeded (5)", request_url=request_url)
raise RetryableError
except ConnectionError:
self.logger.error("Connection error occurred. Retrying")
raise RetryableError
def response_ok(self, res):
request_url = res.url
service = self.service_name(request_url)
res_logger = self.logger
res_logger.bind(request_url=res.url, status=res.status_code)
if res.status_code == 200 or res.status_code == 201:
res_logger.info("Returned from service", response="ok", service=service)
return
elif 400 <= res.status_code < 500:
res_logger.error(
"Returned from service",
response="client error",
service=service)
raise ClientError
else:
res_logger.error(
"Returned from service",
response="service error",
service=service)
raise RetryableError
import sys
import os
parent_dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(parent_dir_path)
from sdc.rabbit import QueuePublisher
from sdc.rabbit.exceptions import PublishMessageError
from app import settings
publisher = QueuePublisher(settings.RABBIT_URLS,
'sdx-survey-notification-durable')
if __name__ == "__main__":
with open('tx_ids', 'r') as fp:
lines = list(fp)
if not lines:
sys.exit("No tx_ids in file, exiting script")
for tx_id in lines:
# Remove newline character at the end of the tx_id (if present)
tx_id = tx_id.rstrip()
store_response_json = f'{{"tx_id": "{tx_id}","feedback":False}}'
print(f"About to put {tx_id} on the queue")
try:
publisher.publish_message(store_response_json,
headers={'tx_id': tx_id})
except PublishMessageError as e:
print(e)
class TestPublisher(unittest.TestCase):
logger = logging.getLogger(__name__)
queue_publisher = QueuePublisher(good_urls, queue_name)
bad_queue_publisher = QueuePublisher(bad_urls, queue_name)
confirm_delivery_queue_publisher = QueuePublisher(good_urls,
queue_name,
confirm_delivery=True)
exchange_publisher = ExchangePublisher(good_urls, exchange_name)
bad_exchange_publisher = ExchangePublisher(bad_urls, exchange_name)
confirm_delivery_exchange_publisher = ExchangePublisher(
good_urls, exchange_name, confirm_delivery=True)
durable_exchange_publisher = DurableExchangePublisher(
good_urls, durable_exchange_name)
bad_durable_exchange_publisher = DurableExchangePublisher(
bad_urls, durable_exchange_name)
confirm_delivery_durable_exchange_publisher = DurableExchangePublisher(
good_urls, durable_exchange_name, confirm_delivery=True)
def test_incomplete_publisher(self):
from sdc.rabbit.publishers import Publisher
class BadPublisher(Publisher):
pass
this_publisher = BadPublisher(good_urls[:1])
with self.assertRaises(NotImplementedError):
this_publisher._do_publish('test')
with self.assertRaises(NotImplementedError):
this_publisher._declare()
with self.assertRaises(PublishMessageError):
this_publisher.publish_message('test')
def test_queue_init(self):
this_publisher = QueuePublisher(good_urls, queue_name)
self.assertEqual(this_publisher._urls, good_urls)
self.assertEqual(this_publisher._queue, queue_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertEqual(this_publisher._durable_queue, True)
def test_exchange_init(self):
this_publisher = ExchangePublisher(good_urls, exchange_name)
self.assertEqual(this_publisher._urls, good_urls)
self.assertEqual(this_publisher._exchange, exchange_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertEqual(this_publisher._durable_exchange, False)
def test_durable_exchange_init(self):
this_publisher = DurableExchangePublisher(good_urls,
durable_exchange_name)
self.assertEqual(this_publisher._urls, good_urls)
self.assertEqual(this_publisher._exchange, durable_exchange_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertEqual(this_publisher._durable_exchange, True)
def test_queue_connect_loops_correctly(self):
this_publisher = QueuePublisher(loop_urls, queue_name)
self.assertEqual(this_publisher._urls, loop_urls)
self.assertEqual(this_publisher._queue, queue_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertTrue(this_publisher._connect())
def test_exchange_connect_loops_correctly(self):
this_publisher = ExchangePublisher(loop_urls, exchange_name)
self.assertEqual(this_publisher._urls, loop_urls)
self.assertEqual(this_publisher._exchange, exchange_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertTrue(this_publisher._connect())
def test_durable_exchange_connect_loops_correctly(self):
this_publisher = DurableExchangePublisher(loop_urls,
durable_exchange_name)
self.assertEqual(this_publisher._urls, loop_urls)
self.assertEqual(this_publisher._exchange, durable_exchange_name)
self.assertEqual(this_publisher._arguments, {})
self.assertEqual(this_publisher._connection, None)
self.assertEqual(this_publisher._channel, None)
self.assertTrue(this_publisher._connect())
def test_queue_connect_amqp_connection_error(self):
with self.assertRaises(AMQPConnectionError):
self.bad_queue_publisher._connect()
def test_queue_connect_confirm_delivery_true(self):
with self.assertLogs(level='INFO') as cm:
self.confirm_delivery_queue_publisher._connect()
msg = 'Enabled delivery confirmation'
self.assertIn(msg, cm.output[8])
def test_exchange_connect_amqp_connection_error(self):
with self.assertRaises(AMQPConnectionError):
self.bad_exchange_publisher._connect()
def test_exchange_connect_confirm_delivery_true(self):
with self.assertLogs(level='INFO') as cm:
self.confirm_delivery_exchange_publisher._connect()
msg = 'Enabled delivery confirmation'
self.assertIn(msg, cm.output[8])
def test_durable_exchange_connect_amqp_connection_error(self):
with self.assertRaises(AMQPConnectionError):
self.bad_durable_exchange_publisher._connect()
def test_durable_exchange_connect_confirm_delivery_true(self):
with self.assertLogs(level='INFO') as cm:
self.confirm_delivery_durable_exchange_publisher._connect()
msg = 'Enabled delivery confirmation'
self.assertIn(msg, cm.output[8])
def test_queue_connect_amqpok(self):
result = self.queue_publisher._connect()
self.assertEqual(result, True)
def test_queue_disconnect_ok(self):
self.queue_publisher._connect()
with self.assertLogs(level='DEBUG') as cm:
self.queue_publisher._disconnect()
msg = 'Disconnected from rabbit'
self.assertIn(msg, cm[1][-1])
def test_exchange_connect_amqpok(self):
result = self.exchange_publisher._connect()
self.assertEqual(result, True)
def test_exchange_disconnect_ok(self):
self.exchange_publisher._connect()
with self.assertLogs(level='DEBUG') as cm:
self.exchange_publisher._disconnect()
msg = 'Disconnected from rabbit'
self.assertIn(msg, cm[1][-1])
def test_durable_exchange_connect_amqpok(self):
result = self.durable_exchange_publisher._connect()
self.assertEqual(result, True)
def test_durable_exchange_disconnect_ok(self):
self.durable_exchange_publisher._connect()
with self.assertLogs(level='DEBUG') as cm:
self.durable_exchange_publisher._disconnect()
msg = 'Disconnected from rabbit'
self.assertIn(msg, cm[1][-1])
def test_queue_disconnect_already_closed_connection(self):
self.queue_publisher._connect()
self.queue_publisher._disconnect()
with self.assertLogs(level='DEBUG') as cm:
self.queue_publisher._disconnect()
msg = 'Close called on closed connection'
self.assertIn(msg, cm.output[1])
def test_exchange_disconnect_already_closed_connection(self):
self.exchange_publisher._connect()
self.exchange_publisher._disconnect()
with self.assertLogs(level='DEBUG') as cm:
self.exchange_publisher._disconnect()
msg = 'Close called on closed connection'
self.assertIn(msg, cm.output[1])
def test_durable_exchange_disconnect_already_closed_connection(self):
self.durable_exchange_publisher._connect()
self.durable_exchange_publisher._disconnect()
with self.assertLogs(level='DEBUG') as cm:
self.durable_exchange_publisher._disconnect()
msg = 'Close called on closed connection'
self.assertIn(msg, cm.output[1])
def test_queue_publish_message_no_connection(self):
with self.assertRaises(PublishMessageError):
self.bad_queue_publisher.publish_message(test_data['valid'])
def test_queue_publish(self):
"""Test that when a message is successfully published, a result of True is given and
the correct messages are logged.
"""
self.queue_publisher._connect()
with self.assertLogs(level='INFO') as cm:
result = self.queue_publisher.publish_message(test_data['valid'])
self.assertEqual(True, result)
self.assertIn('Published message to queue', cm.output[8])
def test_queue_publish_nack_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = NackError('a')
self.queue_publisher._connect()
with self.assertRaises(PublishMessageError):
self.queue_publisher.publish_message(test_data['valid'])
def test_queue_publish_unroutable_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = UnroutableError('a')
self.queue_publisher._connect()
with self.assertRaises(PublishMessageError):
self.queue_publisher.publish_message(test_data['valid'])
def test_queue_publish_generic_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = Exception()
self.queue_publisher._connect()
with self.assertRaises(Exception):
self.queue_publisher.publish_message(test_data['valid'])
def test_exchange_publish_message_no_connection(self):
with self.assertRaises(PublishMessageError):
self.bad_exchange_publisher.publish_message(test_data['valid'])
def test_exchange_publish(self):
"""Test that when a message is successfully published, a result of True is given and
the correct messages are logged.
"""
self.exchange_publisher._connect()
with self.assertLogs(level='INFO') as cm:
result = self.exchange_publisher.publish_message(
test_data['valid'])
self.assertEqual(True, result)
self.assertIn('Published message to exchange', cm.output[8])
def test_exchange_publish_nack_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = NackError('a')
self.exchange_publisher._connect()
with self.assertRaises(PublishMessageError):
self.exchange_publisher.publish_message(test_data['valid'])
def test_exchange_publish_unroutable_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = UnroutableError('a')
self.exchange_publisher._connect()
with self.assertRaises(PublishMessageError):
self.exchange_publisher.publish_message(test_data['valid'])
def test_exchange_publish_generic_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = Exception()
self.exchange_publisher._connect()
with self.assertRaises(Exception):
self.exchange_publisher.publish_message(test_data['valid'])
def test_durable_exchange_publish_message_no_connection(self):
with self.assertRaises(PublishMessageError):
self.bad_durable_exchange_publisher.publish_message(
test_data['valid'])
def test_durable_exchange_publish(self):
"""Test that when a message is successfully published, a result of True is given and
the correct messages are logged.
"""
self.durable_exchange_publisher._connect()
with self.assertLogs(level='INFO') as cm:
result = self.durable_exchange_publisher.publish_message(
test_data['valid'])
self.assertEqual(True, result)
self.assertIn('Published message to exchange', cm.output[8])
def test_durable_exchange_publish_nack_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = NackError('a')
self.durable_exchange_publisher._connect()
with self.assertRaises(PublishMessageError):
self.durable_exchange_publisher.publish_message(
test_data['valid'])
def test_durable_exchange_publish_unroutable_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = UnroutableError('a')
self.durable_exchange_publisher._connect()
with self.assertRaises(PublishMessageError):
self.durable_exchange_publisher.publish_message(
test_data['valid'])
def test_durable_exchange_publish_generic_error(self):
mock_method = 'pika.adapters.blocking_connection.BlockingChannel.basic_publish'
with mock.patch(mock_method) as barMock:
barMock.side_effect = Exception()
self.durable_exchange_publisher._connect()
with self.assertRaises(Exception):
self.durable_exchange_publisher.publish_message(
test_data['valid'])
def __init__(self, logger=logger):
self.logger = logger
self.tx_id = None
self.rrm_publisher = PrivatePublisher(settings.RABBIT_URLS, settings.RABBIT_RRM_RECEIPT_QUEUE)
self.notifications = QueuePublisher(settings.RABBIT_URLS, settings.RABBIT_SURVEY_QUEUE)
self.dap = QueuePublisher(settings.RABBIT_URLS, settings.RABBIT_DAP_QUEUE)