def test_get_notification(self):
result = get_notification({})
self.assertIsNone(result)
result = get_notification({
'header': 'any header',
'type': 'events',
'contents': {
'applied': 'any applied',
'last_event': 'any last_event'
}
})
self.assertEqual(result.type, 'events')
self.assertEqual(result.header, 'any header')
self.assertEqual(result.contents, {
'applied': 'any applied',
'last_event': 'any last_event'
})
def on_dump_listener(msg):
notification = get_notification(msg)
workflow = {'workflow_name': DATA_CONSISTENCY_TEST}
arguments = {
'catalogue': notification.header.get('catalogue'),
'collection': notification.header.get('collection'),
'application': notification.header.get('application'),
}
if can_handle(**arguments):
arguments['process_id'] = notification.header.get('process_id')
start_workflow(workflow, arguments)
def new_events_notification_handler(msg):
notification = get_notification(msg)
if notification.header.get('catalogue') not in LISTEN_TO_CATALOGS:
return
workflow = {'workflow_name': KAFKA_PRODUCE}
arguments = {
'catalogue': notification.header.get('catalogue'),
'collection': notification.header.get('collection'),
'application': notification.header.get('application'),
'process_id': notification.header.get('process_id'),
}
start_workflow(workflow, arguments)
def test_from_msg(self):
msg = {
'type': 'dump',
'header': {
'some': 'header'
},
'contents': {
'catalog': 'SOME CAT',
'collection': 'SOME COLL',
}
}
notification = get_notification(msg)
self.assertIsInstance(notification, DumpNotification)
self.assertEqual({'some': 'header'}, notification.header)
self.assertEqual('SOME CAT', notification.contents.get('catalog'))
self.assertEqual('SOME COLL', notification.contents.get('collection'))
def distribute_on_export_test(msg):
"""
On a successfull export test, distribute the files
:param msg:
:return:
"""
notification = get_notification(msg)
# Start an export cat-col to db workflow to update the analysis database
workflow = {
'workflow_name': DISTRIBUTE
}
arguments = {
'catalogue': notification.contents.get('catalogue'),
'process_id': notification.header.get('process_id'),
}
start_workflow(workflow, arguments)
def dump_on_new_events(msg):
"""
On any creation of events, update the analysis database for the new changes
:param msg:
:return:
"""
notification = get_notification(msg)
# Start an export cat-col to db workflow to update the analysis database
workflow = {'workflow_name': EXPORT}
arguments = {
'catalogue': notification.header.get('catalogue'),
'collection': notification.header.get('collection'),
'application': notification.header.get('application'),
'process_id': notification.header.get('process_id'),
'destination': 'Database',
'include_relations': False,
'retry_time': 10 * 60 # retry for max 10 minutes if already running
}
start_workflow(workflow, arguments)