def start_update_object_workflow(update_object: UpdateObject):
mapper = MapperRegistry.get(update_object.catalogue,
update_object.collection)
workflow = {
'workflow_name': IMPORT_OBJECT,
}
arguments = {
'header': {
'message_id': update_object.update_message.id,
'catalogue': update_object.catalogue,
'entity': update_object.collection,
'collection': update_object.collection,
'entity_id': update_object.entity_id,
'entity_id_attr': mapper.entity_id,
'source': update_object.source,
'application': update_object.application,
'timestamp': datetime.datetime.utcnow().isoformat(),
'version': mapper.version,
'on_workflow_complete': {
# Picked up by Workflow to notify us when import is complete
'exchange': MESSAGE_EXCHANGE,
'key': UPDATE_OBJECT_COMPLETE_KEY,
}
},
'contents': {
**update_object.mapped_entity,
}
}
start_workflow(workflow, arguments)
def execute_workflow(self, workflow: list, workflow_process_id: str):
args = {
'header': {
'workflow': workflow,
'process_id': workflow_process_id,
}
}
workflow = {'workflow_name': 'dynamic'}
start_workflow(workflow, args)
def on_dump_listener(msg):
notification = get_notification(msg)
workflow = {'workflow_name': DATA_CONSISTENCY_TEST}
arguments = {
'catalogue': notification.header.get('catalogue'),
'collection': notification.header.get('collection'),
'application': notification.header.get('application'),
}
if can_handle(**arguments):
arguments['process_id'] = notification.header.get('process_id')
start_workflow(workflow, arguments)
def new_events_notification_handler(msg):
notification = get_notification(msg)
if notification.header.get('catalogue') not in LISTEN_TO_CATALOGS:
return
workflow = {'workflow_name': KAFKA_PRODUCE}
arguments = {
'catalogue': notification.header.get('catalogue'),
'collection': notification.header.get('collection'),
'application': notification.header.get('application'),
'process_id': notification.header.get('process_id'),
}
start_workflow(workflow, arguments)
def distribute_on_export_test(msg):
"""
On a successfull export test, distribute the files
:param msg:
:return:
"""
notification = get_notification(msg)
# Start an export cat-col to db workflow to update the analysis database
workflow = {
'workflow_name': DISTRIBUTE
}
arguments = {
'catalogue': notification.contents.get('catalogue'),
'process_id': notification.header.get('process_id'),
}
start_workflow(workflow, arguments)
def dump_on_new_events(msg):
"""
On any creation of events, update the analysis database for the new changes
:param msg:
:return:
"""
notification = get_notification(msg)
# Start an export cat-col to db workflow to update the analysis database
workflow = {'workflow_name': EXPORT}
arguments = {
'catalogue': notification.header.get('catalogue'),
'collection': notification.header.get('collection'),
'application': notification.header.get('application'),
'process_id': notification.header.get('process_id'),
'destination': 'Database',
'include_relations': False,
'retry_time': 10 * 60 # retry for max 10 minutes if already running
}
start_workflow(workflow, arguments)
def test_start_workflow(self, mock_blocking_connection, mock_basic_properties):
mock_connection = MagicMock()
mock_channel = MagicMock()
mock_blocking_connection.return_value.__enter__.return_value = mock_connection
mock_connection.channel.return_value = mock_channel
start_workflow('any workflow', {'arguments': 'any arguments'})
mock_channel.basic_publish.assert_called_with(
body='{"header": {"arguments": "any arguments"}, "contents": {}, "workflow": "any workflow"}',
exchange='workflow exchange',
properties=mock_basic_properties(
delivery_mode=2 # Make messages persistent
),
routing_key='workflow request key')
start_workflow('any workflow', {'arguments': 'any arguments', 'contents_ref': 'contents ref'})
mock_channel.basic_publish.assert_called_with(
body='{"header": {"arguments": "any arguments", "contents_ref": "contents ref"}, "contents": {}, "workflow": "any workflow", "contents_ref": "contents ref"}',
exchange='workflow exchange',
properties=mock_basic_properties(
delivery_mode=2 # Make messages persistent
),
routing_key='workflow request key')