def work_remote_id_change_action(**kwargs):
decisions = kwargs['decisions']
execution_id = kwargs['execution_id']
names = {
'enrich': f'enrichment-{execution_id}',
'change': f'generate-{execution_id}'
}
kwargs['names'] = names
enrich_signature = _build_enrich_signature(**kwargs)
change_data_signatures = _build_change_data_signatures(**kwargs)
if change_data_signatures is None:
decisions.append(CompleteWork())
return
great_chain = chain(enrich_signature,
group(*tuple(change_data_signatures)))
chain_results = great_chain(**kwargs)
if chain_results is None:
return
fungal_leech = _build_fungal_leech(**kwargs)
fungal_leech(**kwargs)
if not fungal_leech.is_complete:
return
post_process_group = _build_post_process_group(**kwargs)
post_process_results = post_process_group(**kwargs)
if post_process_results is None:
return
decisions.append(CompleteWork())
def _build_group(task_args, **kwargs):
subtask_name = 'work_remote_id_change_action'
signatures = []
execution_id = kwargs['execution_id']
workflow_args = kwargs['workflow_args']
category_id = workflow_args['category_id']
changelog_types = task_args.get_argument_value('changelog_types')
change_category = changelog_types.categories[category_id]
change_actions = task_args.get_argument_value('change_actions')
if isinstance(change_actions, list):
change_actions = change_actions[0]
for action_id, change_action in change_category.change_types.items():
change_action = changelog_types[str(action_id)]
remote_actions = change_actions.get(change_action.action, {})
if not remote_actions:
continue
new_task_args = task_args.replace_argument_value(
subtask_name, {'action_id': action_id}, action_id)
subtask_identifier = f'work_action-{change_action}-{execution_id}'
change_type_signature = SubtaskSignature(subtask_identifier,
subtask_name,
task_args=new_task_args,
**kwargs)
signatures.append(change_type_signature)
tuple_signatures = tuple(signatures)
return group(*tuple_signatures)
def _build_index_group(task_args, **kwargs):
signatures = []
names = kwargs['names']
local_id_values = task_args.get_argument_value('local_id_values')
remote_id_values = task_args.get_argument_value('remote_id_values')
local_linked_values = local_id_values['linked']
new_ids = remote_id_values - local_id_values['all']
newly_linked_ids = (remote_id_values - local_linked_values) - new_ids
operations = [
('unlink_old_ids', local_linked_values - remote_id_values),
('link_new_ids', newly_linked_ids),
('put_new_ids', new_ids),
]
for operation in operations:
operation_name = operation[0]
id_values = operation[1]
operation_identifier = names[operation_name]
new_task_arg = {'id_values': id_values}
new_task_args = task_args.replace_argument_value(
operation_name, new_task_arg, operation_identifier)
operation_signature = ActivitySignature(operation_identifier,
operation_name,
task_args=new_task_args,
**kwargs)
signatures.append(operation_signature)
if not signatures:
return None
return group(*tuple(signatures))
def _build_chain(names, **kwargs):
get_local_ids = ActivitySignature(names['local'], 'get_local_ids', **kwargs)
get_remote_ids = ActivitySignature(names['remote'], 'get_remote_ids', **kwargs)
generate_mapping = ActivitySignature(names['map'], 'build_mapping', **kwargs)
get_ids_group = group(get_local_ids, get_remote_ids)
get_change_type = ActivitySignature(names['change_types'], 'pull_change_types', **kwargs)
get_schema_entry = ActivitySignature(names['schema_entry'], 'pull_schema_entry', **kwargs)
work_fip_links = SubtaskSignature(names['work_links'], 'work_fip_links', **kwargs)
great_chain = chain(get_schema_entry, get_ids_group, work_fip_links, get_change_type, generate_mapping)
return great_chain
def _build_group(names, task_args, **kwargs):
subtask_name = 'work_remote_id'
work_history = kwargs['work_history']
execution_id = kwargs['execution_id']
remote_id_values = work_history.get_result(names['remote'])
work_remote_ids_signatures = []
for remote_id_value in remote_id_values['remote_id_values']:
subtask_identifier = f'work_id-{remote_id_value}-{execution_id}'
new_task_args = task_args.replace_argument_value(subtask_name, {'id_value': remote_id_value}, remote_id_value)
work_remote_id_signature = SubtaskSignature(subtask_identifier, subtask_name, task_args=new_task_args, **kwargs)
work_remote_ids_signatures.append(work_remote_id_signature)
tuple_signatures = tuple(work_remote_ids_signatures)
return group(*tuple_signatures)
def _build_query_data_group(task_args, **kwargs):
execution_id = kwargs['execution_id']
task_name = 'query_credible_data'
report_args = task_args.get_argument_value('report_args')
signatures = []
for query_name, query_args in report_args['queries'].items():
activity_id = f'{query_name}-{execution_id}'
new_task_args = task_args.replace_argument_value(
task_name, query_args, activity_id)
signature = ActivitySignature(activity_id,
task_name,
task_args=new_task_args,
**kwargs)
signatures.append(signature)
if not signatures:
return None
return group(*tuple(signatures))
def _build_post_process_group(execution_id, task_args, **kwargs):
ext_id_values = task_args.get_argument_value('ext_id_values')
signatures = []
encounter_ids = []
for entry in ext_id_values:
id_type = entry[1]
id_value = entry[3]
if id_type == 'ClientVisit':
encounter_ids.append(id_value)
if encounter_ids:
subtask_name = 'post_process_encounters'
subtask_identifier = f'post_process_encounters-{execution_id}'
new_task_args = task_args.replace_argument_value(
subtask_name, {'encounter_ids': encounter_ids}, subtask_identifier)
signatures.append(
SubtaskSignature(subtask_identifier,
subtask_name,
task_args=new_task_args))
return group(*tuple(signatures))
def work_remote_id(task_args, **kwargs):
subtask_name = 'work_remote_id_change_type'
decisions = kwargs['decisions']
execution_id = kwargs['execution_id']
changelog_types = task_args.get_argument_value('changelog_types')
signatures = []
for category_id, change_category in changelog_types.categories.items():
new_task_args = task_args.replace_argument_value(
subtask_name, {'category_id': category_id}, category_id)
subtask_identifier = f'work_type-{change_category}-{execution_id}'
category_signature = SubtaskSignature(subtask_identifier,
subtask_name,
task_args=new_task_args,
**kwargs)
signatures.append(category_signature)
tuple_signatures = tuple(signatures)
great_group = group(*tuple_signatures)
group_results = great_group(task_args=task_args, **kwargs)
if group_results is None:
return
decisions.append(CompleteWork())
def fungal_leech(**kwargs):
decisions = kwargs['decisions']
execution_id = kwargs['execution_id']
names = {
'transform': f'transform-{execution_id}',
'index': f'index-{execution_id}',
'graph': f'graph-{execution_id}',
}
kwargs['names'] = names
transform_signature = _build_transform_signature(**kwargs)
transform_results = transform_signature(**kwargs)
if transform_results is None:
return
assimilate_group = _build_assimilate_group(**kwargs)
assimilate_results = assimilate_group(**kwargs)
if assimilate_results is None:
return
index_signature = _build_index_signature(**kwargs)
graph_signature = _build_graph_signature(**kwargs)
load_group = group(index_signature, graph_signature)
load_results = load_group(**kwargs)
if load_results is None:
return
decisions.append(CompleteWork())
def _build_assimilate_group(task_args, **kwargs):
batch_size = 100
signatures = []
execution_id = kwargs['execution_id']
task_name = 'assimilate'
transform_results = task_args.get_argument_value('transform')
batches = []
batch = []
for transform_result in transform_results:
potentials = transform_result['potentials']
source_vertex = transform_result['source_vertex']
extracted_data = transform_result['extracted_data']
for potential in potentials:
if len(batch) > batch_size:
batches.append(batch)
batch = []
new_arg = {
'rule_entry': potential[1],
'potential_vertex': potential[0],
'source_vertex': source_vertex,
'extracted_data': extracted_data
}
batch.append(new_arg)
if batch:
batches.append(batch)
for batch in batches:
task_identifier = f'assimilate-{batches.index(batch)}-{execution_id}'
new_task_args = task_args.replace_argument_value(
task_name, {'assimilation_batch': batch}, task_identifier)
signature = ActivitySignature(task_identifier,
task_name,
task_args=new_task_args,
**kwargs)
signatures.append(signature)
tuple_signatures = tuple(signatures)
return group(*tuple_signatures)