def _create_build_and_send_chain(task_args, **kwargs): execution_id = kwargs['execution_id'] build_task_name = 'build_reports' send_task_name = 'send_reports' build_id = f'{build_task_name}-{execution_id}' send_id = f'{send_task_name}-{execution_id}' build_signature = ActivitySignature(build_id, build_task_name, **kwargs) send_signature = ActivitySignature(send_id, send_task_name, **kwargs) return chain(build_signature, send_signature)
def _build_chain(names, **kwargs): get_local_ids = ActivitySignature(names['local'], 'get_local_ids', **kwargs) get_remote_ids = ActivitySignature(names['remote'], 'get_remote_ids', **kwargs) generate_mapping = ActivitySignature(names['map'], 'build_mapping', **kwargs) get_ids_group = group(get_local_ids, get_remote_ids) get_change_type = ActivitySignature(names['change_types'], 'pull_change_types', **kwargs) get_schema_entry = ActivitySignature(names['schema_entry'], 'pull_schema_entry', **kwargs) work_fip_links = SubtaskSignature(names['work_links'], 'work_fip_links', **kwargs) great_chain = chain(get_schema_entry, get_ids_group, work_fip_links, get_change_type, generate_mapping) return great_chain
def _build_enrich_signature(**kwargs): names = kwargs['names'] fn_identifier = names['enrich'] enrich_signature = ActivitySignature(fn_identifier, 'get_enrichment_for_change_action', **kwargs) return enrich_signature
def _build_index_group(task_args, **kwargs): signatures = [] names = kwargs['names'] local_id_values = task_args.get_argument_value('local_id_values') remote_id_values = task_args.get_argument_value('remote_id_values') local_linked_values = local_id_values['linked'] new_ids = remote_id_values - local_id_values['all'] newly_linked_ids = (remote_id_values - local_linked_values) - new_ids operations = [ ('unlink_old_ids', local_linked_values - remote_id_values), ('link_new_ids', newly_linked_ids), ('put_new_ids', new_ids), ] for operation in operations: operation_name = operation[0] id_values = operation[1] operation_identifier = names[operation_name] new_task_arg = {'id_values': id_values} new_task_args = task_args.replace_argument_value( operation_name, new_task_arg, operation_identifier) operation_signature = ActivitySignature(operation_identifier, operation_name, task_args=new_task_args, **kwargs) signatures.append(operation_signature) if not signatures: return None return group(*tuple(signatures))
def _build_command_chain(task_args, **kwargs): signatures = [] command_name = task_args.get_argument_value('command_name') command_args = task_args.get_argument_value('command_args') for command_arg in command_args: collected_args = sorted([(x, y) for x, y in command_arg.items()]) command_id = str(tuple(collected_args)) command = {'command': command_name, 'command_args': command_arg} new_task_args = task_args.replace_argument_value(command_id, command) signatures.append(ActivitySignature(command_id, 'run_credible_command', task_args=new_task_args, **kwargs)) return chain(*tuple(signatures))
def _build_query_data_group(task_args, **kwargs): execution_id = kwargs['execution_id'] task_name = 'query_credible_data' report_args = task_args.get_argument_value('report_args') signatures = [] for query_name, query_args in report_args['queries'].items(): activity_id = f'{query_name}-{execution_id}' new_task_args = task_args.replace_argument_value( task_name, query_args, activity_id) signature = ActivitySignature(activity_id, task_name, task_args=new_task_args, **kwargs) signatures.append(signature) if not signatures: return None return group(*tuple(signatures))
def _build_great_chain(**kwargs): execution_id = kwargs['execution_id'] signatures = [ ActivitySignature(f'get_productivity-{execution_id}', 'get_productivity_report_data', **kwargs), ActivitySignature(f'build_team-{execution_id}', 'build_clinical_teams', **kwargs), ActivitySignature(f'build_caseload-{execution_id}', 'build_clinical_caseloads', **kwargs), ActivitySignature(f'build_daily_report-{execution_id}', 'build_daily_report', **kwargs), ActivitySignature(f'write_report_data-{execution_id}', 'write_report_data', **kwargs), ActivitySignature(f'send_report-{execution_id}', 'send_report', **kwargs), ] return chain(*tuple(signatures))
def _build_change_data_signatures(task_args, **kwargs): batch_size = 100 execution_id = kwargs['execution_id'] subtask_name = 'batch_generate_remote_id_change_data' signatures = [] names = kwargs['names'] workflow_args = kwargs['workflow_args'] fn_identifier = names['change'] change_actions = task_args.get_argument_value('change_actions') if isinstance(change_actions, list): change_actions = change_actions[0] changelog_types = task_args.get_argument_value('changelog_types') action_id = workflow_args['action_id'] change_action = changelog_types[str(action_id)] remote_actions = change_actions.get(change_action.action, {}) batches = [] batch = [] for remote_change in remote_actions: if len(batch) > batch_size: batches.append(batch) batch = [] batch.append(remote_change) if batch: batches.append(batch) for batch in batches: batch_identifier = f'{batches.index(batch)}-{subtask_name}-{execution_id}' new_task_args = task_args.replace_argument_value( subtask_name, {'remote_changes': batch}, batch_identifier) lambda_identifier = f'{batch_identifier}-{fn_identifier}' signature = ActivitySignature(lambda_identifier, subtask_name, task_args=new_task_args, **kwargs) signatures.append(signature) if not signatures: return None return signatures
def _build_assimilate_group(task_args, **kwargs): batch_size = 100 signatures = [] execution_id = kwargs['execution_id'] task_name = 'assimilate' transform_results = task_args.get_argument_value('transform') batches = [] batch = [] for transform_result in transform_results: potentials = transform_result['potentials'] source_vertex = transform_result['source_vertex'] extracted_data = transform_result['extracted_data'] for potential in potentials: if len(batch) > batch_size: batches.append(batch) batch = [] new_arg = { 'rule_entry': potential[1], 'potential_vertex': potential[0], 'source_vertex': source_vertex, 'extracted_data': extracted_data } batch.append(new_arg) if batch: batches.append(batch) for batch in batches: task_identifier = f'assimilate-{batches.index(batch)}-{execution_id}' new_task_args = task_args.replace_argument_value( task_name, {'assimilation_batch': batch}, task_identifier) signature = ActivitySignature(task_identifier, task_name, task_args=new_task_args, **kwargs) signatures.append(signature) tuple_signatures = tuple(signatures) return group(*tuple_signatures)
def _build_work_signature(**kwargs): names = kwargs['names'] fn_identifier = names['work'] work_signature = ActivitySignature(fn_identifier, 'work_remote_id_change_type', **kwargs) return work_signature
def _build_local_max_signature(**kwargs): names = kwargs['names'] fn_identifier = names['local_max'] get_local_max_signature = ActivitySignature( fn_identifier, 'get_local_max_change_type_value', **kwargs) return get_local_max_signature
def _build_graph_signature(**kwargs): names = kwargs['names'] graph_task_identifier = names['graph'] graph_signature = ActivitySignature(graph_task_identifier, 'graph', **kwargs) return graph_signature
def _build_index_signature(**kwargs): names = kwargs['names'] index_task_identifier = names['index'] index_signature = ActivitySignature(index_task_identifier, 'index', **kwargs) return index_signature
def _build_transform_signature(**kwargs): names = kwargs['names'] task_identifier = names['transform'] return ActivitySignature(task_identifier, 'transform', **kwargs)
def _build_graph_signature(task_args, **kwargs): operation_name = 'graph_links' names = kwargs['names'] identifier = names['graph'] return ActivitySignature(identifier, operation_name, **kwargs)
def _build_parse_encounters_signature(execution_id, **kwargs): fn_name = 'post_process_parse_encounters' fn_identifier = f'{fn_name}-{execution_id}' return ActivitySignature(fn_identifier, fn_name, **kwargs)