async def import_raw_topic_data(topic_event): topic = get_topic(topic_event.code) if topic is None: raise Exception(topic_event.code + " topic name does not exist") add_audit_columns(topic_event.data, INSERT) save_topic_instance(topic_event.code, topic_event.data) await __trigger_pipeline(topic_event)
def trigger_pipeline(topic_name, instance, trigger_type: TriggerType): log.info("trigger_pipeline topic_name :{0}".format(topic_name)) topic = get_topic(topic_name) pipeline_list = load_pipeline_by_topic_id(topic.topicId) for pipeline in pipeline_list: if __match_trigger_type(trigger_type, pipeline): # log.info("pipeline run: {0}".format(pipeline.json())) run_pipeline(pipeline, instance)
async def rerun_pipeline(topic_name, instance_id, pipeline_id): instance = find_topic_data_by_id_and_topic_name(topic_name, instance_id) topic = get_topic(topic_name) pipeline_list = load_pipeline_by_topic_id(topic.topicId) for pipeline in pipeline_list: if pipeline.pipelineId == pipeline_id: log.info("rerun topic {0} and pipeline {1}".format(topic_name, pipeline.pipelineId)) run_pipeline(pipeline, instance) return {"received": True}
async def save_topic_data(topic_event: TopicEvent): # TODO user check URP topic = get_topic(topic_event.code) if topic is None: raise Exception("topic name does not exist") add_audit_columns(topic_event.data, INSERT) save_topic_instance(topic_event.code, topic_event.data) await __trigger_pipeline(topic_event) return {"received": True}
def trigger_pipeline(topic_name, instance, trigger_type: TriggerType): log.info("trigger_pipeline topic_name :{0}".format(topic_name)) topic = get_topic(topic_name) # TODO validate data with topic schema pipeline_list = load_pipeline_by_topic_id(topic.topicId) # futures =[] for pipeline in pipeline_list: if __match_trigger_type(trigger_type, pipeline): log.debug("pipeline run: {0}".format(pipeline.json())) run_pipeline(pipeline, instance)
def sync_alarm_message(alarm: AlarmMessage): topic_event = TopicEvent(code="alarm", data=alarm.dict()) topic = get_topic(topic_event.code) if topic is None: raise Exception("topic name does not exist") add_audit_columns(topic_event.data, INSERT) save_topic_instance(topic_event.code, topic_event.data) trigger_pipeline(topic_event.code, { pipeline_constants.NEW: topic_event.data, pipeline_constants.OLD: None }, TriggerType.insert) asyncio.ensure_future(send_notifier(alarm))
async def import_raw_topic_data(topic_event, current_user): topic = get_topic(topic_event.code, current_user) if topic is None: raise Exception(topic_event.code + " topic name does not exist") raw_data = await get_input_data(topic, topic_event) add_audit_columns(raw_data, INSERT) flatten_fields = get_flatten_field(topic_event.data, topic.factors) raw_data.update(flatten_fields) save_topic_instance(topic, raw_data, current_user) __trigger_pipeline(topic_event, current_user)
def trigger_pipeline_2(topic_name, instance, trigger_type: TriggerType, current_user=None, trace_id=None): topic = get_topic(topic_name, current_user) pipeline_list = load_pipeline_by_topic_id(topic.topicId, current_user) for pipeline in pipeline_list: if __match_trigger_type(trigger_type, pipeline): pipeline_context = PipelineContext(pipeline, instance, current_user, trace_id) run_pipeline(pipeline_context, current_user)
def sync_pipeline_monitor_data(pipeline_monitor: PipelineRunStatus): topic_event = TopicEvent(code="raw_pipeline_monitor", data=pipeline_monitor.dict()) # asyncio.ensure_future(import_raw_topic_data(topic_event)) topic = get_topic(topic_event.code) if topic is None: raise Exception("topic name does not exist") add_audit_columns(topic_event.data, INSERT) save_topic_instance(topic_event.code, topic_event.data) watchmen.pipeline.index.trigger_pipeline( topic_event.code, { pipeline_constants.NEW: topic_event.data, pipeline_constants.OLD: None }, TriggerType.insert)
async def rerun_pipeline(topic_name, instance_id, pipeline_id=None, current_user: User = Depends(deps.get_current_user)): topic = get_topic(topic_name) trace_id = get_surrogate_key() instance = find_topic_data_by_id_and_topic_name(topic, instance_id) data = {"new": instance, "old": None} pipeline_list = load_pipeline_by_topic_id(topic.topicId) for pipeline in find_execute_pipeline_list(pipeline_id, pipeline_list): log.info("rerun topic {0} and pipeline {1}".format( topic_name, pipeline.name)) pipeline_context = PipelineContext(pipeline, data, current_user, trace_id) run_pipeline(pipeline_context, current_user) return {"received": True, "trace_id": trace_id}
async def rerun_pipeline(topic_name, instance_id, pipeline_id, current_user: User = Depends(deps.get_current_user)): topic = get_topic(topic_name) trace_id = get_surrogate_key() instance = find_topic_data_by_id_and_topic_name(topic, instance_id) pipeline_list = load_pipeline_by_topic_id(topic.topicId) for pipeline in pipeline_list: if pipeline.pipelineId == pipeline_id: log.info("rerun topic {0} and pipeline {1}".format( topic_name, pipeline.pipelineId)) pipeline_context = PipelineContext(pipeline, instance, current_user, trace_id) run_pipeline(pipeline_context, current_user) return {"received": True}
async def __load_topic_definition(topic_name: str, current_user: User) -> Topic: topic = get_topic(topic_name, current_user) if topic is None: raise Exception(f"{topic_name} topic name does not exist") else: return topic