async def import_raw_topic_data(topic_event):
topic = get_topic(topic_event.code)
if topic is None:
raise Exception(topic_event.code + " topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
await __trigger_pipeline(topic_event)
def trigger_pipeline(topic_name, instance, trigger_type: TriggerType):
log.info("trigger_pipeline topic_name :{0}".format(topic_name))
topic = get_topic(topic_name)
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in pipeline_list:
if __match_trigger_type(trigger_type, pipeline):
# log.info("pipeline run: {0}".format(pipeline.json()))
run_pipeline(pipeline, instance)
async def rerun_pipeline(topic_name, instance_id, pipeline_id):
instance = find_topic_data_by_id_and_topic_name(topic_name, instance_id)
topic = get_topic(topic_name)
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in pipeline_list:
if pipeline.pipelineId == pipeline_id:
log.info("rerun topic {0} and pipeline {1}".format(topic_name, pipeline.pipelineId))
run_pipeline(pipeline, instance)
return {"received": True}
async def save_topic_data(topic_event: TopicEvent):
# TODO user check URP
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
await __trigger_pipeline(topic_event)
return {"received": True}
def trigger_pipeline(topic_name, instance, trigger_type: TriggerType):
log.info("trigger_pipeline topic_name :{0}".format(topic_name))
topic = get_topic(topic_name)
# TODO validate data with topic schema
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
# futures =[]
for pipeline in pipeline_list:
if __match_trigger_type(trigger_type, pipeline):
log.debug("pipeline run: {0}".format(pipeline.json()))
run_pipeline(pipeline, instance)
def sync_alarm_message(alarm: AlarmMessage):
topic_event = TopicEvent(code="alarm", data=alarm.dict())
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
trigger_pipeline(topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
asyncio.ensure_future(send_notifier(alarm))
async def import_raw_topic_data(topic_event, current_user):
topic = get_topic(topic_event.code, current_user)
if topic is None:
raise Exception(topic_event.code + " topic name does not exist")
raw_data = await get_input_data(topic, topic_event)
add_audit_columns(raw_data, INSERT)
flatten_fields = get_flatten_field(topic_event.data, topic.factors)
raw_data.update(flatten_fields)
save_topic_instance(topic, raw_data, current_user)
__trigger_pipeline(topic_event, current_user)
def trigger_pipeline_2(topic_name,
instance,
trigger_type: TriggerType,
current_user=None,
trace_id=None):
topic = get_topic(topic_name, current_user)
pipeline_list = load_pipeline_by_topic_id(topic.topicId, current_user)
for pipeline in pipeline_list:
if __match_trigger_type(trigger_type, pipeline):
pipeline_context = PipelineContext(pipeline, instance,
current_user, trace_id)
run_pipeline(pipeline_context, current_user)
def sync_pipeline_monitor_data(pipeline_monitor: PipelineRunStatus):
topic_event = TopicEvent(code="raw_pipeline_monitor",
data=pipeline_monitor.dict())
# asyncio.ensure_future(import_raw_topic_data(topic_event))
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
watchmen.pipeline.index.trigger_pipeline(
topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
async def rerun_pipeline(topic_name,
instance_id,
pipeline_id=None,
current_user: User = Depends(deps.get_current_user)):
topic = get_topic(topic_name)
trace_id = get_surrogate_key()
instance = find_topic_data_by_id_and_topic_name(topic, instance_id)
data = {"new": instance, "old": None}
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in find_execute_pipeline_list(pipeline_id, pipeline_list):
log.info("rerun topic {0} and pipeline {1}".format(
topic_name, pipeline.name))
pipeline_context = PipelineContext(pipeline, data, current_user,
trace_id)
run_pipeline(pipeline_context, current_user)
return {"received": True, "trace_id": trace_id}
async def rerun_pipeline(topic_name,
instance_id,
pipeline_id,
current_user: User = Depends(deps.get_current_user)):
topic = get_topic(topic_name)
trace_id = get_surrogate_key()
instance = find_topic_data_by_id_and_topic_name(topic, instance_id)
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in pipeline_list:
if pipeline.pipelineId == pipeline_id:
log.info("rerun topic {0} and pipeline {1}".format(
topic_name, pipeline.pipelineId))
pipeline_context = PipelineContext(pipeline, instance,
current_user, trace_id)
run_pipeline(pipeline_context, current_user)
return {"received": True}
async def __load_topic_definition(topic_name: str, current_user: User) -> Topic:
topic = get_topic(topic_name, current_user)
if topic is None:
raise Exception(f"{topic_name} topic name does not exist")
else:
return topic
