async def load_pipeline(topic_id, current_user: User = Depends(deps.get_current_user)):
# pipeline_list_produce = []
result = load_pipeline_by_topic_id(topic_id)
pipeline_list_produce = [*result]
for pipeline in result:
for stage in pipeline.stages:
for unit in stage.units:
for action in unit.do:
result = load_pipeline_by_topic_id(action.topicId)
pipeline_list_produce = [*pipeline_list_produce, *result]
return {"topicId": topic_id, "consume": [], "produce": pipeline_list_produce}
async def rerun_pipeline(topic_name, instance_id, pipeline_id):
instance = find_topic_data_by_id_and_topic_name(topic_name, instance_id)
topic = get_topic(topic_name)
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in pipeline_list:
if pipeline.pipelineId == pipeline_id:
log.info("rerun topic {0} and pipeline {1}".format(topic_name, pipeline.pipelineId))
run_pipeline(pipeline, instance)
return {"received": True}
def trigger_pipeline(topic_name, instance, trigger_type: TriggerType):
log.info("trigger_pipeline topic_name :{0}".format(topic_name))
topic = get_topic(topic_name)
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in pipeline_list:
if __match_trigger_type(trigger_type, pipeline):
# log.info("pipeline run: {0}".format(pipeline.json()))
run_pipeline(pipeline, instance)
def trigger_pipeline(topic_name, instance, trigger_type: TriggerType):
log.info("trigger_pipeline topic_name :{0}".format(topic_name))
topic = get_topic(topic_name)
# TODO validate data with topic schema
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
# futures =[]
for pipeline in pipeline_list:
if __match_trigger_type(trigger_type, pipeline):
log.debug("pipeline run: {0}".format(pipeline.json()))
run_pipeline(pipeline, instance)
def trigger_pipeline_2(topic_name,
instance,
trigger_type: TriggerType,
current_user=None,
trace_id=None):
topic = get_topic(topic_name, current_user)
pipeline_list = load_pipeline_by_topic_id(topic.topicId, current_user)
for pipeline in pipeline_list:
if __match_trigger_type(trigger_type, pipeline):
pipeline_context = PipelineContext(pipeline, instance,
current_user, trace_id)
run_pipeline(pipeline_context, current_user)
async def rerun_pipeline(topic_name,
instance_id,
pipeline_id=None,
current_user: User = Depends(deps.get_current_user)):
topic = get_topic(topic_name)
trace_id = get_surrogate_key()
instance = find_topic_data_by_id_and_topic_name(topic, instance_id)
data = {"new": instance, "old": None}
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in find_execute_pipeline_list(pipeline_id, pipeline_list):
log.info("rerun topic {0} and pipeline {1}".format(
topic_name, pipeline.name))
pipeline_context = PipelineContext(pipeline, data, current_user,
trace_id)
run_pipeline(pipeline_context, current_user)
return {"received": True, "trace_id": trace_id}
async def rerun_pipeline(topic_name,
instance_id,
pipeline_id,
current_user: User = Depends(deps.get_current_user)):
topic = get_topic(topic_name)
trace_id = get_surrogate_key()
instance = find_topic_data_by_id_and_topic_name(topic, instance_id)
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in pipeline_list:
if pipeline.pipelineId == pipeline_id:
log.info("rerun topic {0} and pipeline {1}".format(
topic_name, pipeline.pipelineId))
pipeline_context = PipelineContext(pipeline, instance,
current_user, trace_id)
run_pipeline(pipeline_context, current_user)
return {"received": True}
