def should_run(unit_context: UnitContext,
unit_run_status: UnitRunStatus) -> bool:
unit = unit_context.unit
if unit.on is None:
unit_run_status.conditionResult = True
return True
current_data = unit_context.stageContext.pipelineContext.currentOfTriggerData
variables = unit_context.stageContext.pipelineContext.variables
condition_result = parse_parameter_joint(unit.on, current_data, variables)
unit_run_status.conditionResult = condition_result
return condition_result
def run_pipeline(pipeline: Pipeline, data):
pipeline_status = PipelineRunStatus(pipelineId=pipeline.pipelineId,
uid=get_surrogate_key(),
start_time=datetime.now())
if pipeline.enabled:
pipeline_topic = get_topic_by_id(pipeline.topicId)
# TODO pipeline when condition
log.info("start run pipeline {0}".format(pipeline.name))
context = {PIPELINE_UID: pipeline_status.uid}
try:
start = time.time()
for stage in pipeline.stages:
stage_run_status = StageRunStatus()
stage_run_status.name = stage.name
log.info("stage name {0}".format(stage.name))
for unit in stage.units:
# TODO __check_when_condition
# if unit.on is not None:
# result = __check_when_condition(unit.on.children, data)
# if result:
# continue
if unit.do is not None:
unit_run_status = UnitRunStatus()
for action in unit.do:
func = find_action_type_func(
convert_action_type(action.type), action,
pipeline_topic)
# call dynamic action in action folder
# TODO [future] custom folder
out_result, unit_action_status = func(
data, context)
log.debug("out_result :{0}".format(out_result))
context = {**context, **out_result}
unit_run_status.actions.append(unit_action_status)
stage_run_status.units.append(unit_run_status)
else:
log.info("action stage unit {0} do is None".format(
stage.name))
elapsed_time = time.time() - start
pipeline_status.stages.append(stage_run_status)
pipeline_status.complete_time = elapsed_time
pipeline_status.status = FINISHED
log.info("pipeline_status {0} time :{1}".format(
pipeline.name, elapsed_time))
except Exception as e:
log.exception(e)
pipeline_status.error = traceback.format_exc()
pipeline_status.status = ERROR
log.error(pipeline_status)
finally:
# log.info("insert_pipeline_monitor")
if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM:
log.info("pipeline_status is {0}".format(pipeline_status))
else:
sync_pipeline_monitor_data(pipeline_status)
def run_unit(context, data, pipeline_status, pipeline_topic,
pipeline_trigger_merge_list, stage, stage_run_status):
for unit in stage.units:
if unit.do is not None:
match_result = __check_condition(unit, pipeline_topic, data,
context)
if match_result:
unit_run_status = UnitRunStatus()
for action in unit.do:
func = find_action_type_func(
convert_action_type(action.type), action,
pipeline_topic)
# call dynamic action in action folder
# TODO [future] custom folder
out_result, unit_action_status, trigger_pipeline_data_list = func(
data, context)
if trigger_pipeline_data_list:
pipeline_trigger_merge_list = [
*pipeline_trigger_merge_list,
*trigger_pipeline_data_list
]
log.info("out_result :{0}".format(out_result))
context = {**context, **out_result}
# print("context : ",context)
unit_run_status.actions.append(unit_action_status)
stage_run_status.units.append(unit_run_status)
else:
log.debug("action stage unit {0} do is None".format(stage.name))
pipeline_status.stages.append(stage_run_status)
return context, pipeline_trigger_merge_list
def run_pipeline(pipeline: Pipeline, data):
pipeline_status = PipelineRunStatus(pipelineId=pipeline.pipelineId, uid=get_surrogate_key(),
startTime=datetime.now(), topicId=pipeline.pipelineId)
pipeline_status.oldValue = data[pipeline_constants.OLD]
pipeline_status.newValue = data[pipeline_constants.NEW]
if pipeline.enabled:
pipeline_topic = get_topic_by_id(pipeline.topicId)
log.info("start run pipeline {0}".format(pipeline.name))
context = {PIPELINE_UID: pipeline_status.uid}
if __check_condition(pipeline, pipeline_topic, data):
try:
start = time.time()
for stage in pipeline.stages:
if __check_condition(stage, pipeline_topic, data):
stage_run_status = StageRunStatus()
stage_run_status.name = stage.name
log.info("stage name {0}".format(stage.name))
for unit in stage.units:
if unit.do is not None and __check_condition(unit, pipeline_topic, data):
unit_run_status = UnitRunStatus()
for action in unit.do:
func = find_action_type_func(convert_action_type(action.type), action,
pipeline_topic)
# call dynamic action in action folder
# TODO [future] custom folder
out_result, unit_action_status = func(data, context)
log.debug("out_result :{0}".format(out_result))
context = {**context, **out_result}
unit_run_status.actions.append(unit_action_status)
stage_run_status.units.append(unit_run_status)
else:
log.info("action stage unit {0} do is None".format(stage.name))
pipeline_status.stages.append(stage_run_status)
elapsed_time = time.time() - start
pipeline_status.completeTime = elapsed_time
pipeline_status.status = FINISHED
log.info("pipeline_status {0} time :{1}".format(pipeline.name, elapsed_time))
except Exception as e:
log.exception(e)
pipeline_status.error = traceback.format_exc()
pipeline_status.status = ERROR
log.error(pipeline_status)
finally:
if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM:
log.debug("pipeline_status is {0}".format(pipeline_status))
else:
print("sync pipeline monitor")
watchmen.monitor.services.pipeline_monitor_service.sync_pipeline_monitor_data(pipeline_status)
def __init__(self, stageContext, unit):
self.stageContext = stageContext
self.unit = unit
self.unitStatus = UnitRunStatus()