def run_pipeline(pipeline: Pipeline, data):
pipeline_status = PipelineRunStatus(pipelineId=pipeline.pipelineId, uid=get_surrogate_key(),
startTime=datetime.now(), topicId=pipeline.pipelineId)
pipeline_status.oldValue = data[pipeline_constants.OLD]
pipeline_status.newValue = data[pipeline_constants.NEW]
if pipeline.enabled:
pipeline_topic = get_topic_by_id(pipeline.topicId)
log.info("start run pipeline {0}".format(pipeline.name))
context = {PIPELINE_UID: pipeline_status.uid}
if __check_condition(pipeline, pipeline_topic, data):
try:
start = time.time()
for stage in pipeline.stages:
if __check_condition(stage, pipeline_topic, data):
stage_run_status = StageRunStatus()
stage_run_status.name = stage.name
log.info("stage name {0}".format(stage.name))
for unit in stage.units:
if unit.do is not None and __check_condition(unit, pipeline_topic, data):
unit_run_status = UnitRunStatus()
for action in unit.do:
func = find_action_type_func(convert_action_type(action.type), action,
pipeline_topic)
# call dynamic action in action folder
# TODO [future] custom folder
out_result, unit_action_status = func(data, context)
log.debug("out_result :{0}".format(out_result))
context = {**context, **out_result}
unit_run_status.actions.append(unit_action_status)
stage_run_status.units.append(unit_run_status)
else:
log.info("action stage unit {0} do is None".format(stage.name))
pipeline_status.stages.append(stage_run_status)
elapsed_time = time.time() - start
pipeline_status.completeTime = elapsed_time
pipeline_status.status = FINISHED
log.info("pipeline_status {0} time :{1}".format(pipeline.name, elapsed_time))
except Exception as e:
log.exception(e)
pipeline_status.error = traceback.format_exc()
pipeline_status.status = ERROR
log.error(pipeline_status)
finally:
if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM:
log.debug("pipeline_status is {0}".format(pipeline_status))
else:
print("sync pipeline monitor")
watchmen.monitor.services.pipeline_monitor_service.sync_pipeline_monitor_data(pipeline_status)
def run_unit(context, data, pipeline_status, pipeline_topic,
pipeline_trigger_merge_list, stage, stage_run_status):
for unit in stage.units:
if unit.do is not None:
match_result = __check_condition(unit, pipeline_topic, data,
context)
if match_result:
unit_run_status = UnitRunStatus()
for action in unit.do:
func = find_action_type_func(
convert_action_type(action.type), action,
pipeline_topic)
# call dynamic action in action folder
# TODO [future] custom folder
out_result, unit_action_status, trigger_pipeline_data_list = func(
data, context)
if trigger_pipeline_data_list:
pipeline_trigger_merge_list = [
*pipeline_trigger_merge_list,
*trigger_pipeline_data_list
]
log.info("out_result :{0}".format(out_result))
context = {**context, **out_result}
# print("context : ",context)
unit_run_status.actions.append(unit_action_status)
stage_run_status.units.append(unit_run_status)
else:
log.debug("action stage unit {0} do is None".format(stage.name))
pipeline_status.stages.append(stage_run_status)
return context, pipeline_trigger_merge_list
def run_pipeline(pipeline: Pipeline, data):
pipeline_status = PipelineRunStatus(
pipelineId=pipeline.pipelineId,
uid=get_surrogate_key(),
startTime=datetime.now().replace(tzinfo=None),
topicId=pipeline.pipelineId)
pipeline_status.oldValue = data[pipeline_constants.OLD]
pipeline_status.newValue = data[pipeline_constants.NEW]
if pipeline.enabled:
pipeline_topic = get_topic_by_id(pipeline.topicId)
log.info("start run pipeline {0}".format(pipeline.name))
context = {PIPELINE_UID: pipeline_status.uid}
start = time.time()
if __check_condition(pipeline, pipeline_topic, data, context):
try:
pipeline_trigger_merge_list = []
for stage in pipeline.stages:
if __check_condition(stage, pipeline_topic, data, context):
stage_run_status = StageRunStatus(name=stage.name)
log.info("stage name {0}".format(stage.name))
context, pipeline_trigger_merge_list = run_unit(
context, data, pipeline_status, pipeline_topic,
pipeline_trigger_merge_list, stage,
stage_run_status)
elapsed_time = time.time() - start
pipeline_status.completeTime = elapsed_time
pipeline_status.status = FINISHED
log.debug("pipeline_status {0} time :{1}".format(
pipeline.name, elapsed_time))
if pipeline_topic.kind is None or pipeline_topic.kind != pipeline_constants.SYSTEM:
__trigger_all_pipeline(pipeline_trigger_merge_list)
except Exception as e:
log.exception(e)
pipeline_status.error = traceback.format_exc()
pipeline_status.status = ERROR
log.error(pipeline_status)
finally:
if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM:
log.debug("pipeline_status is {0}".format(pipeline_status))
else:
watchmen.monitor.services.pipeline_monitor_service.sync_pipeline_monitor_data(
pipeline_status)
def alarm(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD]
unit_action_status = UnitActionStatus()
unit_action_status.type = action.type
start = time.time()
log.info("alert data")
match_result = __check_condition(action, pipeline_topic, instance)
if match_result:
alarm_message = AlarmMessage(severity=action.severity)
alarm_message.message = __build_message(action.message, pipeline_topic, raw_data, context)
__sync_alarm_message(alarm_message)
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status