def run_pipeline(pipeline: Pipeline, data): pipeline_status = PipelineRunStatus(pipelineId=pipeline.pipelineId, uid=get_surrogate_key(), startTime=datetime.now(), topicId=pipeline.pipelineId) pipeline_status.oldValue = data[pipeline_constants.OLD] pipeline_status.newValue = data[pipeline_constants.NEW] if pipeline.enabled: pipeline_topic = get_topic_by_id(pipeline.topicId) log.info("start run pipeline {0}".format(pipeline.name)) context = {PIPELINE_UID: pipeline_status.uid} if __check_condition(pipeline, pipeline_topic, data): try: start = time.time() for stage in pipeline.stages: if __check_condition(stage, pipeline_topic, data): stage_run_status = StageRunStatus() stage_run_status.name = stage.name log.info("stage name {0}".format(stage.name)) for unit in stage.units: if unit.do is not None and __check_condition(unit, pipeline_topic, data): unit_run_status = UnitRunStatus() for action in unit.do: func = find_action_type_func(convert_action_type(action.type), action, pipeline_topic) # call dynamic action in action folder # TODO [future] custom folder out_result, unit_action_status = func(data, context) log.debug("out_result :{0}".format(out_result)) context = {**context, **out_result} unit_run_status.actions.append(unit_action_status) stage_run_status.units.append(unit_run_status) else: log.info("action stage unit {0} do is None".format(stage.name)) pipeline_status.stages.append(stage_run_status) elapsed_time = time.time() - start pipeline_status.completeTime = elapsed_time pipeline_status.status = FINISHED log.info("pipeline_status {0} time :{1}".format(pipeline.name, elapsed_time)) except Exception as e: log.exception(e) pipeline_status.error = traceback.format_exc() pipeline_status.status = ERROR log.error(pipeline_status) finally: if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM: log.debug("pipeline_status is {0}".format(pipeline_status)) else: print("sync pipeline monitor") watchmen.monitor.services.pipeline_monitor_service.sync_pipeline_monitor_data(pipeline_status)
def run_unit(context, data, pipeline_status, pipeline_topic, pipeline_trigger_merge_list, stage, stage_run_status): for unit in stage.units: if unit.do is not None: match_result = __check_condition(unit, pipeline_topic, data, context) if match_result: unit_run_status = UnitRunStatus() for action in unit.do: func = find_action_type_func( convert_action_type(action.type), action, pipeline_topic) # call dynamic action in action folder # TODO [future] custom folder out_result, unit_action_status, trigger_pipeline_data_list = func( data, context) if trigger_pipeline_data_list: pipeline_trigger_merge_list = [ *pipeline_trigger_merge_list, *trigger_pipeline_data_list ] log.info("out_result :{0}".format(out_result)) context = {**context, **out_result} # print("context : ",context) unit_run_status.actions.append(unit_action_status) stage_run_status.units.append(unit_run_status) else: log.debug("action stage unit {0} do is None".format(stage.name)) pipeline_status.stages.append(stage_run_status) return context, pipeline_trigger_merge_list
def run_pipeline(pipeline: Pipeline, data): pipeline_status = PipelineRunStatus( pipelineId=pipeline.pipelineId, uid=get_surrogate_key(), startTime=datetime.now().replace(tzinfo=None), topicId=pipeline.pipelineId) pipeline_status.oldValue = data[pipeline_constants.OLD] pipeline_status.newValue = data[pipeline_constants.NEW] if pipeline.enabled: pipeline_topic = get_topic_by_id(pipeline.topicId) log.info("start run pipeline {0}".format(pipeline.name)) context = {PIPELINE_UID: pipeline_status.uid} start = time.time() if __check_condition(pipeline, pipeline_topic, data, context): try: pipeline_trigger_merge_list = [] for stage in pipeline.stages: if __check_condition(stage, pipeline_topic, data, context): stage_run_status = StageRunStatus(name=stage.name) log.info("stage name {0}".format(stage.name)) context, pipeline_trigger_merge_list = run_unit( context, data, pipeline_status, pipeline_topic, pipeline_trigger_merge_list, stage, stage_run_status) elapsed_time = time.time() - start pipeline_status.completeTime = elapsed_time pipeline_status.status = FINISHED log.debug("pipeline_status {0} time :{1}".format( pipeline.name, elapsed_time)) if pipeline_topic.kind is None or pipeline_topic.kind != pipeline_constants.SYSTEM: __trigger_all_pipeline(pipeline_trigger_merge_list) except Exception as e: log.exception(e) pipeline_status.error = traceback.format_exc() pipeline_status.status = ERROR log.error(pipeline_status) finally: if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM: log.debug("pipeline_status is {0}".format(pipeline_status)) else: watchmen.monitor.services.pipeline_monitor_service.sync_pipeline_monitor_data( pipeline_status)
def alarm(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD] unit_action_status = UnitActionStatus() unit_action_status.type = action.type start = time.time() log.info("alert data") match_result = __check_condition(action, pipeline_topic, instance) if match_result: alarm_message = AlarmMessage(severity=action.severity) alarm_message.message = __build_message(action.message, pipeline_topic, raw_data, context) __sync_alarm_message(alarm_message) elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status