def should_run(unit_context: UnitContext,
               unit_run_status: UnitRunStatus) -> bool:
    unit = unit_context.unit
    if unit.on is None:
        unit_run_status.conditionResult = True
        return True
    current_data = unit_context.stageContext.pipelineContext.currentOfTriggerData
    variables = unit_context.stageContext.pipelineContext.variables
    condition_result = parse_parameter_joint(unit.on, current_data, variables)
    unit_run_status.conditionResult = condition_result
    return condition_result
Exemplo n.º 2
0
def run_pipeline(pipeline: Pipeline, data):
    pipeline_status = PipelineRunStatus(pipelineId=pipeline.pipelineId,
                                        uid=get_surrogate_key(),
                                        start_time=datetime.now())

    if pipeline.enabled:
        pipeline_topic = get_topic_by_id(pipeline.topicId)
        # TODO pipeline when  condition
        log.info("start run pipeline {0}".format(pipeline.name))
        context = {PIPELINE_UID: pipeline_status.uid}

        try:
            start = time.time()
            for stage in pipeline.stages:
                stage_run_status = StageRunStatus()
                stage_run_status.name = stage.name
                log.info("stage name {0}".format(stage.name))
                for unit in stage.units:
                    # TODO __check_when_condition
                    # if unit.on is not None:
                    #     result = __check_when_condition(unit.on.children, data)
                    #     if result:
                    #         continue

                    if unit.do is not None:
                        unit_run_status = UnitRunStatus()
                        for action in unit.do:
                            func = find_action_type_func(
                                convert_action_type(action.type), action,
                                pipeline_topic)
                            # call dynamic action in action folder
                            # TODO [future] custom folder
                            out_result, unit_action_status = func(
                                data, context)
                            log.debug("out_result :{0}".format(out_result))
                            context = {**context, **out_result}
                            unit_run_status.actions.append(unit_action_status)
                        stage_run_status.units.append(unit_run_status)
                    else:
                        log.info("action stage unit  {0} do is None".format(
                            stage.name))

            elapsed_time = time.time() - start
            pipeline_status.stages.append(stage_run_status)
            pipeline_status.complete_time = elapsed_time
            pipeline_status.status = FINISHED
            log.info("pipeline_status {0} time :{1}".format(
                pipeline.name, elapsed_time))

        except Exception as e:
            log.exception(e)
            pipeline_status.error = traceback.format_exc()
            pipeline_status.status = ERROR
            log.error(pipeline_status)
        finally:
            # log.info("insert_pipeline_monitor")
            if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM:
                log.info("pipeline_status is {0}".format(pipeline_status))
            else:
                sync_pipeline_monitor_data(pipeline_status)
Exemplo n.º 3
0
def run_unit(context, data, pipeline_status, pipeline_topic,
             pipeline_trigger_merge_list, stage, stage_run_status):
    for unit in stage.units:
        if unit.do is not None:
            match_result = __check_condition(unit, pipeline_topic, data,
                                             context)
            if match_result:
                unit_run_status = UnitRunStatus()
                for action in unit.do:

                    func = find_action_type_func(
                        convert_action_type(action.type), action,
                        pipeline_topic)
                    # call dynamic action in action folder
                    # TODO [future] custom folder
                    out_result, unit_action_status, trigger_pipeline_data_list = func(
                        data, context)

                    if trigger_pipeline_data_list:
                        pipeline_trigger_merge_list = [
                            *pipeline_trigger_merge_list,
                            *trigger_pipeline_data_list
                        ]

                    log.info("out_result :{0}".format(out_result))
                    context = {**context, **out_result}
                    # print("context : ",context)
                    unit_run_status.actions.append(unit_action_status)
                stage_run_status.units.append(unit_run_status)
        else:
            log.debug("action stage unit  {0} do is None".format(stage.name))
    pipeline_status.stages.append(stage_run_status)
    return context, pipeline_trigger_merge_list
def run_pipeline(pipeline: Pipeline, data):
    pipeline_status = PipelineRunStatus(pipelineId=pipeline.pipelineId, uid=get_surrogate_key(),
                                        startTime=datetime.now(), topicId=pipeline.pipelineId)
    pipeline_status.oldValue = data[pipeline_constants.OLD]
    pipeline_status.newValue = data[pipeline_constants.NEW]

    if pipeline.enabled:
        pipeline_topic = get_topic_by_id(pipeline.topicId)
        log.info("start run pipeline {0}".format(pipeline.name))
        context = {PIPELINE_UID: pipeline_status.uid}
        if __check_condition(pipeline, pipeline_topic, data):
            try:
                start = time.time()
                for stage in pipeline.stages:
                    if __check_condition(stage, pipeline_topic, data):
                        stage_run_status = StageRunStatus()
                        stage_run_status.name = stage.name
                        log.info("stage name {0}".format(stage.name))
                        for unit in stage.units:
                            if unit.do is not None and __check_condition(unit, pipeline_topic, data):
                                unit_run_status = UnitRunStatus()
                                for action in unit.do:
                                    func = find_action_type_func(convert_action_type(action.type), action,
                                                                 pipeline_topic)
                                    # call dynamic action in action folder
                                    # TODO [future] custom folder
                                    out_result, unit_action_status = func(data, context)
                                    log.debug("out_result :{0}".format(out_result))
                                    context = {**context, **out_result}
                                    unit_run_status.actions.append(unit_action_status)
                                stage_run_status.units.append(unit_run_status)
                            else:
                                log.info("action stage unit  {0} do is None".format(stage.name))
                        pipeline_status.stages.append(stage_run_status)
                elapsed_time = time.time() - start
                pipeline_status.completeTime = elapsed_time
                pipeline_status.status = FINISHED
                log.info("pipeline_status {0} time :{1}".format(pipeline.name, elapsed_time))

            except Exception as e:
                log.exception(e)
                pipeline_status.error = traceback.format_exc()
                pipeline_status.status = ERROR
                log.error(pipeline_status)
            finally:
                if pipeline_topic.kind is not None and pipeline_topic.kind == pipeline_constants.SYSTEM:
                    log.debug("pipeline_status is {0}".format(pipeline_status))
                else:
                    print("sync pipeline monitor")
                    watchmen.monitor.services.pipeline_monitor_service.sync_pipeline_monitor_data(pipeline_status)
 def __init__(self, stageContext, unit):
     self.stageContext = stageContext
     self.unit = unit
     self.unitStatus = UnitRunStatus()