コード例 #1
0
 def __new__(
     cls,
     pipeline_name,
     task_id,
     handle=None,
     environment_dict=None,
     mode=None,
     step_keys=None,
     dag=None,
     instance_ref=None,
     op_kwargs=None,
 ):
     check_storage_specified(environment_dict)
     return super(DagsterOperatorParameters, cls).__new__(
         cls,
         handle=check.opt_inst_param(handle, 'handle',
                                     ExecutionTargetHandle),
         pipeline_name=check.str_param(pipeline_name, 'pipeline_name'),
         environment_dict=check.opt_dict_param(environment_dict,
                                               'environment_dict',
                                               key_type=str),
         mode=check.opt_str_param(mode, 'mode'),
         task_id=check.str_param(task_id, 'task_id'),
         step_keys=check.opt_list_param(step_keys, 'step_keys',
                                        of_type=str),
         dag=check.opt_inst_param(dag, 'dag', DAG),
         instance_ref=check.opt_inst_param(instance_ref, 'instance_ref',
                                           InstanceRef),
         op_kwargs=check.opt_dict_param(op_kwargs.copy(),
                                        'op_kwargs',
                                        key_type=str),
     )
コード例 #2
0
ファイル: factory.py プロジェクト: helloworld/dagster
    def __new__(
        cls,
        pipeline_name,
        task_id,
        recon_repo=None,
        run_config=None,
        mode=None,
        step_keys=None,
        dag=None,
        instance_ref=None,
        op_kwargs=None,
        pipeline_snapshot=None,
        execution_plan_snapshot=None,
        parent_pipeline_snapshot=None,
    ):
        pipeline_def = recon_repo.get_definition().get_pipeline(pipeline_name)

        if mode is None:
            mode = pipeline_def.get_default_mode_name()

        mode_def = pipeline_def.get_mode_definition(mode)

        check_storage_specified(pipeline_def, mode_def)

        return super(DagsterOperatorParameters, cls).__new__(
            cls,
            recon_repo=check.opt_inst_param(recon_repo, "recon_repo",
                                            ReconstructableRepository),
            pipeline_name=check.str_param(pipeline_name, "pipeline_name"),
            run_config=check.opt_dict_param(run_config,
                                            "run_config",
                                            key_type=str),
            mode=check.opt_str_param(mode, "mode"),
            task_id=check.str_param(task_id, "task_id"),
            step_keys=check.opt_list_param(step_keys, "step_keys",
                                           of_type=str),
            dag=check.opt_inst_param(dag, "dag", DAG),
            instance_ref=check.opt_inst_param(instance_ref, "instance_ref",
                                              InstanceRef),
            op_kwargs=check.opt_dict_param(op_kwargs.copy(),
                                           "op_kwargs",
                                           key_type=str),
            pipeline_snapshot=check.inst_param(pipeline_snapshot,
                                               "pipeline_snapshot",
                                               PipelineSnapshot),
            execution_plan_snapshot=check.inst_param(
                execution_plan_snapshot, "execution_plan_snapshot",
                ExecutionPlanSnapshot),
            parent_pipeline_snapshot=check.opt_inst_param(
                parent_pipeline_snapshot, "parent_pipeline_snapshot",
                PipelineSnapshot),
        )
コード例 #3
0
    def __new__(
        cls,
        pipeline_name,
        task_id,
        recon_repo=None,
        run_config=None,
        mode=None,
        step_keys=None,
        dag=None,
        instance_ref=None,
        op_kwargs=None,
        pipeline_snapshot=None,
        execution_plan_snapshot=None,
        parent_pipeline_snapshot=None,
        environment_dict=None,
    ):
        run_config = canonicalize_run_config(run_config,
                                             environment_dict)  # backcompact
        check_storage_specified(run_config)

        return super(DagsterOperatorParameters, cls).__new__(
            cls,
            recon_repo=check.opt_inst_param(recon_repo, 'recon_repo',
                                            ReconstructableRepository),
            pipeline_name=check.str_param(pipeline_name, 'pipeline_name'),
            run_config=check.opt_dict_param(run_config,
                                            'run_config',
                                            key_type=str),
            mode=check.opt_str_param(mode, 'mode'),
            task_id=check.str_param(task_id, 'task_id'),
            step_keys=check.opt_list_param(step_keys, 'step_keys',
                                           of_type=str),
            dag=check.opt_inst_param(dag, 'dag', DAG),
            instance_ref=check.opt_inst_param(instance_ref, 'instance_ref',
                                              InstanceRef),
            op_kwargs=check.opt_dict_param(op_kwargs.copy(),
                                           'op_kwargs',
                                           key_type=str),
            pipeline_snapshot=check.inst_param(pipeline_snapshot,
                                               'pipeline_snapshot',
                                               PipelineSnapshot),
            execution_plan_snapshot=check.inst_param(
                execution_plan_snapshot, 'execution_plan_snapshot',
                ExecutionPlanSnapshot),
            parent_pipeline_snapshot=check.opt_inst_param(
                parent_pipeline_snapshot, 'parent_pipeline_snapshot',
                PipelineSnapshot),
        )