Python dagster_airflow_docker_operator_pipeline Examples

Example #1

def test_s3_storage(dagster_airflow_docker_operator_pipeline,
                    dagster_docker_image):  # pylint: disable=redefined-outer-name
    pipeline_name = 'demo_pipeline'
    environments_path = test_project_environments_path()
    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        handle=ReconstructableRepository.for_module('test_pipelines.repo',
                                                    pipeline_name),
        environment_yaml=[
            os.path.join(environments_path, 'env.yaml'),
            os.path.join(environments_path, 'env_s3.yaml'),
        ],
        image=dagster_docker_image,
    )
    validate_pipeline_execution(results)

Example #2

def test_skip_operator(dagster_airflow_docker_operator_pipeline,
                       dagster_docker_image):  # pylint: disable=redefined-outer-name
    pipeline_name = 'optional_outputs'
    environments_path = test_project_environments_path()
    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        handle=ReconstructableRepository.for_module('test_pipelines.repo',
                                                    pipeline_name),
        environment_yaml=[
            os.path.join(environments_path, 'env_filesystem.yaml')
        ],
        op_kwargs={'host_tmp_dir': '/tmp'},
        image=dagster_docker_image,
    )
    validate_skip_pipeline_execution(results)

Example #3

def test_fs_storage_no_explicit_base_dir(
        dagster_airflow_docker_operator_pipeline, dagster_docker_image):  # pylint: disable=redefined-outer-name
    pipeline_name = 'demo_pipeline'
    environments_path = test_project_environments_path()
    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        handle=ExecutionTargetHandle.for_pipeline_module(
            'test_pipelines.repo', pipeline_name),
        environment_yaml=[
            os.path.join(environments_path, 'env.yaml'),
            os.path.join(environments_path,
                         'env_filesystem_no_explicit_base_dir.yaml'),
        ],
        image=dagster_docker_image,
    )
    validate_pipeline_execution(results)

Example #4

def test_gcs_storage(
    dagster_airflow_docker_operator_pipeline,
    dagster_docker_image,
    environments_path,
):  # pylint: disable=redefined-outer-name
    pipeline_name = 'demo_pipeline_gcs'
    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        handle=ExecutionTargetHandle.for_pipeline_module(
            'test_pipelines.repo', pipeline_name),
        environment_yaml=[
            os.path.join(environments_path, 'env.yaml'),
            os.path.join(environments_path, 'env_gcs.yaml'),
        ],
        image=dagster_docker_image,
    )
    validate_pipeline_execution(results)

Example #5

def test_s3_storage(dagster_airflow_docker_operator_pipeline,
                    dagster_docker_image):  # pylint: disable=redefined-outer-name
    pipeline_name = "demo_pipeline"
    environments_path = test_project_environments_path()
    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        recon_repo=ReconstructableRepository.for_module(
            "dagster_test.test_project.test_pipelines.repo",
            "define_demo_execution_repo",
        ),
        environment_yaml=[
            os.path.join(environments_path, "env.yaml"),
            os.path.join(environments_path, "env_s3.yaml"),
        ],
        image=dagster_docker_image,
    )
    validate_pipeline_execution(results)

Example #6

def test_skip_operator(dagster_airflow_docker_operator_pipeline,
                       dagster_docker_image):  # pylint: disable=redefined-outer-name
    pipeline_name = "optional_outputs"
    environments_path = test_project_environments_path()
    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        recon_repo=ReconstructableRepository.for_module(
            "dagster_test.test_project.test_pipelines.repo",
            "define_demo_execution_repo",
        ),
        environment_yaml=[
            os.path.join(environments_path, "env_filesystem.yaml")
        ],
        op_kwargs={"host_tmp_dir": "/tmp"},
        image=dagster_docker_image,
    )
    validate_skip_pipeline_execution(results)

Example #7

def test_skip_operator(
    dagster_airflow_docker_operator_pipeline,
    dagster_docker_image,
    environments_path,
):  # pylint: disable=redefined-outer-name
    pipeline_name = 'optional_outputs'

    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        handle=ExecutionTargetHandle.for_pipeline_module(
            'test_pipelines', pipeline_name),
        environment_yaml=[
            os.path.join(environments_path, 'env_filesystem.yaml')
        ],
        op_kwargs={'host_tmp_dir': '/tmp'},
        image=dagster_docker_image,
    )
    validate_skip_pipeline_execution(results)

Example #8

File: test_docker.py Project: varokas/dagster-1

def test_fs_storage_no_explicit_base_dir(
        dagster_airflow_docker_operator_pipeline, dagster_docker_image):  # pylint: disable=redefined-outer-name
    pipeline_name = 'demo_pipeline'
    environments_path = test_project_environments_path()
    results = dagster_airflow_docker_operator_pipeline(
        pipeline_name=pipeline_name,
        recon_repo=ReconstructableRepository.for_module(
            'dagster_test.test_project.test_pipelines.repo',
            'define_demo_execution_repo',
        ),
        environment_yaml=[
            os.path.join(environments_path, 'env.yaml'),
            os.path.join(environments_path,
                         'env_filesystem_no_explicit_base_dir.yaml'),
        ],
        image=dagster_docker_image,
    )
    validate_pipeline_execution(results)