Example #1
0
def test_all_airline_pipelines():
    download_config_object = load_yaml_from_globs(
        script_relative_path('../../environments/local_base.yml'),
        script_relative_path('../../environments/local_fast_download.yml'),
    )

    result_download = execute_pipeline(define_airline_demo_download_pipeline(),
                                       download_config_object)

    assert result_download.success

    ingest_config_object = load_yaml_from_globs(
        script_relative_path('../../environments/local_base.yml'),
        script_relative_path('../../environments/local_fast_ingest.yml'),
    )

    result_ingest = execute_pipeline(define_airline_demo_ingest_pipeline(),
                                     ingest_config_object)

    assert result_ingest.success

    warehouse_config_object = load_yaml_from_globs(
        script_relative_path('../../environments/local_base.yml'),
        script_relative_path('../../environments/local_fast_warehouse.yml'),
    )

    result_warehouse = execute_pipeline(
        define_airline_demo_warehouse_pipeline(), warehouse_config_object)
    assert result_warehouse.success
Example #2
0
def test_airline_pipeline_1_warehouse(docker_compose_db):
    warehouse_config_object = load_yaml_from_globs(
        script_relative_path('../environments/local_base.yml'),
        script_relative_path('../environments/local_warehouse.yml'),
    )

    result_warehouse = execute_pipeline(
        define_airline_demo_warehouse_pipeline(), warehouse_config_object)
    assert result_warehouse.success
def test_pipeline_warehouse():
    now = datetime.datetime.utcnow()
    timestamp = now.strftime('%Y_%m_%d_%H_%M_%S')
    result = execute_pipeline(
        define_airline_demo_warehouse_pipeline(),
        {
            'context': {
                'local': {
                    'resources': {
                        'db_info': {
                            'config': {
                                'postgres_username': '******',
                                'postgres_password': '******',
                                'postgres_hostname': 'localhost',
                                'postgres_db_name': 'test',
                            }
                        }
                    }
                }
            },
            'solids': {
                'db_url': {
                    'config': 'postgresql://*****:*****@127.0.0.1:5432/test'
                },
                'upload_outbound_avg_delay_pdf_plots': {
                    'config': {
                        'bucket':
                        'dagster-airline-demo-sink',
                        'key':
                        'sfo_outbound_avg_delay_plots_{timestamp}.pdf'.format(
                            timestamp=timestamp),
                    }
                },
                'upload_delays_vs_fares_pdf_plots': {
                    'config': {
                        'bucket':
                        'dagster-airline-demo-sink',
                        'key':
                        'delays_vs_fares_{timestamp}.pdf'.format(
                            timestamp=timestamp),
                    }
                },
                'upload_delays_by_geography_pdf_plots': {
                    'config': {
                        'bucket':
                        'dagster-airline-demo-sink',
                        'key':
                        'delays_by_geography_{timestamp}.pdf'.format(
                            timestamp=timestamp),
                    }
                },
            },
        },
    )

    assert result.success
Example #4
0
class TestAirflowPython_1WarehouseExecution:
    pipeline = define_airline_demo_warehouse_pipeline()
    config_yaml = [
        script_relative_path(
            os.path.join('..', 'environments', 'local_base.yml')),
        script_relative_path(
            os.path.join('..', 'environments', 'local_airflow.yml')),
        script_relative_path(
            os.path.join('..', 'environments', 'local_warehouse.yml')),
    ]

    def test_airflow_run_warehouse_pipeline(
            self, dagster_airflow_python_operator_pipeline):
        pass
Example #5
0
def test_construct_warehouse_pipeline():
    assert define_airline_demo_warehouse_pipeline()