def test_all_airline_pipelines():
download_config_object = load_yaml_from_globs(
script_relative_path('../../environments/local_base.yml'),
script_relative_path('../../environments/local_fast_download.yml'),
)
result_download = execute_pipeline(define_airline_demo_download_pipeline(),
download_config_object)
assert result_download.success
ingest_config_object = load_yaml_from_globs(
script_relative_path('../../environments/local_base.yml'),
script_relative_path('../../environments/local_fast_ingest.yml'),
)
result_ingest = execute_pipeline(define_airline_demo_ingest_pipeline(),
ingest_config_object)
assert result_ingest.success
warehouse_config_object = load_yaml_from_globs(
script_relative_path('../../environments/local_base.yml'),
script_relative_path('../../environments/local_fast_warehouse.yml'),
)
result_warehouse = execute_pipeline(
define_airline_demo_warehouse_pipeline(), warehouse_config_object)
assert result_warehouse.success
def test_airline_pipeline_1_warehouse(docker_compose_db):
warehouse_config_object = load_yaml_from_globs(
script_relative_path('../environments/local_base.yml'),
script_relative_path('../environments/local_warehouse.yml'),
)
result_warehouse = execute_pipeline(
define_airline_demo_warehouse_pipeline(), warehouse_config_object)
assert result_warehouse.success
def test_pipeline_warehouse():
now = datetime.datetime.utcnow()
timestamp = now.strftime('%Y_%m_%d_%H_%M_%S')
result = execute_pipeline(
define_airline_demo_warehouse_pipeline(),
{
'context': {
'local': {
'resources': {
'db_info': {
'config': {
'postgres_username': '******',
'postgres_password': '******',
'postgres_hostname': 'localhost',
'postgres_db_name': 'test',
}
}
}
}
},
'solids': {
'db_url': {
'config': 'postgresql://*****:*****@127.0.0.1:5432/test'
},
'upload_outbound_avg_delay_pdf_plots': {
'config': {
'bucket':
'dagster-airline-demo-sink',
'key':
'sfo_outbound_avg_delay_plots_{timestamp}.pdf'.format(
timestamp=timestamp),
}
},
'upload_delays_vs_fares_pdf_plots': {
'config': {
'bucket':
'dagster-airline-demo-sink',
'key':
'delays_vs_fares_{timestamp}.pdf'.format(
timestamp=timestamp),
}
},
'upload_delays_by_geography_pdf_plots': {
'config': {
'bucket':
'dagster-airline-demo-sink',
'key':
'delays_by_geography_{timestamp}.pdf'.format(
timestamp=timestamp),
}
},
},
},
)
assert result.success
class TestAirflowPython_1WarehouseExecution:
pipeline = define_airline_demo_warehouse_pipeline()
config_yaml = [
script_relative_path(
os.path.join('..', 'environments', 'local_base.yml')),
script_relative_path(
os.path.join('..', 'environments', 'local_airflow.yml')),
script_relative_path(
os.path.join('..', 'environments', 'local_warehouse.yml')),
]
def test_airflow_run_warehouse_pipeline(
self, dagster_airflow_python_operator_pipeline):
pass
def test_construct_warehouse_pipeline():
assert define_airline_demo_warehouse_pipeline()
