def test_all_airline_pipelines():
download_config_object = load_yaml_from_globs(
script_relative_path('../../environments/local_base.yml'),
script_relative_path('../../environments/local_fast_download.yml'),
)
result_download = execute_pipeline(define_airline_demo_download_pipeline(),
download_config_object)
assert result_download.success
ingest_config_object = load_yaml_from_globs(
script_relative_path('../../environments/local_base.yml'),
script_relative_path('../../environments/local_fast_ingest.yml'),
)
result_ingest = execute_pipeline(define_airline_demo_ingest_pipeline(),
ingest_config_object)
assert result_ingest.success
warehouse_config_object = load_yaml_from_globs(
script_relative_path('../../environments/local_base.yml'),
script_relative_path('../../environments/local_fast_warehouse.yml'),
)
result_warehouse = execute_pipeline(
define_airline_demo_warehouse_pipeline(), warehouse_config_object)
assert result_warehouse.success
def test_ingest_pipeline_fast(postgres, pg_hostname):
with tempfile.TemporaryDirectory() as temp_dir:
with instance_for_test() as instance:
ingest_config_dict = load_yaml_from_globs(
config_path("test_base.yaml"),
config_path("local_fast_ingest.yaml"),
)
ingest_config_dict["resources"]["io_manager"] = {
"config": {
"base_dir": temp_dir
}
}
ingest_config_dict["resources"]["pyspark_io_manager"] = {
"config": {
"base_dir": temp_dir
}
}
result_ingest = execute_pipeline(
pipeline=ingest_pipeline,
mode="local",
run_config=ingest_config_dict,
instance=instance,
)
assert result_ingest.success
def test_airline_pipeline_1_warehouse(docker_compose_db):
warehouse_config_object = load_yaml_from_globs(
config_path('local_base.yaml'), config_path('local_warehouse.yaml'))
warehouse_config_object = enviroment_overrides(warehouse_config_object)
result_warehouse = execute_pipeline(warehouse_pipeline_def,
warehouse_config_object,
run_config=RunConfig(mode='local'))
assert result_warehouse.success
def test_airline_pipeline_1_warehouse(docker_compose_db):
warehouse_config_object = load_yaml_from_globs(
script_relative_path('../environments/local_base.yml'),
script_relative_path('../environments/local_warehouse.yml'),
)
result_warehouse = execute_pipeline(
define_airline_demo_warehouse_pipeline(), warehouse_config_object)
assert result_warehouse.success
def test_airline_pipeline_s3_1_warehouse(docker_compose_db):
warehouse_config_object = load_yaml_from_globs(
config_path('local_base.yaml'),
config_path('local_airflow.yaml'),
config_path('local_warehouse.yaml'),
)
result_warehouse = execute_pipeline(warehouse_pipeline_def, warehouse_config_object)
assert result_warehouse.success
def test_ingest_pipeline_fast(docker_compose_db):
ingest_config_dict = load_yaml_from_globs(
config_path('local_base.yaml'), config_path('local_fast_ingest.yaml'))
ingest_config_dict = enviroment_overrides(ingest_config_dict)
result_ingest = execute_pipeline(ingest_pipeline_def,
ingest_config_dict,
run_config=RunConfig(mode='local'))
assert result_ingest.success
def test_airline_pipeline_1_warehouse(postgres, pg_hostname):
warehouse_config_object = load_yaml_from_globs(
config_path('test_base.yaml'), config_path('local_warehouse.yaml'))
result_warehouse = execute_pipeline(
warehouse_pipeline_def,
warehouse_config_object,
run_config=RunConfig(mode='local'),
instance=DagsterInstance.local_temp(),
)
assert result_warehouse.success
def test_airline_pipeline_s3_0_ingest(docker_compose_db):
ingest_config_dict = load_yaml_from_globs(
config_path('local_base.yaml'),
config_path('local_airflow.yaml'),
config_path('local_fast_ingest.yaml'),
)
result_ingest = execute_pipeline(ingest_pipeline_def, ingest_config_dict)
assert result_ingest.success
def test_airline_pipeline_0_ingest(docker_compose_db):
ingest_config_object = load_yaml_from_globs(
script_relative_path('../environments/local_base.yml'),
script_relative_path('../environments/local_fast_ingest.yml'),
)
result_ingest = execute_pipeline(define_airline_demo_ingest_pipeline(),
ingest_config_object)
assert result_ingest.success
def test_airline_pipeline_1_warehouse(postgres, pg_hostname):
warehouse_config_object = load_yaml_from_globs(
config_path('test_base.yaml'), config_path('local_warehouse.yaml'))
result_warehouse = execute_pipeline_with_mode(
pipeline=warehouse_pipeline_def,
mode='local',
environment_dict=warehouse_config_object,
instance=DagsterInstance.local_temp(),
)
assert result_warehouse.success
def test_airline_pipeline_1_warehouse(postgres, pg_hostname):
warehouse_config_object = load_yaml_from_globs(
config_path("test_base.yaml"), config_path("local_warehouse.yaml"))
result_warehouse = execute_pipeline(
pipeline=warehouse_pipeline,
mode="local",
run_config=warehouse_config_object,
instance=DagsterInstance.local_temp(),
)
assert result_warehouse.success
def test_ingest_pipeline_fast(postgres, pg_hostname):
ingest_config_dict = load_yaml_from_globs(
config_path("test_base.yaml"), config_path("local_fast_ingest.yaml"))
result_ingest = execute_pipeline(
pipeline=ingest_pipeline,
mode="local",
run_config=ingest_config_dict,
instance=DagsterInstance.local_temp(),
)
assert result_ingest.success
def test_airline_pipeline_s3_1_warehouse(postgres, pg_hostname):
warehouse_config_object = load_yaml_from_globs(
config_path('test_base.yaml'),
config_path('s3_storage.yaml'),
config_path('local_warehouse.yaml'),
)
result_warehouse = execute_pipeline(
warehouse_pipeline, warehouse_config_object, instance=DagsterInstance.local_temp()
)
assert result_warehouse.success
def test_ingest_pipeline_fast(postgres, pg_hostname):
ingest_config_dict = load_yaml_from_globs(
config_path('test_base.yaml'), config_path('local_fast_ingest.yaml'))
result_ingest = execute_pipeline_with_mode(
pipeline=ingest_pipeline_def,
mode='local',
environment_dict=ingest_config_dict,
instance=DagsterInstance.local_temp(),
)
assert result_ingest.success
def test_ingest_pipeline_fast(postgres, pg_hostname):
ingest_config_dict = load_yaml_from_globs(
config_path('test_base.yaml'), config_path('local_fast_ingest.yaml'))
result_ingest = execute_pipeline(
ingest_pipeline_def,
ingest_config_dict,
run_config=RunConfig(mode='local'),
instance=DagsterInstance.local_temp(),
)
assert result_ingest.success
def run_pipeline(self,
pipeline: PipelineDefinition,
config_name: str,
extra_config: dict[str, Any] = {},
pipeline_mode='test') -> PipelineExecutionResult:
config_dict = load_yaml_from_globs(config_path(config_name))
config_dict = deep_merge_dicts(config_dict, extra_config)
return execute_pipeline(pipeline,
run_config=config_dict,
mode=pipeline_mode)
def test_airline_pipeline_s3_0_ingest(postgres, pg_hostname):
ingest_config_dict = load_yaml_from_globs(
config_path('test_base.yaml'),
config_path('s3_storage.yaml'),
config_path('local_fast_ingest.yaml'),
)
result_ingest = execute_pipeline(
ingest_pipeline_def, ingest_config_dict, instance=DagsterInstance.local_temp()
)
assert result_ingest.success
def test_event_pipeline(snowflake_connect):
spark_home_set = True
if os.getenv('SPARK_HOME') is None:
spark_home_set = False
try:
if not spark_home_set:
try:
pyspark_show = subprocess.check_output(
['pip', 'show', 'pyspark']
)
except subprocess.CalledProcessError:
pass
else:
os.environ['SPARK_HOME'] = os.path.join(
list(
filter(
lambda x: 'Location' in x,
pyspark_show.decode('utf-8').split('\n'),
)
)[0].split(' ')[1],
'pyspark',
)
config = load_yaml_from_globs(
script_relative_path('../environments/default.yml')
)
result_pipeline = execute_pipeline(
define_event_ingest_pipeline(), config
)
assert result_pipeline.success
# We're not testing Snowflake loads here, so at least test that we called the connect
# appropriately
snowflake_connect.assert_called_once_with(
user='******',
password='******',
account='<< SET ME >>',
database='TESTDB',
schema='TESTSCHEMA',
warehouse='TINY_WAREHOUSE',
)
finally:
if not spark_home_set:
try:
del os.environ['SPARK_HOME']
except KeyError:
pass
def test_ingest_pipeline_fast_filesystem_storage(postgres, pg_hostname):
ingest_config_dict = load_yaml_from_globs(
config_path('test_base.yaml'),
config_path('local_fast_ingest.yaml'),
config_path('filesystem_storage.yaml'),
)
result_ingest = execute_pipeline(
pipeline=ingest_pipeline,
mode='local',
run_config=ingest_config_dict,
instance=DagsterInstance.local_temp(),
)
assert result_ingest.success
def test_ingest_pipeline_fast_filesystem_storage(docker_compose_db):
ingest_config_dict = load_yaml_from_globs(
config_path('local_base.yaml'),
config_path('local_fast_ingest.yaml'),
config_path('filesystem_storage.yaml'),
)
ingest_config_dict = enviroment_overrides(ingest_config_dict)
result_ingest = execute_pipeline(
define_airline_demo_ingest_pipeline(),
ingest_config_dict,
run_config=RunConfig(mode='local'),
)
assert result_ingest.success
def test_ingest_pipeline_fast_filesystem_storage(postgres, pg_hostname):
with instance_for_test() as instance:
ingest_config_dict = load_yaml_from_globs(
config_path("test_base.yaml"),
config_path("local_fast_ingest.yaml"),
config_path("filesystem_storage.yaml"),
)
result_ingest = execute_pipeline(
pipeline=ingest_pipeline,
mode="local",
run_config=ingest_config_dict,
instance=instance,
)
assert result_ingest.success
def test_ingest_pipeline_fast_filesystem_storage(postgres):
ingest_config_dict = load_yaml_from_globs(
config_path('local_base.yaml'),
config_path('local_fast_ingest.yaml'),
config_path('filesystem_storage.yaml'),
)
ingest_config_dict = enviroment_overrides(ingest_config_dict)
result_ingest = execute_pipeline(
ingest_pipeline_def,
ingest_config_dict,
run_config=RunConfig(mode='local'),
instance=DagsterInstance.local_temp(),
)
assert result_ingest.success
def test_airline_pipeline_1_warehouse(docker_compose_db):
warehouse_config_object = load_yaml_from_globs(
script_relative_path(
'../../dagster_examples/airline_demo/environments/local_base.yaml'
),
script_relative_path(
'../../dagster_examples/airline_demo/environments/local_warehouse.yaml'
),
)
warehouse_config_object = enviroment_overrides(warehouse_config_object)
result_warehouse = execute_pipeline(
define_airline_demo_warehouse_pipeline(),
warehouse_config_object,
run_config=RunConfig(mode='local'),
)
assert result_warehouse.success
def test_event_pipeline(
snowflake_connect, events_jar, spark_home,
): # pylint: disable=redefined-outer-name, unused-argument
config = load_yaml_from_globs(
script_relative_path('../../dagster_examples/event_pipeline_demo/environments/default.yaml')
)
config['solids']['event_ingest']['config']['application_jar'] = events_jar
result_pipeline = execute_pipeline(event_ingest_pipeline, config)
assert result_pipeline.success
# We're not testing Snowflake loads here, so at least test that we called the connect
# appropriately
snowflake_connect.assert_called_with(
user='******',
password='******',
account='<< SET ME >>',
database='TESTDB',
schema='TESTSCHEMA',
warehouse='TINY_WAREHOUSE',
)
