def scope_dbt_cli_config_disable_assets():
# start_marker_dbt_cli_config_disable_assets
config = {"yield_materializations": False}
from dagster_dbt import dbt_cli_run
custom_solid = dbt_cli_run.configured(config, name="custom_solid")
def scope_dbt_cli_config_exclude_models():
# start_marker_dbt_cli_config_exclude_models
config = {"exclude": ["my_dbt_model+", "path.to.models", "tag:nightly"]}
from dagster_dbt import dbt_cli_run
custom_solid = dbt_cli_run.configured(config, name="custom_solid")
def scope_dbt_cli_config_vars():
# start_marker_dbt_cli_config_vars
config = {"vars": {"key": "value"}}
from dagster_dbt import dbt_cli_run
custom_solid = dbt_cli_run.configured(config, name="custom_solid")
def scope_dbt_cli_config_executable():
# start_marker_dbt_cli_config_executable
config = {"dbt_executable": "path/to/dbt/executable"}
from dagster_dbt import dbt_cli_run
custom_solid = dbt_cli_run.configured(config, name="custom_solid")
def scope_dbt_cli_config_profile_and_target():
PROFILE_NAME, TARGET_NAME = "", ""
# start_marker_dbt_cli_config_profile_and_target
config = {"profile": PROFILE_NAME, "target": TARGET_NAME}
from dagster_dbt import dbt_cli_run
custom_solid = dbt_cli_run.configured(config, name="custom_solid")
def scope_dbt_cli_run():
# start_marker_dbt_cli_run
from dagster import pipeline
from dagster_dbt import dbt_cli_run
config = {"project-dir": "path/to/dbt/project"}
run_all_models = dbt_cli_run.configured(config, name="run_dbt_project")
@pipeline
def my_dbt_pipeline():
run_all_models()
def scope_dbt_cli_run_specific_models():
# start_marker_dbt_cli_run_specific_models
from dagster import pipeline
from dagster_dbt import dbt_cli_run
config = {"project-dir": "path/to/dbt/project", "models": ["tag:staging"]}
run_staging_models = dbt_cli_run.configured(config,
name="run_staging_models")
@pipeline
def my_dbt_pipeline():
run_staging_models()
def my_gcp_dataops_pipeline():
sql_process = bq_solid_for_queries([
cfg.sql_query["QUERY"]
]).alias("export_bq_table_gcs")(dbt_cli_run.configured(
{"project-dir": cfg.dbt_config["DBT_PROJECT_DIR"]},
name="run_bq_dbt")())
spark_process = delete_dataproc_cluster(
data_proc_spark_operator(create_dataproc_cluster(sql_process)))
jupyter_process = dm.define_dagstermill_solid(
"view_data_matplot",
script_relative_path("jupyter/view_data.ipynb"),
input_defs=[InputDefinition("start",
Nothing)])(download_file(spark_process))
conn.execute("drop table if exists cereals cascade")
cereals_df.to_sql(name="cereals", con=conn)
@solid(config_schema={"channels": Array(str)},
required_resource_keys={"slack"})
def post_plot_to_slack(context, plot_path):
context.resources.slack.files_upload(channels=",".join(
context.solid_config["channels"]),
file=plot_path)
run_cereals_models = dbt_cli_run.configured(
name="run_cereals_models",
config_or_config_fn={
"project-dir": PROJECT_DIR,
"profiles-dir": PROFILES_DIR
},
)
test_cereals_models = dbt_cli_test.configured(
name="test_cereals_models",
config_or_config_fn={
"project-dir": PROJECT_DIR,
"profiles-dir": PROFILES_DIR
},
)
analyze_cereals = define_dagstermill_solid(
"analyze_cereals",
file_relative_path(__file__, "notebooks/Analyze Cereals.ipynb"),
with context.resources.db.connect() as conn:
conn.execute("drop table if exists cereals cascade")
cereals_df.to_sql(name="cereals", con=conn)
@solid(config_schema={"channels": Array(str)},
required_resource_keys={"slack"})
def post_plot_to_slack(context, plot_path):
context.resources.slack.files_upload(channels=",".join(
context.solid_config["channels"]),
file=plot_path)
# start_solid_marker_0
run_cereals_models = dbt_cli_run.configured(
config_or_config_fn={"project-dir": PROJECT_DIR},
name="run_cereals_models",
)
# end_solid_marker_0
test_cereals_models = dbt_cli_test.configured(
config_or_config_fn={"project-dir": PROJECT_DIR},
name="test_cereals_models",
)
analyze_cereals = define_dagstermill_solid(
"analyze_cereals",
file_relative_path(__file__, "notebooks/Analyze Cereals.ipynb"),
input_defs=[InputDefinition("run_results", dagster_type=DbtCliOutput)],
output_defs=[OutputDefinition(str)],
required_resource_keys={"db"},
)