name="default",
resource_defs={
"pandas_csv": df_csv_io_manager,
"cumulus_root": csv_root_input,
"jstor_root": xls_root_input,
"wikidata_root": csv_root_input,
"portals_root": csv_root_input,
"camera_root": geojson_root_input,
"images_root": csv_root_input,
},
)
],
preset_defs=[
dg.PresetDefinition(
"default",
run_config=preset,
mode="default",
)
],
)
def metadata_pipeline():
metadata = create_metadata()
ok = metadata_jstor(metadata=metadata)
# push_new_data(ok)
################ SENSORS ##################
@dg.sensor(pipeline_name="metadata_pipeline")
def trigger_metadata(context):
),
dg.ModeDefinition(
name="test",
resource_defs={
"metadata_root": csv_root_input,
"mapping_root": csv_root_input,
"iiif_manager": json_local_io_manager,
},
),
],
preset_defs=[
dg.PresetDefinition(
name="prod_all",
run_config=default,
mode="prod",
solid_selection=[
"get_items",
"tile_image",
"write_manifests",
],
),
dg.PresetDefinition(
name="prod_manifests",
run_config=default,
mode="prod",
solid_selection=[
"get_items",
"write_manifests",
],
),
dg.PresetDefinition(
name="prod_tile",
@dg.pipeline(
mode_defs=[
dg.ModeDefinition(
name="default",
resource_defs={
"pandas_csv": df_csv_io_manager,
"metadata_root": csv_root_input,
},
),
],
preset_defs=[
dg.PresetDefinition(
"default",
run_config=preset,
mode="default",
),
dg.PresetDefinition(
"preset_omeka",
run_config=preset_omeka,
solid_selection=["query_omeka", "omeka_dataframe", "update_metadata","push_new_data"],
mode="default",
),
dg.PresetDefinition(
"preset_wikidata",
run_config=preset_wikidata,
solid_selection=["query_wikidata", "wikidata_dataframe", "update_metadata","push_new_data"],
mode="default",
),
dg.PresetDefinition(
"metadata_root": csv_root_input,
"smapshot_root": csv_root_input,
"cumulus_root": csv_root_input,
"wikidata_root": csv_root_input,
"camera_root": geojson_root_input,
"images_root": csv_root_input,
"omeka_root": csv_root_input,
"mapping_root": csv_root_input,
"portals_root": csv_root_input,
},
)
],
preset_defs=[
dg.PresetDefinition(
"default",
run_config=preset,
mode="default",
),
dg.PresetDefinition(
"preset_html",
run_config=preset_html,
solid_selection=[
"format_values_chart",
"create_hbar",
"create_pie",
"export_html",
# "push_new_data",
],
mode="default",
),
],