def test_default_asset_store_reexecution():
with seven.TemporaryDirectory() as tmpdir_path:
default_asset_store = default_filesystem_asset_store.configured(
{"base_dir": tmpdir_path})
pipeline_def = define_asset_pipeline(default_asset_store, {})
instance = DagsterInstance.ephemeral()
result = execute_pipeline(pipeline_def,
run_config={"storage": {
"filesystem": {}
}},
instance=instance)
assert result.success
re_result = reexecute_pipeline(
pipeline_def,
result.run_id,
run_config={"storage": {
"filesystem": {}
}},
instance=instance,
step_selection=["solid_b.compute"],
)
# re-execution should yield asset_store_operation events instead of intermediate events
get_asset_events = list(
filter(
lambda evt: evt.is_asset_store_operation and
AssetStoreOperationType(evt.event_specific_data.op
) == AssetStoreOperationType.GET_ASSET,
re_result.event_list,
))
assert len(get_asset_events) == 1
assert get_asset_events[
0].event_specific_data.step_key == "solid_a.compute"
def test_default_asset_store():
with seven.TemporaryDirectory() as tmpdir_path:
default_asset_store = default_filesystem_asset_store.configured(
{"base_dir": tmpdir_path})
pipeline_def = define_asset_pipeline(default_asset_store, {})
result = execute_pipeline(pipeline_def)
assert result.success
asset_store_operation_events = list(
filter(lambda evt: evt.is_asset_store_operation,
result.event_list))
assert len(asset_store_operation_events) == 3
# SET ASSET for step "solid_a.compute" output "result"
assert (asset_store_operation_events[0].event_specific_data.op ==
AssetStoreOperationType.SET_ASSET)
filepath_a = os.path.join(tmpdir_path, result.run_id,
"solid_a.compute", "result")
assert os.path.isfile(filepath_a)
with open(filepath_a, "rb") as read_obj:
assert pickle.load(read_obj) == [1, 2, 3]
# GET ASSET for step "solid_b.compute" input "_df"
assert (asset_store_operation_events[1].event_specific_data.op ==
AssetStoreOperationType.GET_ASSET)
assert (StepOutputHandle("solid_a.compute",
"result") == asset_store_operation_events[1].
event_specific_data.step_output_handle)
# SET ASSET for step "solid_b.compute" output "result"
assert (asset_store_operation_events[2].event_specific_data.op ==
AssetStoreOperationType.SET_ASSET)
filepath_b = os.path.join(tmpdir_path, result.run_id,
"solid_b.compute", "result")
assert os.path.isfile(filepath_b)
with open(filepath_b, "rb") as read_obj:
assert pickle.load(read_obj) == 1
ModeDefinition,
OutputDefinition,
PresetDefinition,
execute_pipeline,
pipeline,
solid,
)
from dagster.core.definitions.utils import struct_to_string
from dagster.core.storage.asset_store import default_filesystem_asset_store
def train(df):
return len(df)
local_asset_store = default_filesystem_asset_store.configured(
{"base_dir": "uncommitted/intermediates/"})
@solid(
output_defs=[OutputDefinition(asset_store_key="default_fs_asset_store")])
def call_api(_):
return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
@solid(
output_defs=[OutputDefinition(asset_store_key="default_fs_asset_store")])
def parse_df(context, df):
context.log.info(struct_to_string(df))
result_df = df[:5]
return result_df