def test_default_object_manager_reexecution():
with seven.TemporaryDirectory() as tmpdir_path:
default_asset_store = fs_object_manager.configured(
{"base_dir": tmpdir_path})
pipeline_def = define_pipeline(default_asset_store, {})
instance = DagsterInstance.ephemeral()
result = execute_pipeline(pipeline_def, instance=instance)
assert result.success
re_result = reexecute_pipeline(
pipeline_def,
result.run_id,
instance=instance,
step_selection=["solid_b"],
)
# re-execution should yield asset_store_operation events instead of intermediate events
get_asset_events = list(
filter(
lambda evt: evt.is_asset_store_operation and
AssetStoreOperationType(evt.event_specific_data.op
) == AssetStoreOperationType.GET_ASSET,
re_result.event_list,
))
assert len(get_asset_events) == 1
assert get_asset_events[0].event_specific_data.step_key == "solid_a"
def test_result_output():
with seven.TemporaryDirectory() as tmpdir_path:
asset_store = fs_object_manager.configured({"base_dir": tmpdir_path})
pipeline_def = define_pipeline(asset_store, {})
result = execute_pipeline(pipeline_def)
assert result.success
# test output_value
assert result.result_for_solid("solid_a").output_value() == [1, 2, 3]
assert result.result_for_solid("solid_b").output_value() == 1
def test_fan_in():
with seven.TemporaryDirectory() as tmpdir_path:
asset_store = fs_object_manager.configured({"base_dir": tmpdir_path})
@solid
def input_solid1(_):
return 1
@solid
def input_solid2(_):
return 2
@solid
def solid1(_, input1):
assert input1 == [1, 2]
@pipeline(mode_defs=[
ModeDefinition(resource_defs={"object_manager": asset_store})
])
def my_pipeline():
solid1(input1=[input_solid1(), input_solid2()])
execute_pipeline(my_pipeline)
def test_fs_object_manager():
with seven.TemporaryDirectory() as tmpdir_path:
asset_store = fs_object_manager.configured({"base_dir": tmpdir_path})
pipeline_def = define_pipeline(asset_store)
result = execute_pipeline(pipeline_def)
assert result.success
asset_store_operation_events = list(
filter(lambda evt: evt.is_asset_store_operation,
result.event_list))
assert len(asset_store_operation_events) == 3
# SET ASSET for step "solid_a" output "result"
assert (asset_store_operation_events[0].event_specific_data.op ==
AssetStoreOperationType.SET_ASSET)
filepath_a = os.path.join(tmpdir_path, result.run_id, "solid_a",
"result")
assert os.path.isfile(filepath_a)
with open(filepath_a, "rb") as read_obj:
assert pickle.load(read_obj) == [1, 2, 3]
# GET ASSET for step "solid_b" input "_df"
assert (asset_store_operation_events[1].event_specific_data.op ==
AssetStoreOperationType.GET_ASSET)
assert "solid_a" == asset_store_operation_events[
1].event_specific_data.step_key
# SET ASSET for step "solid_b" output "result"
assert (asset_store_operation_events[2].event_specific_data.op ==
AssetStoreOperationType.SET_ASSET)
filepath_b = os.path.join(tmpdir_path, result.run_id, "solid_b",
"result")
assert os.path.isfile(filepath_b)
with open(filepath_b, "rb") as read_obj:
assert pickle.load(read_obj) == 1