def test_unused_source_asset():
foo = SourceAsset(key=AssetKey("foo"), description="abc")
bar = SourceAsset(key=AssetKey("bar"), description="def")
external_asset_nodes = external_asset_graph_from_defs(
[], source_assets_by_key={
AssetKey("foo"): foo,
AssetKey("bar"): bar
})
assert external_asset_nodes == [
ExternalAssetNode(
asset_key=AssetKey("foo"),
op_description="abc",
dependencies=[],
depended_by=[],
job_names=[],
),
ExternalAssetNode(
asset_key=AssetKey("bar"),
op_description="def",
dependencies=[],
depended_by=[],
job_names=[],
),
]
def test_source_asset():
@asset
def asset1(source1):
assert source1 == 5
return 1
class MyIOManager(IOManager):
def handle_output(self, context, obj):
pass
def load_input(self, context):
return 5
@io_manager
def my_io_manager(_):
return MyIOManager()
job = build_assets_job(
"a",
[asset1],
source_assets=[
SourceAsset(AssetKey("source1"),
io_manager_key="special_io_manager")
],
resource_defs={"special_io_manager": my_io_manager},
)
assert job.graph.node_defs == [asset1.op]
assert job.execute_in_process().success
def test_input_name_matches_output_name():
not_result = SourceAsset(key=AssetKey("not_result"), description=None)
@asset(ins={"result": AssetIn(asset_key=AssetKey("not_result"))})
def something(result): # pylint: disable=unused-argument
pass
assets_job = build_assets_job("assets_job", [something], source_assets=[not_result])
external_asset_nodes = external_asset_graph_from_defs([assets_job], source_assets_by_key={})
assert external_asset_nodes == [
ExternalAssetNode(
asset_key=AssetKey("not_result"),
dependencies=[],
depended_by=[
ExternalAssetDependedBy(
downstream_asset_key=AssetKey("something"), input_name="result"
)
],
job_names=[],
),
ExternalAssetNode(
asset_key=AssetKey("something"),
dependencies=[
ExternalAssetDependency(
upstream_asset_key=AssetKey("not_result"), input_name="result"
)
],
depended_by=[],
op_name="something",
output_name="result",
job_names=["assets_job"],
),
]
def test_used_source_asset():
bar = SourceAsset(key=AssetKey("bar"), description="def")
@asset
def foo(bar):
assert bar
job1 = build_assets_job("job1", [foo], source_assets=[bar])
external_asset_nodes = external_asset_graph_from_defs(
[job1], source_assets_by_key={AssetKey("bar"): bar}
)
assert external_asset_nodes == [
ExternalAssetNode(
asset_key=AssetKey("bar"),
op_description="def",
dependencies=[],
depended_by=[
ExternalAssetDependedBy(downstream_asset_key=AssetKey(["foo"]), input_name="bar")
],
job_names=[],
),
ExternalAssetNode(
asset_key=AssetKey("foo"),
op_name="foo",
op_description=None,
dependencies=[
ExternalAssetDependency(upstream_asset_key=AssetKey(["bar"]), input_name="bar")
],
depended_by=[],
job_names=["job1"],
output_name="result",
output_description=None,
),
]
def test_source_asset_with_op():
foo = SourceAsset(key=AssetKey("foo"), description=None)
@asset
def bar(foo): # pylint: disable=unused-argument
pass
assets_job = build_assets_job("assets_job", [bar], source_assets=[foo])
external_asset_nodes = external_asset_graph_from_defs([assets_job], source_assets_by_key={})
assert external_asset_nodes == [
ExternalAssetNode(
asset_key=AssetKey("foo"),
op_description=None,
dependencies=[],
depended_by=[ExternalAssetDependedBy(AssetKey("bar"), input_name="foo")],
job_names=[],
),
ExternalAssetNode(
asset_key=AssetKey("bar"),
op_name="bar",
op_description=None,
dependencies=[ExternalAssetDependency(AssetKey("foo"), input_name="foo")],
depended_by=[],
job_names=["assets_job"],
output_name="result",
),
]
def test_asset_group_source_asset():
foo_fa = SourceAsset(key=AssetKey("foo"), io_manager_key="the_manager")
@asset
def asset_depends_on_source(foo):
return foo
class MyIOManager(IOManager):
def handle_output(self, context, obj):
pass
def load_input(self, context):
return 5
@io_manager
def the_manager():
return MyIOManager()
group = AssetGroup(
assets=[asset_depends_on_source],
source_assets=[foo_fa],
resource_defs={"the_manager": the_manager},
)
@repository
def the_repo():
return [group]
asset_group_underlying_job = the_repo.get_all_jobs()[0]
assert asset_group_underlying_job.name == group.all_assets_job_name
result = asset_group_underlying_job.execute_in_process()
assert result.success
def test_source_asset():
@asset
def asset1(source1):
assert source1 == 5
return 1
class MyIOManager(IOManager):
def handle_output(self, context, obj):
pass
def load_input(self, context):
assert context.resource_config["a"] == 7
assert context.resources.subresource == 9
assert context.upstream_output.resources.subresource == 9
return 5
@io_manager(config_schema={"a": int},
required_resource_keys={"subresource"})
def my_io_manager(_):
return MyIOManager()
job = build_assets_job(
"a",
[asset1],
source_assets=[
SourceAsset(AssetKey("source1"),
io_manager_key="special_io_manager")
],
resource_defs={
"special_io_manager": my_io_manager.configured({"a": 7}),
"subresource": ResourceDefinition.hardcoded_resource(9),
},
)
assert job.graph.node_defs == [asset1.op]
assert job.execute_in_process().success
def test_source_asset_conflicts_with_asset():
bar_source_asset = SourceAsset(key=AssetKey("bar"), description="def")
@asset
def bar():
pass
job1 = build_assets_job("job1", [bar])
with pytest.raises(DagsterInvariantViolationError):
external_asset_graph_from_defs(
[job1], source_assets_by_key={AssetKey("bar"): bar_source_asset})
def test_asset_group_missing_resources():
@asset(required_resource_keys={"foo"})
def asset_foo(context):
return context.resources.foo
with pytest.raises(
DagsterInvalidDefinitionError,
match=r"AssetGroup is missing required resource keys for asset 'asset_foo'. Missing resource keys: \['foo'\]",
):
AssetGroup([asset_foo])
source_asset_io_req = SourceAsset(key=AssetKey("foo"), io_manager_key="foo")
with pytest.raises(
DagsterInvalidDefinitionError,
match=r"SourceAsset with key AssetKey\(\['foo'\]\) requires io manager with key 'foo', which was not provided on AssetGroup. Provided keys: \['io_manager', 'root_manager'\]",
):
AssetGroup([], source_assets=[source_asset_io_req])
@resource(config_schema={"file": Field(String)})
def hanging_asset_resource(context):
# Hack to allow asset to get value from run config
return context.resource_config.get("file")
class DummyIOManager(IOManager):
def handle_output(self, context, obj):
pass
def load_input(self, context):
pass
dummy_source_asset = SourceAsset(key=AssetKey("dummy_source_asset"))
@asset
def first_asset(dummy_source_asset): # pylint: disable=redefined-outer-name,unused-argument
return 1
@asset(required_resource_keys={"hanging_asset_resource"})
def hanging_asset(context, first_asset): # pylint: disable=redefined-outer-name,unused-argument
"""
Asset that hangs forever, used to test in-progress ops.
"""
with open(context.resources.hanging_asset_resource, "w") as ff:
ff.write("yup")
with pytest.raises(
DagsterInvalidDefinitionError,
match=re.escape(
"Asset key AssetKey(['little_richard']) is defined multiple times. "
"Definitions found in modules: dagster_tests.core_tests.asset_defs_tests.asset_package."
),
):
AssetGroup.from_modules([asset_package, module_with_assets])
@asset
def asset_in_current_module():
pass
source_asset_in_current_module = SourceAsset(AssetKey("source_asset_in_current_module"))
def test_asset_group_from_current_module():
group = AssetGroup.from_current_module()
assert {asset.op.name for asset in group.assets} == {"asset_in_current_module"}
assert len(group.assets) == 1
assert {source_asset.key for source_asset in group.source_assets} == {
AssetKey("source_asset_in_current_module")
}
assert len(group.source_assets) == 1
def test_default_io_manager():
@asset
def asset_foo():