def test_asset_group_from_modules(monkeypatch):
from . import asset_package
from .asset_package import module_with_assets
collection_1 = AssetGroup.from_modules([asset_package, module_with_assets])
assets_1 = [asset.op.name for asset in collection_1.assets]
source_assets_1 = [source_asset.key for source_asset in collection_1.source_assets]
collection_2 = AssetGroup.from_modules([asset_package, module_with_assets])
assets_2 = [asset.op.name for asset in collection_2.assets]
source_assets_2 = [source_asset.key for source_asset in collection_2.source_assets]
assert assets_1 == assets_2
assert source_assets_1 == source_assets_2
with monkeypatch.context() as m:
@asset
def little_richard():
pass
m.setattr(asset_package, "little_richard_dup", little_richard, raising=False)
with pytest.raises(
DagsterInvalidDefinitionError,
match=re.escape(
"Asset key AssetKey(['little_richard']) is defined multiple times. "
"Definitions found in modules: dagster_tests.core_tests.asset_defs_tests.asset_package."
),
):
AssetGroup.from_modules([asset_package, module_with_assets])
def _load_target_from_module(module: ModuleType, fn_name: str,
error_suffix: str) -> object:
from dagster.core.asset_defs import AssetGroup
from dagster.core.workspace.autodiscovery import LOAD_ALL_ASSETS
if fn_name == LOAD_ALL_ASSETS:
# LOAD_ALL_ASSETS is a special symbol that's returned when, instead of loading a particular
# attribute, we should load all the assets in the module.
return AssetGroup.from_modules([module])
else:
if not hasattr(module, fn_name):
raise DagsterInvariantViolationError(
f"{fn_name} not found {error_suffix}")
return getattr(module, fn_name)
def loadable_targets_from_loaded_module(
module: ModuleType) -> Sequence[LoadableTarget]:
loadable_repos = _loadable_targets_of_type(module, RepositoryDefinition)
if loadable_repos:
return loadable_repos
loadable_pipelines = _loadable_targets_of_type(module, PipelineDefinition)
loadable_jobs = _loadable_targets_of_type(module, JobDefinition)
if len(loadable_pipelines) == 1:
return loadable_pipelines
elif len(loadable_pipelines) > 1:
target_type = "job" if len(loadable_jobs) > 1 else "pipeline"
raise DagsterInvariantViolationError((
'No repository and more than one {target_type} found in "{module_name}". If you load '
"a file or module directly it must have only one {target_type} "
"in scope. Found {target_type}s defined in variables or decorated "
"functions: {pipeline_symbols}.").format(
module_name=module.__name__,
pipeline_symbols=repr(
[p.attribute for p in loadable_pipelines]),
target_type=target_type,
))
loadable_graphs = _loadable_targets_of_type(module, GraphDefinition)
if len(loadable_graphs) == 1:
return loadable_graphs
elif len(loadable_graphs) > 1:
raise DagsterInvariantViolationError((
'More than one graph found in "{module_name}". '
"If you load a file or module directly and it has no repositories, jobs, or "
"pipelines in scope, it must have no more than one graph in scope. "
"Found graphs defined in variables or decorated functions: {graph_symbols}."
).format(
module_name=module.__name__,
graph_symbols=repr([g.attribute for g in loadable_graphs]),
))
loadable_asset_groups = _loadable_targets_of_type(module, AssetGroup)
if len(loadable_asset_groups) == 1:
return loadable_asset_groups
elif len(loadable_asset_groups) > 1:
var_names = repr([a.attribute for a in loadable_asset_groups])
raise DagsterInvariantViolationError((
f'More than one asset group found in "{module.__name__}". '
"If you load a file or module directly and it has no repositories, jobs, "
"pipeline, or graphs in scope, it must have no more than one asset group in scope. "
f"Found asset groups defined in variables: {var_names}."))
asset_group_from_module_assets = AssetGroup.from_modules([module])
if (len(asset_group_from_module_assets.assets) > 0
or len(asset_group_from_module_assets.source_assets) > 0):
return [
LoadableTarget(LOAD_ALL_ASSETS, asset_group_from_module_assets)
]
raise DagsterInvariantViolationError(
"No repositories, jobs, pipelines, graphs, asset groups, or asset definitions found in "
f'"{module.__name__}".')