def test_asset_group_from_modules(monkeypatch): from . import asset_package from .asset_package import module_with_assets collection_1 = AssetGroup.from_modules([asset_package, module_with_assets]) assets_1 = [asset.op.name for asset in collection_1.assets] source_assets_1 = [source_asset.key for source_asset in collection_1.source_assets] collection_2 = AssetGroup.from_modules([asset_package, module_with_assets]) assets_2 = [asset.op.name for asset in collection_2.assets] source_assets_2 = [source_asset.key for source_asset in collection_2.source_assets] assert assets_1 == assets_2 assert source_assets_1 == source_assets_2 with monkeypatch.context() as m: @asset def little_richard(): pass m.setattr(asset_package, "little_richard_dup", little_richard, raising=False) with pytest.raises( DagsterInvalidDefinitionError, match=re.escape( "Asset key AssetKey(['little_richard']) is defined multiple times. " "Definitions found in modules: dagster_tests.core_tests.asset_defs_tests.asset_package." ), ): AssetGroup.from_modules([asset_package, module_with_assets])
def _load_target_from_module(module: ModuleType, fn_name: str, error_suffix: str) -> object: from dagster.core.asset_defs import AssetGroup from dagster.core.workspace.autodiscovery import LOAD_ALL_ASSETS if fn_name == LOAD_ALL_ASSETS: # LOAD_ALL_ASSETS is a special symbol that's returned when, instead of loading a particular # attribute, we should load all the assets in the module. return AssetGroup.from_modules([module]) else: if not hasattr(module, fn_name): raise DagsterInvariantViolationError( f"{fn_name} not found {error_suffix}") return getattr(module, fn_name)
def loadable_targets_from_loaded_module( module: ModuleType) -> Sequence[LoadableTarget]: loadable_repos = _loadable_targets_of_type(module, RepositoryDefinition) if loadable_repos: return loadable_repos loadable_pipelines = _loadable_targets_of_type(module, PipelineDefinition) loadable_jobs = _loadable_targets_of_type(module, JobDefinition) if len(loadable_pipelines) == 1: return loadable_pipelines elif len(loadable_pipelines) > 1: target_type = "job" if len(loadable_jobs) > 1 else "pipeline" raise DagsterInvariantViolationError(( 'No repository and more than one {target_type} found in "{module_name}". If you load ' "a file or module directly it must have only one {target_type} " "in scope. Found {target_type}s defined in variables or decorated " "functions: {pipeline_symbols}.").format( module_name=module.__name__, pipeline_symbols=repr( [p.attribute for p in loadable_pipelines]), target_type=target_type, )) loadable_graphs = _loadable_targets_of_type(module, GraphDefinition) if len(loadable_graphs) == 1: return loadable_graphs elif len(loadable_graphs) > 1: raise DagsterInvariantViolationError(( 'More than one graph found in "{module_name}". ' "If you load a file or module directly and it has no repositories, jobs, or " "pipelines in scope, it must have no more than one graph in scope. " "Found graphs defined in variables or decorated functions: {graph_symbols}." ).format( module_name=module.__name__, graph_symbols=repr([g.attribute for g in loadable_graphs]), )) loadable_asset_groups = _loadable_targets_of_type(module, AssetGroup) if len(loadable_asset_groups) == 1: return loadable_asset_groups elif len(loadable_asset_groups) > 1: var_names = repr([a.attribute for a in loadable_asset_groups]) raise DagsterInvariantViolationError(( f'More than one asset group found in "{module.__name__}". ' "If you load a file or module directly and it has no repositories, jobs, " "pipeline, or graphs in scope, it must have no more than one asset group in scope. " f"Found asset groups defined in variables: {var_names}.")) asset_group_from_module_assets = AssetGroup.from_modules([module]) if (len(asset_group_from_module_assets.assets) > 0 or len(asset_group_from_module_assets.source_assets) > 0): return [ LoadableTarget(LOAD_ALL_ASSETS, asset_group_from_module_assets) ] raise DagsterInvariantViolationError( "No repositories, jobs, pipelines, graphs, asset groups, or asset definitions found in " f'"{module.__name__}".')