def test_no_loadable_targets(): with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file( file_relative_path(__file__, "nada.py")) assert str( exc_info.value ) == 'No jobs, pipelines, graphs, or repositories found in "nada".'
def test_double_asset_group(): path = file_relative_path(__file__, "double_asset_group.py") with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file(path) assert str(exc_info.value) == ( 'More than one asset group found in "double_asset_group". ' "If you load a file or module directly and it has no repositories, jobs, " "pipeline, or graphs in scope, it must have no more than one asset group in scope. " "Found asset groups defined in variables: ['ac1', 'ac2'].")
def test_double_graph(): double_pipeline_path = file_relative_path(__file__, "double_graph.py") with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file(double_pipeline_path) assert str(exc_info.value) == ( 'No repository, job, or pipeline, and more than one graph found in "double_graph". ' "If you load a file or module directly it must either have one repository, " "one job, one pipeline, or one graph in scope. Found graphs defined in variables or decorated " "functions: ['graph_one', 'graph_two'].")
def test_double_pipeline(): double_pipeline_path = file_relative_path(__file__, "double_pipeline.py") with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file(double_pipeline_path) assert str(exc_info.value) == ( 'No repository and more than one pipeline found in "double_pipeline". ' "If you load a file or module directly it must have only one pipeline " "in scope. Found pipelines defined in variables or decorated " "functions: ['pipe_one', 'pipe_two'].")
def test_double_asset_group(): path = file_relative_path(__file__, "double_asset_group.py") with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file(path) assert str(exc_info.value) == ( 'More than one asset collection found in "double_asset_group". ' "If you load a file or module directly it must either have one repository, one " "job, one pipeline, one graph, or one asset collection scope. Found asset " "collections defined in variables: ['ac1', 'ac2'].")
def test_double_graph(): double_pipeline_path = file_relative_path(__file__, "double_graph.py") with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file(double_pipeline_path) assert str(exc_info.value) == ( 'More than one graph found in "double_graph". ' "If you load a file or module directly and it has no repositories, jobs, or " "pipelines in scope, it must have no more than one graph in scope. " "Found graphs defined in variables or decorated functions: ['graph_one', 'graph_two']." )
def test_local_directory_file(): path = file_relative_path(__file__, "autodiscover_file_in_directory/repository.py") with restore_sys_modules(): with pytest.raises(DagsterImportError) as exc_info: loadable_targets_from_python_file(path) assert "No module named 'autodiscover_src'" in str(exc_info.value) with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]): loadable_targets_from_python_file( path, working_directory=os.path.dirname(path))
def get_loadable_targets(python_file, module_name, package_name, working_directory, attribute) -> Sequence["LoadableTarget"]: from dagster.core.workspace.autodiscovery import ( LoadableTarget, loadable_targets_from_python_file, loadable_targets_from_python_module, loadable_targets_from_python_package, ) if python_file: return ([ LoadableTarget( attribute, load_def_in_python_file(python_file, attribute, working_directory)) ] if attribute else loadable_targets_from_python_file( python_file, working_directory)) elif module_name: return ([ LoadableTarget( attribute, load_def_in_module(module_name, attribute, working_directory)) ] if attribute else loadable_targets_from_python_module( module_name, working_directory)) elif package_name: return ([ LoadableTarget( attribute, load_def_in_package(package_name, attribute, working_directory)) ] if attribute else loadable_targets_from_python_package( package_name, working_directory)) else: check.failed("invalid")
def test_single_repository(): single_repo_path = file_relative_path(__file__, "single_repository.py") loadable_targets = loadable_targets_from_python_file(single_repo_path) assert len(loadable_targets) == 1 symbol = loadable_targets[0].attribute assert symbol == "single_repository" repo_def = CodePointer.from_python_file(single_repo_path, symbol, None).load_target() isinstance(repo_def, RepositoryDefinition) assert repo_def.name == "single_repository"
def test_single_graph(): single_graph_path = file_relative_path(__file__, "single_graph.py") loadable_targets = loadable_targets_from_python_file(single_graph_path) assert len(loadable_targets) == 1 symbol = loadable_targets[0].attribute assert symbol == "graph_one" repo_def = repository_def_from_pointer( CodePointer.from_python_file(single_graph_path, symbol, None)) isinstance(repo_def, RepositoryDefinition) assert repo_def.get_pipeline("graph_one")
def test_local_directory_file(): path = file_relative_path(__file__, "autodiscover_file_in_directory/repository.py") with restore_sys_modules(): with pytest.raises(ImportError) as exc_info: loadable_targets_from_python_file(path) assert str(exc_info.value) == "No module named 'autodiscover_src'" with pytest.warns( UserWarning, match=re.escape( ( "Module `{module}` was resolved using the working directory. The ability to " "implicitly load modules from the working directory is deprecated and will be removed " "in a future release. Please explicitly specify the `working_directory` config option " "in your workspace.yaml or install `{module}` to your python environment." ).format(module="autodiscover_src") ), ): with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]): loadable_targets_from_python_file(path)
def test_single_asset_group(): path = file_relative_path(__file__, "single_asset_group.py") loadable_targets = loadable_targets_from_python_file(path) assert len(loadable_targets) == 1 symbol = loadable_targets[0].attribute assert symbol == "my_asset_group" repo_def = repository_def_from_pointer( CodePointer.from_python_file(path, symbol, None)) isinstance(repo_def, RepositoryDefinition) the_job = repo_def.get_job("__ASSET_GROUP") assert len(the_job.graph.node_defs) == 2
def test_double_repository(): loadable_repos = loadable_targets_from_python_file( file_relative_path(__file__, "double_repository.py"), ) assert set([lr.target_definition.name for lr in loadable_repos]) == {"repo_one", "repo_two"}