def test_no_loadable_targets():
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(
file_relative_path(__file__, "nada.py"))
assert str(
exc_info.value
) == 'No jobs, pipelines, graphs, or repositories found in "nada".'
def test_double_asset_group():
path = file_relative_path(__file__, "double_asset_group.py")
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(path)
assert str(exc_info.value) == (
'More than one asset group found in "double_asset_group". '
"If you load a file or module directly and it has no repositories, jobs, "
"pipeline, or graphs in scope, it must have no more than one asset group in scope. "
"Found asset groups defined in variables: ['ac1', 'ac2'].")
def test_double_graph():
double_pipeline_path = file_relative_path(__file__, "double_graph.py")
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(double_pipeline_path)
assert str(exc_info.value) == (
'No repository, job, or pipeline, and more than one graph found in "double_graph". '
"If you load a file or module directly it must either have one repository, "
"one job, one pipeline, or one graph in scope. Found graphs defined in variables or decorated "
"functions: ['graph_one', 'graph_two'].")
def test_double_pipeline():
double_pipeline_path = file_relative_path(__file__, "double_pipeline.py")
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(double_pipeline_path)
assert str(exc_info.value) == (
'No repository and more than one pipeline found in "double_pipeline". '
"If you load a file or module directly it must have only one pipeline "
"in scope. Found pipelines defined in variables or decorated "
"functions: ['pipe_one', 'pipe_two'].")
def test_double_asset_group():
path = file_relative_path(__file__, "double_asset_group.py")
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(path)
assert str(exc_info.value) == (
'More than one asset collection found in "double_asset_group". '
"If you load a file or module directly it must either have one repository, one "
"job, one pipeline, one graph, or one asset collection scope. Found asset "
"collections defined in variables: ['ac1', 'ac2'].")
def test_double_graph():
double_pipeline_path = file_relative_path(__file__, "double_graph.py")
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(double_pipeline_path)
assert str(exc_info.value) == (
'More than one graph found in "double_graph". '
"If you load a file or module directly and it has no repositories, jobs, or "
"pipelines in scope, it must have no more than one graph in scope. "
"Found graphs defined in variables or decorated functions: ['graph_one', 'graph_two']."
)
def test_local_directory_file():
path = file_relative_path(__file__,
"autodiscover_file_in_directory/repository.py")
with restore_sys_modules():
with pytest.raises(DagsterImportError) as exc_info:
loadable_targets_from_python_file(path)
assert "No module named 'autodiscover_src'" in str(exc_info.value)
with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]):
loadable_targets_from_python_file(
path, working_directory=os.path.dirname(path))
def get_loadable_targets(python_file, module_name, package_name,
working_directory,
attribute) -> Sequence["LoadableTarget"]:
from dagster.core.workspace.autodiscovery import (
LoadableTarget,
loadable_targets_from_python_file,
loadable_targets_from_python_module,
loadable_targets_from_python_package,
)
if python_file:
return ([
LoadableTarget(
attribute,
load_def_in_python_file(python_file, attribute,
working_directory))
] if attribute else loadable_targets_from_python_file(
python_file, working_directory))
elif module_name:
return ([
LoadableTarget(
attribute,
load_def_in_module(module_name, attribute, working_directory))
] if attribute else loadable_targets_from_python_module(
module_name, working_directory))
elif package_name:
return ([
LoadableTarget(
attribute,
load_def_in_package(package_name, attribute,
working_directory))
] if attribute else loadable_targets_from_python_package(
package_name, working_directory))
else:
check.failed("invalid")
def test_single_repository():
single_repo_path = file_relative_path(__file__, "single_repository.py")
loadable_targets = loadable_targets_from_python_file(single_repo_path)
assert len(loadable_targets) == 1
symbol = loadable_targets[0].attribute
assert symbol == "single_repository"
repo_def = CodePointer.from_python_file(single_repo_path, symbol, None).load_target()
isinstance(repo_def, RepositoryDefinition)
assert repo_def.name == "single_repository"
def test_single_graph():
single_graph_path = file_relative_path(__file__, "single_graph.py")
loadable_targets = loadable_targets_from_python_file(single_graph_path)
assert len(loadable_targets) == 1
symbol = loadable_targets[0].attribute
assert symbol == "graph_one"
repo_def = repository_def_from_pointer(
CodePointer.from_python_file(single_graph_path, symbol, None))
isinstance(repo_def, RepositoryDefinition)
assert repo_def.get_pipeline("graph_one")
def test_local_directory_file():
path = file_relative_path(__file__, "autodiscover_file_in_directory/repository.py")
with restore_sys_modules():
with pytest.raises(ImportError) as exc_info:
loadable_targets_from_python_file(path)
assert str(exc_info.value) == "No module named 'autodiscover_src'"
with pytest.warns(
UserWarning,
match=re.escape(
(
"Module `{module}` was resolved using the working directory. The ability to "
"implicitly load modules from the working directory is deprecated and will be removed "
"in a future release. Please explicitly specify the `working_directory` config option "
"in your workspace.yaml or install `{module}` to your python environment."
).format(module="autodiscover_src")
),
):
with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]):
loadable_targets_from_python_file(path)
def test_single_asset_group():
path = file_relative_path(__file__, "single_asset_group.py")
loadable_targets = loadable_targets_from_python_file(path)
assert len(loadable_targets) == 1
symbol = loadable_targets[0].attribute
assert symbol == "my_asset_group"
repo_def = repository_def_from_pointer(
CodePointer.from_python_file(path, symbol, None))
isinstance(repo_def, RepositoryDefinition)
the_job = repo_def.get_job("__ASSET_GROUP")
assert len(the_job.graph.node_defs) == 2
def test_double_repository():
loadable_repos = loadable_targets_from_python_file(
file_relative_path(__file__, "double_repository.py"), )
assert set([lr.target_definition.name
for lr in loadable_repos]) == {"repo_one", "repo_two"}
