def test_nada():
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(
file_relative_path(__file__, "nada.py"))
assert str(
exc_info.value) == 'No pipelines or repositories found in "nada".'
def test_double_pipeline():
double_pipeline_path = file_relative_path(__file__, "double_pipeline.py")
with pytest.raises(DagsterInvariantViolationError) as exc_info:
loadable_targets_from_python_file(double_pipeline_path)
assert str(exc_info.value) == (
'No repository and more than one pipeline found in "double_pipeline". '
"If you load a file or module directly it must either have one repository "
"or one pipeline in scope. Found pipelines defined in variables or decorated "
"functions: ['pipe_one', 'pipe_two'].")
def _get_loadable_targets(python_file, module_name):
if python_file:
return loadable_targets_from_python_file(python_file)
elif module_name:
return loadable_targets_from_python_module(module_name)
else:
check.failed('invalid')
def get_loadable_targets(python_file, module_name, package_name, working_directory, attribute):
from dagster.cli.workspace.autodiscovery import (
LoadableTarget,
loadable_targets_from_python_file,
loadable_targets_from_python_module,
loadable_targets_from_python_package,
)
if python_file:
return (
[
LoadableTarget(
attribute, load_def_in_python_file(python_file, attribute, working_directory)
)
]
if attribute
else loadable_targets_from_python_file(python_file, working_directory)
)
elif module_name:
return (
[LoadableTarget(attribute, load_def_in_module(module_name, attribute))]
if attribute
else loadable_targets_from_python_module(module_name)
)
elif package_name:
return (
[LoadableTarget(attribute, load_def_in_package(package_name, attribute))]
if attribute
else loadable_targets_from_python_package(package_name)
)
else:
check.failed("invalid")
def test_single_repository():
single_repo_path = file_relative_path(__file__, 'single_repository.py')
loadable_targets = loadable_targets_from_python_file(single_repo_path)
assert len(loadable_targets) == 1
symbol = loadable_targets[0].attribute
assert symbol == 'single_repository'
repo_def = CodePointer.from_python_file(single_repo_path, symbol).load_target()
isinstance(repo_def, RepositoryDefinition)
assert repo_def.name == 'single_repository'
def get_pipelines():
pipelines = {}
for filename in os.listdir(file_relative_path(__file__, PIPELINES_DIR)):
if filename.endswith('.yml') or filename.endswith('.yaml'):
yaml_config = load_yaml_from_path(file_relative_path(__file__, f'{PIPELINES_DIR}/{filename}'))
pipeline_name = yaml_config['name']
pipelines[pipeline_name] = partial(define_pipeline_from_yaml, yaml_config)
elif filename.endswith('.py') and filename != '__init__.py':
targets = loadable_targets_from_python_file(file_relative_path(__file__, f'{PIPELINES_DIR}/{filename}'))
for target in targets:
pipelines[target.attribute] = target.target_definition
return pipelines
def test_local_directory_file():
path = file_relative_path(__file__,
"autodiscover_file_in_directory/repository.py")
with restore_sys_modules():
with pytest.raises(ImportError) as exc_info:
loadable_targets_from_python_file(path)
assert str(exc_info.value) == "No module named 'autodiscover_src'"
with pytest.warns(
UserWarning,
match=re.escape((
"Module `{module}` was resolved using the working directory. The ability to "
"implicitly load modules from the working directory is deprecated and will be removed "
"in a future release. Please explicitly specify the `working_directory` config option "
"in your workspace.yaml or install `{module}` to your python environment."
).format(module="autodiscover_src")),
):
with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]):
loadable_targets_from_python_file(path)
def test_single_pipeline():
single_pipeline_path = file_relative_path(__file__, "single_pipeline.py")
loadable_targets = loadable_targets_from_python_file(single_pipeline_path)
assert len(loadable_targets) == 1
symbol = loadable_targets[0].attribute
assert symbol == "a_pipeline"
repo_def = repository_def_from_pointer(
CodePointer.from_python_file(single_pipeline_path, symbol, None))
isinstance(repo_def, RepositoryDefinition)
assert repo_def.get_pipeline("a_pipeline")
def test_local_directory_file():
path = file_relative_path(__file__,
'autodiscover_file_in_directory/repository.py')
is_py27 = sys.version_info < (3, )
with restore_sys_modules():
with pytest.raises(ImportError) as exc_info:
loadable_targets_from_python_file(path)
assert str(exc_info.value) in [
'No module named \'autodiscover_src\'', # py3+
'No module named autodiscover_src.pipelines', # py27
]
with pytest.warns(
UserWarning,
match=re.escape((
'Module `{module}` was resolved using the working directory. The ability to '
'implicitly load modules from the working directory is deprecated and will be removed '
'in a future release. Please explicitly specify the `working_directory` config option '
'in your workspace.yaml or install `{module}` to your python environment.'
).format(module='repository' if is_py27 else 'autodiscover_src')),
):
with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]):
loadable_targets_from_python_file(path)
def test_double_repository():
loadable_repos = loadable_targets_from_python_file(
file_relative_path(__file__, "double_repository.py"), )
assert set([lr.target_definition.name
for lr in loadable_repos]) == {"repo_one", "repo_two"}
