def test_nada(): with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file( file_relative_path(__file__, "nada.py")) assert str( exc_info.value) == 'No pipelines or repositories found in "nada".'
def test_double_pipeline(): double_pipeline_path = file_relative_path(__file__, "double_pipeline.py") with pytest.raises(DagsterInvariantViolationError) as exc_info: loadable_targets_from_python_file(double_pipeline_path) assert str(exc_info.value) == ( 'No repository and more than one pipeline found in "double_pipeline". ' "If you load a file or module directly it must either have one repository " "or one pipeline in scope. Found pipelines defined in variables or decorated " "functions: ['pipe_one', 'pipe_two'].")
def _get_loadable_targets(python_file, module_name): if python_file: return loadable_targets_from_python_file(python_file) elif module_name: return loadable_targets_from_python_module(module_name) else: check.failed('invalid')
def get_loadable_targets(python_file, module_name, package_name, working_directory, attribute): from dagster.cli.workspace.autodiscovery import ( LoadableTarget, loadable_targets_from_python_file, loadable_targets_from_python_module, loadable_targets_from_python_package, ) if python_file: return ( [ LoadableTarget( attribute, load_def_in_python_file(python_file, attribute, working_directory) ) ] if attribute else loadable_targets_from_python_file(python_file, working_directory) ) elif module_name: return ( [LoadableTarget(attribute, load_def_in_module(module_name, attribute))] if attribute else loadable_targets_from_python_module(module_name) ) elif package_name: return ( [LoadableTarget(attribute, load_def_in_package(package_name, attribute))] if attribute else loadable_targets_from_python_package(package_name) ) else: check.failed("invalid")
def test_single_repository(): single_repo_path = file_relative_path(__file__, 'single_repository.py') loadable_targets = loadable_targets_from_python_file(single_repo_path) assert len(loadable_targets) == 1 symbol = loadable_targets[0].attribute assert symbol == 'single_repository' repo_def = CodePointer.from_python_file(single_repo_path, symbol).load_target() isinstance(repo_def, RepositoryDefinition) assert repo_def.name == 'single_repository'
def get_pipelines(): pipelines = {} for filename in os.listdir(file_relative_path(__file__, PIPELINES_DIR)): if filename.endswith('.yml') or filename.endswith('.yaml'): yaml_config = load_yaml_from_path(file_relative_path(__file__, f'{PIPELINES_DIR}/{filename}')) pipeline_name = yaml_config['name'] pipelines[pipeline_name] = partial(define_pipeline_from_yaml, yaml_config) elif filename.endswith('.py') and filename != '__init__.py': targets = loadable_targets_from_python_file(file_relative_path(__file__, f'{PIPELINES_DIR}/{filename}')) for target in targets: pipelines[target.attribute] = target.target_definition return pipelines
def test_local_directory_file(): path = file_relative_path(__file__, "autodiscover_file_in_directory/repository.py") with restore_sys_modules(): with pytest.raises(ImportError) as exc_info: loadable_targets_from_python_file(path) assert str(exc_info.value) == "No module named 'autodiscover_src'" with pytest.warns( UserWarning, match=re.escape(( "Module `{module}` was resolved using the working directory. The ability to " "implicitly load modules from the working directory is deprecated and will be removed " "in a future release. Please explicitly specify the `working_directory` config option " "in your workspace.yaml or install `{module}` to your python environment." ).format(module="autodiscover_src")), ): with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]): loadable_targets_from_python_file(path)
def test_single_pipeline(): single_pipeline_path = file_relative_path(__file__, "single_pipeline.py") loadable_targets = loadable_targets_from_python_file(single_pipeline_path) assert len(loadable_targets) == 1 symbol = loadable_targets[0].attribute assert symbol == "a_pipeline" repo_def = repository_def_from_pointer( CodePointer.from_python_file(single_pipeline_path, symbol, None)) isinstance(repo_def, RepositoryDefinition) assert repo_def.get_pipeline("a_pipeline")
def test_local_directory_file(): path = file_relative_path(__file__, 'autodiscover_file_in_directory/repository.py') is_py27 = sys.version_info < (3, ) with restore_sys_modules(): with pytest.raises(ImportError) as exc_info: loadable_targets_from_python_file(path) assert str(exc_info.value) in [ 'No module named \'autodiscover_src\'', # py3+ 'No module named autodiscover_src.pipelines', # py27 ] with pytest.warns( UserWarning, match=re.escape(( 'Module `{module}` was resolved using the working directory. The ability to ' 'implicitly load modules from the working directory is deprecated and will be removed ' 'in a future release. Please explicitly specify the `working_directory` config option ' 'in your workspace.yaml or install `{module}` to your python environment.' ).format(module='repository' if is_py27 else 'autodiscover_src')), ): with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]): loadable_targets_from_python_file(path)
def test_double_repository(): loadable_repos = loadable_targets_from_python_file( file_relative_path(__file__, "double_repository.py"), ) assert set([lr.target_definition.name for lr in loadable_repos]) == {"repo_one", "repo_two"}