def _default_spec_load(starting_dir=None, lazy_import=False, reload=False):
"""
NOTE: this is a private API. Use DAGSpec.find() instead
Looks for a pipeline.yaml, generates a DAGSpec and returns a DAG.
Currently, this is only used by the PloomberContentsManager, this is
not intended to be a public API since initializing specs from paths
where we have to recursively look for a pipeline.yaml has some
considerations regarding relative paths that make this confusing,
inside the contents manager, all those things are all handled for that
use case.
The pipeline.yaml parent folder is temporarily added to sys.path when
calling DAGSpec.to_dag() to make sure imports work as expected
Returns DAG and the directory where the pipeline.yaml file is located.
"""
root_path = starting_dir or os.getcwd()
path_to_entry_point = default.entry_point(root_path=root_path)
try:
spec = DAGSpec(path_to_entry_point,
env=None,
lazy_import=lazy_import,
reload=reload)
path_to_spec = Path(path_to_entry_point)
return spec, path_to_spec.parent, path_to_spec
except Exception as e:
exc = DAGSpecInitializationError('Error initializing DAG from '
f'{path_to_entry_point!s}')
raise exc from e
def test_entry_point_pkg_location_multiple_pkgs(tmp_directory):
for pkg in ['package_a', 'package_b']:
parent = Path('src', pkg)
parent.mkdir(parents=True)
pkg_location = (parent / 'pipeline.yaml')
pkg_location.touch()
assert default.entry_point() == str(
Path('src', 'package_a', 'pipeline.yaml'))
def test_entry_point_in_src_while_in_sibling_folder(tmp_directory):
Path('setup.py').touch()
pkg = Path('src', 'package')
pkg.mkdir(parents=True)
(pkg / 'pipeline.yaml').touch()
Path('tests').mkdir()
os.chdir('tests')
assert default.entry_point() == str(
Path('..', 'src', 'package', 'pipeline.yaml'))
def __init__(self, *args, **kwargs):
self.DEFAULT_ENTRY_POINT = default.entry_point()
self.static_args = []
self.finished_static_api = False
self.in_context = False
self.finished_init = False
super().__init__(*args, **kwargs)
self.add_argument('--log',
'-l',
help='Enables logging to stdout at the '
'specified level',
default=None)
self.add_argument(
'--entry-point',
'-e',
help=f'Entry point, defaults to {self.DEFAULT_ENTRY_POINT}',
default=self.DEFAULT_ENTRY_POINT)
self.finished_init = True
def test_entry_point_in_parent_folder(tmp_directory):
Path('dir').mkdir()
Path('pipeline.yaml').touch()
os.chdir('dir')
assert default.entry_point() == str(Path('..', 'pipeline.yaml'))
def test_entry_point():
assert default.entry_point() == 'pipeline.yaml'
def test_entry_point_pkg_location_and_yaml(tmp_directory, pkg_location):
Path('pipeline.yaml').touch()
assert default.entry_point() == 'pipeline.yaml'
def test_entry_point_pkg_location(tmp_directory, pkg_location):
assert default.entry_point() == str(pkg_location)
def test_entry_point_env_var(monkeypatch, tmp_directory, pkg_location):
monkeypatch.setenv('ENTRY_POINT', 'some.entry.point')
assert default.entry_point() == 'some.entry.point'