def load_python_file(python_file, working_directory):
'''
Takes a path to a python file and returns a loaded module
'''
check.str_param(python_file, 'python_file')
module_name = os.path.splitext(os.path.basename(python_file))[0]
cwd = sys.path[0]
if working_directory:
with alter_sys_path(to_add=[working_directory], to_remove=[cwd]):
return import_module_from_path(module_name, python_file)
error = None
sys_modules = {k: v for k, v in sys.modules.items()}
with alter_sys_path(to_add=[], to_remove=[cwd]):
try:
module = import_module_from_path(module_name, python_file)
except ImportError as ie:
# importing alters sys.modules in ways that may interfere with the import below, even
# if the import has failed. to work around this, we need to manually clear any modules
# that have been cached in sys.modules due to the speculative import call
# Also, we are mutating sys.modules instead of straight-up assigning to sys_modules,
# because some packages will do similar shenanigans to sys.modules (e.g. numpy)
to_delete = set(sys.modules) - set(sys_modules)
for key in to_delete:
del sys.modules[key]
error = ie
if not error:
return module
try:
module = import_module_from_path(module_name, python_file)
# if here, we were able to resolve the module with the working directory on the
# path, but should error because we may not always invoke from the same directory
# (e.g. from cron)
warnings.warn(
(
'Module `{module}` was resolved using the working directory. The ability to '
'implicitly load modules from the working directory is deprecated and '
'will be removed in a future release. Please explicitly specify the '
'`working_directory` config option in your workspace.yaml or install `{module}` to '
'your python environment.'
).format(module=error.name if hasattr(error, 'name') else module_name)
)
return module
except ImportError:
raise error
def load_python_file(python_file):
'''
Takes a path to a python file and returns a loaded module
'''
check.str_param(python_file, 'python_file')
module_name = os.path.splitext(os.path.basename(python_file))[0]
return import_module_from_path(module_name, python_file)
def test_repository_python_file():
python_file = script_relative_path('bar_repo.py')
module = import_module_from_path('bar_repo', python_file)
handle = handle_for_pipeline_cli_args(
{'pipeline_name': 'foo', 'python_file': python_file, 'fn_name': 'define_bar_repo'}
)
assert handle.mode == _ExecutionTargetMode.PIPELINE
assert handle.entrypoint == LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo', handle)
assert handle.data.pipeline_name == 'foo'
assert handle.entrypoint.from_handle == handle
with pytest.raises(UsageError):
handle_for_pipeline_cli_args(
{
'module_name': 'kdjfkd',
'pipeline_name': 'foo',
'python_file': script_relative_path('bar_repo.py'),
'fn_name': 'define_bar_repo',
'repository_yaml': None,
}
)
with pytest.raises(UsageError):
handle_for_pipeline_cli_args(
{
'module_name': None,
'pipeline_name': 'foo',
'python_file': script_relative_path('bar_repo.py'),
'fn_name': 'define_bar_repo',
'repository_yaml': 'kjdfkdjf',
}
)
def from_file_target(python_file, fn_name, from_handle=None):
file_directory = os.path.dirname(python_file)
if file_directory not in sys.path:
sys.path.append(file_directory)
module_name = os.path.splitext(os.path.basename(python_file))[0]
module = import_module_from_path(module_name, python_file)
return PartitionLoaderEntrypoint(module, module_name, fn_name, from_handle)
def load_target(self):
module_name = os.path.splitext(os.path.basename(self.python_file))[0]
module = import_module_from_path(module_name, self.python_file)
if not hasattr(module, self.fn_name):
raise DagsterInvariantViolationError(
'{name} not found at module scope in file {file}.'.format(
name=self.fn_name, file=self.python_file))
return getattr(module, self.fn_name)
def load_python_file(python_file, working_directory):
'''
Takes a path to a python file and returns a loaded module
'''
check.str_param(python_file, 'python_file')
module_name = os.path.splitext(os.path.basename(python_file))[0]
cwd = sys.path[0]
if working_directory:
with alter_sys_path(to_add=[working_directory], to_remove=[cwd]):
return import_module_from_path(module_name, python_file)
error = None
sys_modules = {k: v for k, v in sys.modules.items()}
with alter_sys_path(to_add=[], to_remove=[cwd]):
try:
module = import_module_from_path(module_name, python_file)
except ImportError as ie:
sys.modules = sys_modules
error = ie
if not error:
return module
try:
module = import_module_from_path(module_name, python_file)
# if here, we were able to resolve the module with the working directory on the
# path, but should error because we may not always invoke from the same directory
# (e.g. from cron)
warnings.warn((
'Module `{module}` was resolved using the working directory. The ability to '
'implicitly load modules from the working directory is deprecated and '
'will be removed in a future release. Please explicitly specify the '
'`working_directory` config option in your workspace.yaml or install `{module}` to '
'your python environment.').format(
module=error.name if hasattr(error, 'name') else module_name))
return module
except ImportError:
raise error
def load_python_file(python_file: str,
working_directory: Optional[str]) -> ModuleType:
"""
Takes a path to a python file and returns a loaded module
"""
check.str_param(python_file, "python_file")
check.opt_str_param(working_directory, "working_directory")
# First verify that the file exists
os.stat(python_file)
module_name = os.path.splitext(os.path.basename(python_file))[0]
# Use the passed in working directory for local imports (sys.path[0] isn't
# consistently set in the different entry points that Dagster uses to import code)
script_path = sys.path[0]
try:
with alter_sys_path(
to_add=([working_directory] if working_directory else []),
to_remove=[script_path]):
return import_module_from_path(module_name, python_file)
except ImportError as ie:
python_file = os.path.abspath(os.path.expanduser(python_file))
msg = get_import_error_message(ie)
if msg == "attempted relative import with no known parent package":
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name} from "
f"file {python_file}. Consider using the module-based options `-m` for "
"CLI-based targets or the `python_module` workspace target."
) from ie
if working_directory:
abs_working_directory = os.path.abspath(
os.path.expanduser(working_directory))
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name} from "
f"file {python_file}. Local modules were resolved using the working "
f"directory `{abs_working_directory}`. If another working directory should be "
"used, please explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace targets. "
) from ie
else:
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name} from file"
f" {python_file}. If relying on the working directory to resolve modules, please "
"explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace targets. "
) from ie
def workspace():
old_path = sys.path[:]
if '' not in sys.path:
sys.path.insert(0, '')
# TODO support schedules too?
objects = {'pipelines': {}}
workspace_folder = pathlib.Path('.')
for py_file in workspace_folder.glob('**/*.py'):
if py_file.name == 'solids.py' or py_file.name == 'workspace.py':
continue
mod = import_module_from_path(py_file.stem, str(py_file.absolute()))
for name, obj in inspect.getmembers(mod):
if isinstance(obj, PipelineDefinition):
objects['pipelines'][name] = obj
sys.path = old_path
return objects
def test_repo_entrypoints():
module = importlib.import_module('dagster_examples.intro_tutorial.repos')
expected = LoaderEntrypoint(module,
'dagster_examples.intro_tutorial.repos',
'define_repo')
handle = ExecutionTargetHandle.for_repo_yaml(
file_relative_path(__file__, 'repository.yaml'))
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
assert handle.entrypoint.from_handle == handle
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
module = importlib.import_module('dagster')
expected = LoaderEntrypoint(module, 'dagster', 'define_bar_repo')
handle = ExecutionTargetHandle.for_repo_module(module_name='dagster',
fn_name='define_bar_repo')
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
assert handle.entrypoint.from_handle == handle
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
python_file = file_relative_path(__file__, 'bar_repo.py')
module = import_module_from_path('bar_repo', python_file)
expected = LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo')
handle = ExecutionTargetHandle.for_repo_python_file(
python_file=python_file, fn_name='define_bar_repo')
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
assert handle.entrypoint.from_handle == handle
handle = ExecutionTargetHandle.from_dict(handle.to_dict())
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
def test_pipeline_python_file():
python_file = file_relative_path(__file__, 'foo_pipeline.py')
module = import_module_from_path('foo_pipeline', python_file)
handle = handle_for_pipeline_cli_args({
'module_name': None,
'fn_name': 'define_pipeline',
'pipeline_name': None,
'python_file': python_file,
'repository_yaml': None,
})
assert handle.mode == _ExecutionTargetMode.PIPELINE
expected = LoaderEntrypoint(module, 'foo_pipeline', 'define_pipeline')
assert handle.entrypoint.module == expected.module
assert handle.entrypoint.module_name == expected.module_name
assert handle.entrypoint.fn_name == expected.fn_name
assert handle.entrypoint.from_handle == handle
def get_app(config_yaml, config_module, config_file):
check.invariant(
not (sum(int(bool(x)) for x in [config_module, config_file, config_yaml]) > 1),
'Can only set one of --config-yaml/-y --config-module/-m or --config-file/-f',
)
if config_yaml:
celery_config = CeleryConfig(**get_config_value_from_yaml(config_yaml))
else:
celery_config = CeleryConfig()
app = make_app(celery_config)
if config_module:
app.config_from_object(config_module)
if config_file:
config_object = seven.import_module_from_path(
os.path.basename(config_file)[:-3], config_file
)
app.config_from_object(config_object)
return app
def load_python_file(python_file, working_directory):
"""
Takes a path to a python file and returns a loaded module
"""
check.str_param(python_file, "python_file")
module_name = os.path.splitext(os.path.basename(python_file))[0]
cwd = sys.path[0]
if working_directory:
try:
with alter_sys_path(to_add=[working_directory], to_remove=[cwd]):
return import_module_from_path(module_name, python_file)
except ImportError as ie:
msg = get_import_error_message(ie)
if msg == "attempted relative import with no known parent package":
six.raise_from(
DagsterImportError((
"Encountered ImportError: `{msg}` while importing module {module} from "
"file {python_file}. Consider using the module-based options `-m` for "
"CLI-based targets or the `python_package` workspace.yaml target."
).format(
msg=msg,
module=module_name,
python_file=os.path.abspath(
os.path.expanduser(python_file)),
)),
ie,
)
six.raise_from(
DagsterImportError((
"Encountered ImportError: `{msg}` while importing module {module} from "
"file {python_file}. Local modules were resolved using the working "
"directory `{working_directory}`. If another working directory should be "
"used, please explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace.yaml targets. "
).format(
msg=msg,
module=module_name,
python_file=os.path.abspath(
os.path.expanduser(python_file)),
working_directory=os.path.abspath(
os.path.expanduser(working_directory)),
)),
ie,
)
error = None
sys_modules = {k: v for k, v in sys.modules.items()}
with alter_sys_path(to_add=[], to_remove=[cwd]):
try:
module = import_module_from_path(module_name, python_file)
except ImportError as ie:
# importing alters sys.modules in ways that may interfere with the import below, even
# if the import has failed. to work around this, we need to manually clear any modules
# that have been cached in sys.modules due to the speculative import call
# Also, we are mutating sys.modules instead of straight-up assigning to sys_modules,
# because some packages will do similar shenanigans to sys.modules (e.g. numpy)
to_delete = set(sys.modules) - set(sys_modules)
for key in to_delete:
del sys.modules[key]
error = ie
if not error:
return module
try:
module = import_module_from_path(module_name, python_file)
# if here, we were able to resolve the module with the working directory on the
# path, but should warn because we may not always invoke from the same directory
# (e.g. from cron)
warnings.warn((
"Module `{module}` was resolved using the working directory. The ability to "
"implicitly load modules from the working directory is deprecated and "
"will be removed in a future release. Please explicitly specify the "
"`working_directory` config option in your workspace.yaml or install `{module}` to "
"your python environment.").format(
module=error.name if hasattr(error, "name") else module_name))
return module
except RuntimeError:
# We might be here because numpy throws run time errors at import time when being imported
# multiple times... we should also use the original import error as the root
six.raise_from(
DagsterImportError((
"Encountered ImportError: `{msg}` while importing module {module} from file "
"{python_file}. If relying on the working directory to resolve modules, please "
"explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace.yaml targets. "
+ error.msg).format(
msg=error.msg,
module=module_name,
python_file=os.path.abspath(
os.path.expanduser(python_file)),
)),
error,
)
except ImportError:
# raise the original import error
raise error
def test_import_module_from_path():
foo_module = seven.import_module_from_path(
'foo_module', file_relative_path(__file__, 'foo_module.py')
)
assert foo_module.FOO == 7
def test_import_module_from_path():
foo_module = seven.import_module_from_path(
"foo_module", file_relative_path(__file__, "foo_module.py"))
assert foo_module.FOO == 7
def test_import_module_from_path():
foo_module = seven.import_module_from_path(
'foo_module', script_relative_path('foo_module.py'))
assert foo_module.FOO == 7
