def load_python_file(python_file, working_directory):
'''
Takes a path to a python file and returns a loaded module
'''
check.str_param(python_file, 'python_file')
module_name = os.path.splitext(os.path.basename(python_file))[0]
cwd = sys.path[0]
if working_directory:
with alter_sys_path(to_add=[working_directory], to_remove=[cwd]):
return import_module_from_path(module_name, python_file)
error = None
sys_modules = {k: v for k, v in sys.modules.items()}
with alter_sys_path(to_add=[], to_remove=[cwd]):
try:
module = import_module_from_path(module_name, python_file)
except ImportError as ie:
# importing alters sys.modules in ways that may interfere with the import below, even
# if the import has failed. to work around this, we need to manually clear any modules
# that have been cached in sys.modules due to the speculative import call
# Also, we are mutating sys.modules instead of straight-up assigning to sys_modules,
# because some packages will do similar shenanigans to sys.modules (e.g. numpy)
to_delete = set(sys.modules) - set(sys_modules)
for key in to_delete:
del sys.modules[key]
error = ie
if not error:
return module
try:
module = import_module_from_path(module_name, python_file)
# if here, we were able to resolve the module with the working directory on the
# path, but should error because we may not always invoke from the same directory
# (e.g. from cron)
warnings.warn(
(
'Module `{module}` was resolved using the working directory. The ability to '
'implicitly load modules from the working directory is deprecated and '
'will be removed in a future release. Please explicitly specify the '
'`working_directory` config option in your workspace.yaml or install `{module}` to '
'your python environment.'
).format(module=error.name if hasattr(error, 'name') else module_name)
)
return module
except ImportError:
raise error
def load_python_module(module_name, working_directory, remove_from_path_fn=None):
check.str_param(module_name, "module_name")
check.opt_str_param(working_directory, "working_directory")
check.opt_callable_param(remove_from_path_fn, "remove_from_path_fn")
# Use the passed in working directory for local imports (sys.path[0] isn't
# consistently set in the different entry points that Dagster uses to import code)
remove_paths = remove_from_path_fn() if remove_from_path_fn else [] # hook for tests
remove_paths.insert(0, sys.path[0]) # remove the script path
with alter_sys_path(
to_add=([working_directory] if working_directory else []), to_remove=remove_paths
):
try:
return importlib.import_module(module_name)
except ImportError as ie:
msg = get_import_error_message(ie)
if working_directory:
abs_working_directory = os.path.abspath(os.path.expanduser(working_directory))
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name}. "
f"Local modules were resolved using the working "
f"directory `{abs_working_directory}`. If another working directory should be "
"used, please explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for workspace targets. "
) from ie
else:
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name}. "
f"If relying on the working directory to resolve modules, please "
"explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for workspace targets. "
) from ie
def test_local_directory_file():
path = file_relative_path(__file__,
"autodiscover_file_in_directory/repository.py")
with restore_sys_modules():
with pytest.raises(DagsterImportError) as exc_info:
loadable_targets_from_python_file(path)
assert "No module named 'autodiscover_src'" in str(exc_info.value)
with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]):
loadable_targets_from_python_file(
path, working_directory=os.path.dirname(path))
def load_python_file(python_file, working_directory):
'''
Takes a path to a python file and returns a loaded module
'''
check.str_param(python_file, 'python_file')
module_name = os.path.splitext(os.path.basename(python_file))[0]
cwd = sys.path[0]
if working_directory:
with alter_sys_path(to_add=[working_directory], to_remove=[cwd]):
return import_module_from_path(module_name, python_file)
error = None
sys_modules = {k: v for k, v in sys.modules.items()}
with alter_sys_path(to_add=[], to_remove=[cwd]):
try:
module = import_module_from_path(module_name, python_file)
except ImportError as ie:
sys.modules = sys_modules
error = ie
if not error:
return module
try:
module = import_module_from_path(module_name, python_file)
# if here, we were able to resolve the module with the working directory on the
# path, but should error because we may not always invoke from the same directory
# (e.g. from cron)
warnings.warn((
'Module `{module}` was resolved using the working directory. The ability to '
'implicitly load modules from the working directory is deprecated and '
'will be removed in a future release. Please explicitly specify the '
'`working_directory` config option in your workspace.yaml or install `{module}` to '
'your python environment.').format(
module=error.name if hasattr(error, 'name') else module_name))
return module
except ImportError:
raise error
def load_python_file(python_file: str,
working_directory: Optional[str]) -> ModuleType:
"""
Takes a path to a python file and returns a loaded module
"""
check.str_param(python_file, "python_file")
check.opt_str_param(working_directory, "working_directory")
# First verify that the file exists
os.stat(python_file)
module_name = os.path.splitext(os.path.basename(python_file))[0]
# Use the passed in working directory for local imports (sys.path[0] isn't
# consistently set in the different entry points that Dagster uses to import code)
script_path = sys.path[0]
try:
with alter_sys_path(
to_add=([working_directory] if working_directory else []),
to_remove=[script_path]):
return import_module_from_path(module_name, python_file)
except ImportError as ie:
python_file = os.path.abspath(os.path.expanduser(python_file))
msg = get_import_error_message(ie)
if msg == "attempted relative import with no known parent package":
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name} from "
f"file {python_file}. Consider using the module-based options `-m` for "
"CLI-based targets or the `python_module` workspace target."
) from ie
if working_directory:
abs_working_directory = os.path.abspath(
os.path.expanduser(working_directory))
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name} from "
f"file {python_file}. Local modules were resolved using the working "
f"directory `{abs_working_directory}`. If another working directory should be "
"used, please explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace targets. "
) from ie
else:
raise DagsterImportError(
f"Encountered ImportError: `{msg}` while importing module {module_name} from file"
f" {python_file}. If relying on the working directory to resolve modules, please "
"explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace targets. "
) from ie
def load_python_module(module_name, warn_only=False, remove_from_path_fn=None):
check.str_param(module_name, "module_name")
check.bool_param(warn_only, "warn_only")
check.opt_callable_param(remove_from_path_fn, "remove_from_path_fn")
error = None
remove_paths = remove_from_path_fn() if remove_from_path_fn else [] # hook for tests
remove_paths.insert(0, sys.path[0]) # remove the working directory
with alter_sys_path(to_add=[], to_remove=remove_paths):
try:
module = importlib.import_module(module_name)
except ImportError as ie:
error = ie
if error:
try:
module = importlib.import_module(module_name)
# if here, we were able to resolve the module with the working directory on the path,
# but should error because we may not always invoke from the same directory (e.g. from
# cron)
if warn_only:
warnings.warn(
(
"Module {module} was resolved using the working directory. The ability to "
"load uninstalled modules from the working directory is deprecated and "
"will be removed in a future release. Please use the python-file based "
"load arguments or install {module} to your python environment."
).format(module=module_name)
)
else:
six.raise_from(
DagsterInvariantViolationError(
(
"Module {module} not found. Packages must be installed rather than "
"relying on the working directory to resolve module loading."
).format(module=module_name)
),
error,
)
except RuntimeError:
# We might be here because numpy throws run time errors at import time when being
# imported multiple times, just raise the original import error
raise error
except ImportError as ie:
raise error
return module
def test_local_directory_file():
path = file_relative_path(__file__,
"autodiscover_file_in_directory/repository.py")
with restore_sys_modules():
with pytest.raises(ImportError) as exc_info:
loadable_targets_from_python_file(path)
assert str(exc_info.value) == "No module named 'autodiscover_src'"
with pytest.warns(
UserWarning,
match=re.escape((
"Module `{module}` was resolved using the working directory. The ability to "
"implicitly load modules from the working directory is deprecated and will be removed "
"in a future release. Please explicitly specify the `working_directory` config option "
"in your workspace.yaml or install `{module}` to your python environment."
).format(module="autodiscover_src")),
):
with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]):
loadable_targets_from_python_file(path)
def test_local_directory_file():
path = file_relative_path(__file__,
'autodiscover_file_in_directory/repository.py')
is_py27 = sys.version_info < (3, )
with restore_sys_modules():
with pytest.raises(ImportError) as exc_info:
loadable_targets_from_python_file(path)
assert str(exc_info.value) in [
'No module named \'autodiscover_src\'', # py3+
'No module named autodiscover_src.pipelines', # py27
]
with pytest.warns(
UserWarning,
match=re.escape((
'Module `{module}` was resolved using the working directory. The ability to '
'implicitly load modules from the working directory is deprecated and will be removed '
'in a future release. Please explicitly specify the `working_directory` config option '
'in your workspace.yaml or install `{module}` to your python environment.'
).format(module='repository' if is_py27 else 'autodiscover_src')),
):
with alter_sys_path(to_add=[os.path.dirname(path)], to_remove=[]):
loadable_targets_from_python_file(path)
def load_python_file(python_file, working_directory):
"""
Takes a path to a python file and returns a loaded module
"""
check.str_param(python_file, "python_file")
module_name = os.path.splitext(os.path.basename(python_file))[0]
cwd = sys.path[0]
if working_directory:
try:
with alter_sys_path(to_add=[working_directory], to_remove=[cwd]):
return import_module_from_path(module_name, python_file)
except ImportError as ie:
msg = get_import_error_message(ie)
if msg == "attempted relative import with no known parent package":
six.raise_from(
DagsterImportError((
"Encountered ImportError: `{msg}` while importing module {module} from "
"file {python_file}. Consider using the module-based options `-m` for "
"CLI-based targets or the `python_package` workspace.yaml target."
).format(
msg=msg,
module=module_name,
python_file=os.path.abspath(
os.path.expanduser(python_file)),
)),
ie,
)
six.raise_from(
DagsterImportError((
"Encountered ImportError: `{msg}` while importing module {module} from "
"file {python_file}. Local modules were resolved using the working "
"directory `{working_directory}`. If another working directory should be "
"used, please explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace.yaml targets. "
).format(
msg=msg,
module=module_name,
python_file=os.path.abspath(
os.path.expanduser(python_file)),
working_directory=os.path.abspath(
os.path.expanduser(working_directory)),
)),
ie,
)
error = None
sys_modules = {k: v for k, v in sys.modules.items()}
with alter_sys_path(to_add=[], to_remove=[cwd]):
try:
module = import_module_from_path(module_name, python_file)
except ImportError as ie:
# importing alters sys.modules in ways that may interfere with the import below, even
# if the import has failed. to work around this, we need to manually clear any modules
# that have been cached in sys.modules due to the speculative import call
# Also, we are mutating sys.modules instead of straight-up assigning to sys_modules,
# because some packages will do similar shenanigans to sys.modules (e.g. numpy)
to_delete = set(sys.modules) - set(sys_modules)
for key in to_delete:
del sys.modules[key]
error = ie
if not error:
return module
try:
module = import_module_from_path(module_name, python_file)
# if here, we were able to resolve the module with the working directory on the
# path, but should warn because we may not always invoke from the same directory
# (e.g. from cron)
warnings.warn((
"Module `{module}` was resolved using the working directory. The ability to "
"implicitly load modules from the working directory is deprecated and "
"will be removed in a future release. Please explicitly specify the "
"`working_directory` config option in your workspace.yaml or install `{module}` to "
"your python environment.").format(
module=error.name if hasattr(error, "name") else module_name))
return module
except RuntimeError:
# We might be here because numpy throws run time errors at import time when being imported
# multiple times... we should also use the original import error as the root
six.raise_from(
DagsterImportError((
"Encountered ImportError: `{msg}` while importing module {module} from file "
"{python_file}. If relying on the working directory to resolve modules, please "
"explicitly specify the appropriate path using the `-d` or "
"`--working-directory` for CLI based targets or the `working_directory` "
"configuration option for `python_file`-based workspace.yaml targets. "
+ error.msg).format(
msg=error.msg,
module=module_name,
python_file=os.path.abspath(
os.path.expanduser(python_file)),
)),
error,
)
except ImportError:
# raise the original import error
raise error
