def load_python_file(python_file, working_directory): ''' Takes a path to a python file and returns a loaded module ''' check.str_param(python_file, 'python_file') module_name = os.path.splitext(os.path.basename(python_file))[0] cwd = sys.path[0] if working_directory: with alter_sys_path(to_add=[working_directory], to_remove=[cwd]): return import_module_from_path(module_name, python_file) error = None sys_modules = {k: v for k, v in sys.modules.items()} with alter_sys_path(to_add=[], to_remove=[cwd]): try: module = import_module_from_path(module_name, python_file) except ImportError as ie: # importing alters sys.modules in ways that may interfere with the import below, even # if the import has failed. to work around this, we need to manually clear any modules # that have been cached in sys.modules due to the speculative import call # Also, we are mutating sys.modules instead of straight-up assigning to sys_modules, # because some packages will do similar shenanigans to sys.modules (e.g. numpy) to_delete = set(sys.modules) - set(sys_modules) for key in to_delete: del sys.modules[key] error = ie if not error: return module try: module = import_module_from_path(module_name, python_file) # if here, we were able to resolve the module with the working directory on the # path, but should error because we may not always invoke from the same directory # (e.g. from cron) warnings.warn( ( 'Module `{module}` was resolved using the working directory. The ability to ' 'implicitly load modules from the working directory is deprecated and ' 'will be removed in a future release. Please explicitly specify the ' '`working_directory` config option in your workspace.yaml or install `{module}` to ' 'your python environment.' ).format(module=error.name if hasattr(error, 'name') else module_name) ) return module except ImportError: raise error
def load_python_file(python_file): ''' Takes a path to a python file and returns a loaded module ''' check.str_param(python_file, 'python_file') module_name = os.path.splitext(os.path.basename(python_file))[0] return import_module_from_path(module_name, python_file)
def test_repository_python_file(): python_file = script_relative_path('bar_repo.py') module = import_module_from_path('bar_repo', python_file) handle = handle_for_pipeline_cli_args( {'pipeline_name': 'foo', 'python_file': python_file, 'fn_name': 'define_bar_repo'} ) assert handle.mode == _ExecutionTargetMode.PIPELINE assert handle.entrypoint == LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo', handle) assert handle.data.pipeline_name == 'foo' assert handle.entrypoint.from_handle == handle with pytest.raises(UsageError): handle_for_pipeline_cli_args( { 'module_name': 'kdjfkd', 'pipeline_name': 'foo', 'python_file': script_relative_path('bar_repo.py'), 'fn_name': 'define_bar_repo', 'repository_yaml': None, } ) with pytest.raises(UsageError): handle_for_pipeline_cli_args( { 'module_name': None, 'pipeline_name': 'foo', 'python_file': script_relative_path('bar_repo.py'), 'fn_name': 'define_bar_repo', 'repository_yaml': 'kjdfkdjf', } )
def from_file_target(python_file, fn_name, from_handle=None): file_directory = os.path.dirname(python_file) if file_directory not in sys.path: sys.path.append(file_directory) module_name = os.path.splitext(os.path.basename(python_file))[0] module = import_module_from_path(module_name, python_file) return PartitionLoaderEntrypoint(module, module_name, fn_name, from_handle)
def load_target(self): module_name = os.path.splitext(os.path.basename(self.python_file))[0] module = import_module_from_path(module_name, self.python_file) if not hasattr(module, self.fn_name): raise DagsterInvariantViolationError( '{name} not found at module scope in file {file}.'.format( name=self.fn_name, file=self.python_file)) return getattr(module, self.fn_name)
def load_python_file(python_file, working_directory): ''' Takes a path to a python file and returns a loaded module ''' check.str_param(python_file, 'python_file') module_name = os.path.splitext(os.path.basename(python_file))[0] cwd = sys.path[0] if working_directory: with alter_sys_path(to_add=[working_directory], to_remove=[cwd]): return import_module_from_path(module_name, python_file) error = None sys_modules = {k: v for k, v in sys.modules.items()} with alter_sys_path(to_add=[], to_remove=[cwd]): try: module = import_module_from_path(module_name, python_file) except ImportError as ie: sys.modules = sys_modules error = ie if not error: return module try: module = import_module_from_path(module_name, python_file) # if here, we were able to resolve the module with the working directory on the # path, but should error because we may not always invoke from the same directory # (e.g. from cron) warnings.warn(( 'Module `{module}` was resolved using the working directory. The ability to ' 'implicitly load modules from the working directory is deprecated and ' 'will be removed in a future release. Please explicitly specify the ' '`working_directory` config option in your workspace.yaml or install `{module}` to ' 'your python environment.').format( module=error.name if hasattr(error, 'name') else module_name)) return module except ImportError: raise error
def load_python_file(python_file: str, working_directory: Optional[str]) -> ModuleType: """ Takes a path to a python file and returns a loaded module """ check.str_param(python_file, "python_file") check.opt_str_param(working_directory, "working_directory") # First verify that the file exists os.stat(python_file) module_name = os.path.splitext(os.path.basename(python_file))[0] # Use the passed in working directory for local imports (sys.path[0] isn't # consistently set in the different entry points that Dagster uses to import code) script_path = sys.path[0] try: with alter_sys_path( to_add=([working_directory] if working_directory else []), to_remove=[script_path]): return import_module_from_path(module_name, python_file) except ImportError as ie: python_file = os.path.abspath(os.path.expanduser(python_file)) msg = get_import_error_message(ie) if msg == "attempted relative import with no known parent package": raise DagsterImportError( f"Encountered ImportError: `{msg}` while importing module {module_name} from " f"file {python_file}. Consider using the module-based options `-m` for " "CLI-based targets or the `python_module` workspace target." ) from ie if working_directory: abs_working_directory = os.path.abspath( os.path.expanduser(working_directory)) raise DagsterImportError( f"Encountered ImportError: `{msg}` while importing module {module_name} from " f"file {python_file}. Local modules were resolved using the working " f"directory `{abs_working_directory}`. If another working directory should be " "used, please explicitly specify the appropriate path using the `-d` or " "`--working-directory` for CLI based targets or the `working_directory` " "configuration option for `python_file`-based workspace targets. " ) from ie else: raise DagsterImportError( f"Encountered ImportError: `{msg}` while importing module {module_name} from file" f" {python_file}. If relying on the working directory to resolve modules, please " "explicitly specify the appropriate path using the `-d` or " "`--working-directory` for CLI based targets or the `working_directory` " "configuration option for `python_file`-based workspace targets. " ) from ie
def workspace(): old_path = sys.path[:] if '' not in sys.path: sys.path.insert(0, '') # TODO support schedules too? objects = {'pipelines': {}} workspace_folder = pathlib.Path('.') for py_file in workspace_folder.glob('**/*.py'): if py_file.name == 'solids.py' or py_file.name == 'workspace.py': continue mod = import_module_from_path(py_file.stem, str(py_file.absolute())) for name, obj in inspect.getmembers(mod): if isinstance(obj, PipelineDefinition): objects['pipelines'][name] = obj sys.path = old_path return objects
def test_repo_entrypoints(): module = importlib.import_module('dagster_examples.intro_tutorial.repos') expected = LoaderEntrypoint(module, 'dagster_examples.intro_tutorial.repos', 'define_repo') handle = ExecutionTargetHandle.for_repo_yaml( file_relative_path(__file__, 'repository.yaml')) assert handle.entrypoint.module == expected.module assert handle.entrypoint.module_name == expected.module_name assert handle.entrypoint.fn_name == expected.fn_name assert handle.entrypoint.from_handle == handle handle = ExecutionTargetHandle.from_dict(handle.to_dict()) assert handle.entrypoint.module == expected.module assert handle.entrypoint.module_name == expected.module_name assert handle.entrypoint.fn_name == expected.fn_name module = importlib.import_module('dagster') expected = LoaderEntrypoint(module, 'dagster', 'define_bar_repo') handle = ExecutionTargetHandle.for_repo_module(module_name='dagster', fn_name='define_bar_repo') assert handle.entrypoint.module == expected.module assert handle.entrypoint.module_name == expected.module_name assert handle.entrypoint.fn_name == expected.fn_name assert handle.entrypoint.from_handle == handle handle = ExecutionTargetHandle.from_dict(handle.to_dict()) assert handle.entrypoint.module == expected.module assert handle.entrypoint.module_name == expected.module_name assert handle.entrypoint.fn_name == expected.fn_name python_file = file_relative_path(__file__, 'bar_repo.py') module = import_module_from_path('bar_repo', python_file) expected = LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo') handle = ExecutionTargetHandle.for_repo_python_file( python_file=python_file, fn_name='define_bar_repo') assert handle.entrypoint.module == expected.module assert handle.entrypoint.module_name == expected.module_name assert handle.entrypoint.fn_name == expected.fn_name assert handle.entrypoint.from_handle == handle handle = ExecutionTargetHandle.from_dict(handle.to_dict()) assert handle.entrypoint.module == expected.module assert handle.entrypoint.module_name == expected.module_name assert handle.entrypoint.fn_name == expected.fn_name
def test_pipeline_python_file(): python_file = file_relative_path(__file__, 'foo_pipeline.py') module = import_module_from_path('foo_pipeline', python_file) handle = handle_for_pipeline_cli_args({ 'module_name': None, 'fn_name': 'define_pipeline', 'pipeline_name': None, 'python_file': python_file, 'repository_yaml': None, }) assert handle.mode == _ExecutionTargetMode.PIPELINE expected = LoaderEntrypoint(module, 'foo_pipeline', 'define_pipeline') assert handle.entrypoint.module == expected.module assert handle.entrypoint.module_name == expected.module_name assert handle.entrypoint.fn_name == expected.fn_name assert handle.entrypoint.from_handle == handle
def get_app(config_yaml, config_module, config_file): check.invariant( not (sum(int(bool(x)) for x in [config_module, config_file, config_yaml]) > 1), 'Can only set one of --config-yaml/-y --config-module/-m or --config-file/-f', ) if config_yaml: celery_config = CeleryConfig(**get_config_value_from_yaml(config_yaml)) else: celery_config = CeleryConfig() app = make_app(celery_config) if config_module: app.config_from_object(config_module) if config_file: config_object = seven.import_module_from_path( os.path.basename(config_file)[:-3], config_file ) app.config_from_object(config_object) return app
def load_python_file(python_file, working_directory): """ Takes a path to a python file and returns a loaded module """ check.str_param(python_file, "python_file") module_name = os.path.splitext(os.path.basename(python_file))[0] cwd = sys.path[0] if working_directory: try: with alter_sys_path(to_add=[working_directory], to_remove=[cwd]): return import_module_from_path(module_name, python_file) except ImportError as ie: msg = get_import_error_message(ie) if msg == "attempted relative import with no known parent package": six.raise_from( DagsterImportError(( "Encountered ImportError: `{msg}` while importing module {module} from " "file {python_file}. Consider using the module-based options `-m` for " "CLI-based targets or the `python_package` workspace.yaml target." ).format( msg=msg, module=module_name, python_file=os.path.abspath( os.path.expanduser(python_file)), )), ie, ) six.raise_from( DagsterImportError(( "Encountered ImportError: `{msg}` while importing module {module} from " "file {python_file}. Local modules were resolved using the working " "directory `{working_directory}`. If another working directory should be " "used, please explicitly specify the appropriate path using the `-d` or " "`--working-directory` for CLI based targets or the `working_directory` " "configuration option for `python_file`-based workspace.yaml targets. " ).format( msg=msg, module=module_name, python_file=os.path.abspath( os.path.expanduser(python_file)), working_directory=os.path.abspath( os.path.expanduser(working_directory)), )), ie, ) error = None sys_modules = {k: v for k, v in sys.modules.items()} with alter_sys_path(to_add=[], to_remove=[cwd]): try: module = import_module_from_path(module_name, python_file) except ImportError as ie: # importing alters sys.modules in ways that may interfere with the import below, even # if the import has failed. to work around this, we need to manually clear any modules # that have been cached in sys.modules due to the speculative import call # Also, we are mutating sys.modules instead of straight-up assigning to sys_modules, # because some packages will do similar shenanigans to sys.modules (e.g. numpy) to_delete = set(sys.modules) - set(sys_modules) for key in to_delete: del sys.modules[key] error = ie if not error: return module try: module = import_module_from_path(module_name, python_file) # if here, we were able to resolve the module with the working directory on the # path, but should warn because we may not always invoke from the same directory # (e.g. from cron) warnings.warn(( "Module `{module}` was resolved using the working directory. The ability to " "implicitly load modules from the working directory is deprecated and " "will be removed in a future release. Please explicitly specify the " "`working_directory` config option in your workspace.yaml or install `{module}` to " "your python environment.").format( module=error.name if hasattr(error, "name") else module_name)) return module except RuntimeError: # We might be here because numpy throws run time errors at import time when being imported # multiple times... we should also use the original import error as the root six.raise_from( DagsterImportError(( "Encountered ImportError: `{msg}` while importing module {module} from file " "{python_file}. If relying on the working directory to resolve modules, please " "explicitly specify the appropriate path using the `-d` or " "`--working-directory` for CLI based targets or the `working_directory` " "configuration option for `python_file`-based workspace.yaml targets. " + error.msg).format( msg=error.msg, module=module_name, python_file=os.path.abspath( os.path.expanduser(python_file)), )), error, ) except ImportError: # raise the original import error raise error
def test_import_module_from_path(): foo_module = seven.import_module_from_path( 'foo_module', file_relative_path(__file__, 'foo_module.py') ) assert foo_module.FOO == 7
def test_import_module_from_path(): foo_module = seven.import_module_from_path( "foo_module", file_relative_path(__file__, "foo_module.py")) assert foo_module.FOO == 7
def test_import_module_from_path(): foo_module = seven.import_module_from_path( 'foo_module', script_relative_path('foo_module.py')) assert foo_module.FOO == 7