def reload_kedro(path, line=None): """Line magic which reloads all Kedro default variables.""" global startup_error global context global catalog global session try: import kedro.config.default_logger from kedro.framework.hooks import get_hook_manager from kedro.framework.project import configure_project from kedro.framework.session import KedroSession from kedro.framework.session.session import _activate_session from kedro.framework.cli.jupyter import collect_line_magic except ImportError: logging.error( "Kedro appears not to be installed in your current environment " "or your current IPython session was not started in a valid Kedro project." ) raise try: path = path or project_path # clear hook manager hook_manager = get_hook_manager() name_plugin_pairs = hook_manager.list_name_plugin() for name, plugin in name_plugin_pairs: hook_manager.unregister(name=name, plugin=plugin) # remove cached user modules metadata = _get_project_metadata(path) to_remove = [ mod for mod in sys.modules if mod.startswith(metadata.package_name) ] # `del` is used instead of `reload()` because: If the new version of a module does not # define a name that was defined by the old version, the old definition remains. for module in to_remove: del sys.modules[module] configure_project(metadata.package_name) session = KedroSession.create(metadata.package_name, path) _activate_session(session, force=True) logging.debug("Loading the context from %s", str(path)) context = session.load_context() catalog = context.catalog logging.info("** Kedro project %s", str(metadata.project_name)) logging.info("Defined global variable `context` and `catalog`") for line_magic in collect_line_magic(): register_line_magic(needs_local_scope(line_magic)) logging.info("Registered line magic `%s`", line_magic.__name__) except Exception as err: startup_error = err logging.exception("Kedro's ipython session startup script failed:\n%s", str(err)) raise err
def reload_kedro(path, line=None): """Line magic which reloads all Kedro default variables.""" global startup_error global context global catalog try: import kedro.config.default_logger # noqa from kedro.framework.cli.jupyter import collect_line_magic from kedro.framework.context import load_context except ImportError: logging.error( "Kedro appears not to be installed in your current environment " "or your current IPython session was not started in a valid Kedro project." ) raise try: path = path or project_path # remove cached user modules context = load_context(path) to_remove = [ mod for mod in sys.modules if mod.startswith(context.package_name) ] # `del` is used instead of `reload()` because: If the new version of a module does not # define a name that was defined by the old version, the old definition remains. for module in to_remove: del sys.modules[module] # clear hook manager; hook implementations will be re-registered when the # context is instantiated again in `load_context()` below hook_manager = get_hook_manager() name_plugin_pairs = hook_manager.list_name_plugin() for name, plugin in name_plugin_pairs: hook_manager.unregister(name=name, plugin=plugin) logging.debug("Loading the context from %s", str(path)) # Reload context to fix `pickle` related error (it is unable to serialize reloaded objects) # Some details can be found here: # https://modwsgi.readthedocs.io/en/develop/user-guides/issues-with-pickle-module.html#packing-and-script-reloading context = load_context(path) catalog = context.catalog logging.info("** Kedro project %s", str(context.project_name)) logging.info("Defined global variable `context` and `catalog`") for line_magic in collect_line_magic(): register_line_magic(needs_local_scope(line_magic)) logging.info("Registered line magic `%s`", line_magic.__name__) except Exception as err: startup_error = err logging.exception("Kedro's ipython session startup script failed:\n%s", str(err)) raise err
def reload_kedro(path, env: str = None, extra_params: Dict[str, Any] = None): """Line magic which reloads all Kedro default variables.""" import kedro.config.default_logger # noqa: F401 # pylint: disable=unused-import from kedro.framework.cli import load_entry_points from kedro.framework.project import pipelines from kedro.framework.session import KedroSession from kedro.framework.session.session import _activate_session from kedro.framework.startup import bootstrap_project _clear_hook_manager() path = path or project_path metadata = bootstrap_project(path) _remove_cached_modules(metadata.package_name) session = KedroSession.create(metadata.package_name, path, env=env, extra_params=extra_params) _activate_session(session, force=True) logging.debug("Loading the context from %s", str(path)) context = session.load_context() catalog = context.catalog get_ipython().push( variables={ "context": context, "catalog": catalog, "session": session, "pipelines": pipelines, }) logging.info("** Kedro project %s", str(metadata.project_name)) logging.info( "Defined global variable `context`, `session`, `catalog` and `pipelines`" ) for line_magic in load_entry_points("line_magic"): register_line_magic(needs_local_scope(line_magic)) logging.info("Registered line magic `%s`", line_magic.__name__) # type: ignore
def load_kedro_objects(path, line=None): # pylint: disable=unused-argument """Line magic which reloads all Kedro default variables.""" import kedro.config.default_logger # noqa: F401 # pylint: disable=unused-import from kedro.framework.cli import load_entry_points from kedro.framework.cli.utils import _add_src_to_path from kedro.framework.project import configure_project from kedro.framework.session import KedroSession from kedro.framework.session.session import _activate_session from kedro.framework.startup import _get_project_metadata global context global catalog global session path = path or project_path metadata = _get_project_metadata(path) _add_src_to_path(metadata.source_dir, path) configure_project(metadata.package_name) _clear_hook_manager() _remove_cached_modules(metadata.package_name) session = KedroSession.create(metadata.package_name, path) _activate_session(session) logging.debug("Loading the context from %s", str(path)) context = session.load_context() catalog = context.catalog get_ipython().push(variables={ "context": context, "catalog": catalog, "session": session }) logging.info("** Kedro project %s", str(metadata.project_name)) logging.info("Defined global variable `context`, `session` and `catalog`") for line_magic in load_entry_points("line_magic"): register_line_magic(needs_local_scope(line_magic)) logging.info("Registered line magic `%s`", line_magic.__name__)
def nbdev_add2all(line, local_ns): """To add something to `__all__` if it's not picked automatically, write an exported cell with something like: `%nbdev_add2all name_1, name_2 ...`""" if line.strip() == '': print(f'UsageError: List of names is missing. Usage `%nbdev_add2all name_1, name_2`') return try: [eval(s, local_ns) for s in parse_line(line)] except Exception as ex: print(f'UsageError: {ex}') def _new_test_flag_fn(flag): "Create a new test flag function and magic" # don't create "empty" test flags if tst_flags is not set, set to whitespace, has trailing | etc if not flag.strip(): return def _(line): _validate_param(line, f'nbdev_{flag}_test', fixed_value='all') _.__doc__ = f"""Put an `%nbdev_{flag}_test` magic on each "{flag}" test cell that you do not want to be run by default. To apply this flag to all tests in a notebook, one cell should contain: `%nbdev_{flag}_test all` These tests can be executed with the command line tool: `nbdev_test_nbs --flags {flag}`'.""" register_line_magic(f'nbdev_{flag}_test')(_) if IN_IPYTHON: from IPython.core.magic import register_line_magic, needs_local_scope fns = [nbdev_default_export, nbdev_export, nbdev_export_and_show, nbdev_export_internal, nbdev_hide, nbdev_hide_input, nbdev_hide_output, nbdev_default_class_level, nbdev_collapse_input, nbdev_collapse_output, needs_local_scope(nbdev_add2all)] for fn in fns: register_line_magic(fn) try: for flag in Config().get('tst_flags', '').split('|'): _new_test_flag_fn(flag) except: pass # do not fail if we can't find config
`%nbdev_show_doc name_1, name_2, title_level=3`. To show doc for a class and some of its members and specify class level: `%nbdev_show_doc MyClass . __init__, my_method, default_cls_level=3` To show doc for a class and all of its "public" members: `%nbdev_show_doc MyClass *`""" names, wild_names, kwargs = parse_nbdev_show_doc(line, local_ns) for k,v in kwargs.items(): if not 1 <= v <= 6: print(f'UsageError: Invalid {k} "{v}". Usage `%nbdev_show_doc name_1 {k}=[int between 1 and 6]`') if not names: print(f'UsageError: List of names is missing. Usage `%nbdev_show_doc name_1, name_2`') for name in names: show_doc(eval(name,local_ns), name=name, **kwargs) if IN_IPYTHON: from IPython.core.magic import register_line_magic, needs_local_scope register_line_magic(needs_local_scope(nbdev_show_doc)) # Cell def md2html(md): "Convert markdown `md` to HTML code" import nbconvert if nbconvert.__version__ < '5.5.0': return HTMLExporter().markdown2html(md) else: return HTMLExporter().markdown2html(collections.defaultdict(lambda: collections.defaultdict(dict)), md) # Cell def get_doc_link(func): mod = inspect.getmodule(func) module = mod.__name__.replace('.', '/') + '.py' try: nbdev_mod = importlib.import_module(mod.__package__.split('.')[0] + '._nbdev') try_pack = source_nb(func, mod=nbdev_mod)
def load_ipython_extension(ipython): """Load the ipython magic, when the module is called via ``%load_ext``.""" from IPython.core.magic import register_line_cell_magic, needs_local_scope register_line_cell_magic(needs_local_scope(pytest))
except Exception as ex: print(f'UsageError: {ex}') def _new_test_flag_fn(flag): "Create a new test flag function and magic" # don't create "empty" test flags if tst_flags is not set, set to whitespace, has trailing | etc if not flag.strip(): return def _(line): _validate_param(line, f'nbdev_{flag}_test', fixed_value='all') _.__doc__ = f"""Put an `%nbdev_{flag}_test` magic on each "{flag}" test cell that you do not want to be run by default. To apply this flag to all tests in a notebook, one cell should contain: `%nbdev_{flag}_test all` These tests can be executed with the command line tool: `nbdev_test_nbs --flags {flag}`'.""" register_line_magic(f'nbdev_{flag}_test')(_) if IN_IPYTHON: from IPython.core.magic import register_line_magic, needs_local_scope fns = [ nbdev_default_export, nbdev_export, nbdev_export_and_show, nbdev_export_internal, nbdev_hide, nbdev_hide_input, nbdev_hide_output, nbdev_default_class_level, nbdev_collapse_input, nbdev_collapse_output, needs_local_scope(nbdev_add2all) ] for fn in fns: register_line_magic(fn) for flag in Config().get('tst_flags', '').split('|'): _new_test_flag_fn(flag)