def reload_kedro(path, line=None):
"""Line magic which reloads all Kedro default variables."""
global startup_error
global context
global catalog
global session
try:
import kedro.config.default_logger
from kedro.framework.hooks import get_hook_manager
from kedro.framework.project import configure_project
from kedro.framework.session import KedroSession
from kedro.framework.session.session import _activate_session
from kedro.framework.cli.jupyter import collect_line_magic
except ImportError:
logging.error(
"Kedro appears not to be installed in your current environment "
"or your current IPython session was not started in a valid Kedro project."
)
raise
try:
path = path or project_path
# clear hook manager
hook_manager = get_hook_manager()
name_plugin_pairs = hook_manager.list_name_plugin()
for name, plugin in name_plugin_pairs:
hook_manager.unregister(name=name, plugin=plugin)
# remove cached user modules
metadata = _get_project_metadata(path)
to_remove = [
mod for mod in sys.modules if mod.startswith(metadata.package_name)
]
# `del` is used instead of `reload()` because: If the new version of a module does not
# define a name that was defined by the old version, the old definition remains.
for module in to_remove:
del sys.modules[module]
configure_project(metadata.package_name)
session = KedroSession.create(metadata.package_name, path)
_activate_session(session, force=True)
logging.debug("Loading the context from %s", str(path))
context = session.load_context()
catalog = context.catalog
logging.info("** Kedro project %s", str(metadata.project_name))
logging.info("Defined global variable `context` and `catalog`")
for line_magic in collect_line_magic():
register_line_magic(needs_local_scope(line_magic))
logging.info("Registered line magic `%s`", line_magic.__name__)
except Exception as err:
startup_error = err
logging.exception("Kedro's ipython session startup script failed:\n%s",
str(err))
raise err
def reload_kedro(path, line=None):
"""Line magic which reloads all Kedro default variables."""
global startup_error
global context
global catalog
try:
import kedro.config.default_logger # noqa
from kedro.framework.cli.jupyter import collect_line_magic
from kedro.framework.context import load_context
except ImportError:
logging.error(
"Kedro appears not to be installed in your current environment "
"or your current IPython session was not started in a valid Kedro project."
)
raise
try:
path = path or project_path
# remove cached user modules
context = load_context(path)
to_remove = [
mod for mod in sys.modules if mod.startswith(context.package_name)
]
# `del` is used instead of `reload()` because: If the new version of a module does not
# define a name that was defined by the old version, the old definition remains.
for module in to_remove:
del sys.modules[module]
# clear hook manager; hook implementations will be re-registered when the
# context is instantiated again in `load_context()` below
hook_manager = get_hook_manager()
name_plugin_pairs = hook_manager.list_name_plugin()
for name, plugin in name_plugin_pairs:
hook_manager.unregister(name=name, plugin=plugin)
logging.debug("Loading the context from %s", str(path))
# Reload context to fix `pickle` related error (it is unable to serialize reloaded objects)
# Some details can be found here:
# https://modwsgi.readthedocs.io/en/develop/user-guides/issues-with-pickle-module.html#packing-and-script-reloading
context = load_context(path)
catalog = context.catalog
logging.info("** Kedro project %s", str(context.project_name))
logging.info("Defined global variable `context` and `catalog`")
for line_magic in collect_line_magic():
register_line_magic(needs_local_scope(line_magic))
logging.info("Registered line magic `%s`", line_magic.__name__)
except Exception as err:
startup_error = err
logging.exception("Kedro's ipython session startup script failed:\n%s",
str(err))
raise err
def reload_kedro(path, env: str = None, extra_params: Dict[str, Any] = None):
"""Line magic which reloads all Kedro default variables."""
import kedro.config.default_logger # noqa: F401 # pylint: disable=unused-import
from kedro.framework.cli import load_entry_points
from kedro.framework.project import pipelines
from kedro.framework.session import KedroSession
from kedro.framework.session.session import _activate_session
from kedro.framework.startup import bootstrap_project
_clear_hook_manager()
path = path or project_path
metadata = bootstrap_project(path)
_remove_cached_modules(metadata.package_name)
session = KedroSession.create(metadata.package_name,
path,
env=env,
extra_params=extra_params)
_activate_session(session, force=True)
logging.debug("Loading the context from %s", str(path))
context = session.load_context()
catalog = context.catalog
get_ipython().push(
variables={
"context": context,
"catalog": catalog,
"session": session,
"pipelines": pipelines,
})
logging.info("** Kedro project %s", str(metadata.project_name))
logging.info(
"Defined global variable `context`, `session`, `catalog` and `pipelines`"
)
for line_magic in load_entry_points("line_magic"):
register_line_magic(needs_local_scope(line_magic))
logging.info("Registered line magic `%s`",
line_magic.__name__) # type: ignore
def load_kedro_objects(path, line=None): # pylint: disable=unused-argument
"""Line magic which reloads all Kedro default variables."""
import kedro.config.default_logger # noqa: F401 # pylint: disable=unused-import
from kedro.framework.cli import load_entry_points
from kedro.framework.cli.utils import _add_src_to_path
from kedro.framework.project import configure_project
from kedro.framework.session import KedroSession
from kedro.framework.session.session import _activate_session
from kedro.framework.startup import _get_project_metadata
global context
global catalog
global session
path = path or project_path
metadata = _get_project_metadata(path)
_add_src_to_path(metadata.source_dir, path)
configure_project(metadata.package_name)
_clear_hook_manager()
_remove_cached_modules(metadata.package_name)
session = KedroSession.create(metadata.package_name, path)
_activate_session(session)
logging.debug("Loading the context from %s", str(path))
context = session.load_context()
catalog = context.catalog
get_ipython().push(variables={
"context": context,
"catalog": catalog,
"session": session
})
logging.info("** Kedro project %s", str(metadata.project_name))
logging.info("Defined global variable `context`, `session` and `catalog`")
for line_magic in load_entry_points("line_magic"):
register_line_magic(needs_local_scope(line_magic))
logging.info("Registered line magic `%s`", line_magic.__name__)
def nbdev_add2all(line, local_ns):
"""To add something to `__all__` if it's not picked automatically,
write an exported cell with something like: `%nbdev_add2all name_1, name_2 ...`"""
if line.strip() == '':
print(f'UsageError: List of names is missing. Usage `%nbdev_add2all name_1, name_2`')
return
try: [eval(s, local_ns) for s in parse_line(line)]
except Exception as ex: print(f'UsageError: {ex}')
def _new_test_flag_fn(flag):
"Create a new test flag function and magic"
# don't create "empty" test flags if tst_flags is not set, set to whitespace, has trailing | etc
if not flag.strip(): return
def _(line): _validate_param(line, f'nbdev_{flag}_test', fixed_value='all')
_.__doc__ = f"""Put an `%nbdev_{flag}_test` magic on each "{flag}" test cell that you do not want to be run by default.
To apply this flag to all tests in a notebook, one cell should contain: `%nbdev_{flag}_test all`
These tests can be executed with the command line tool: `nbdev_test_nbs --flags {flag}`'."""
register_line_magic(f'nbdev_{flag}_test')(_)
if IN_IPYTHON:
from IPython.core.magic import register_line_magic, needs_local_scope
fns = [nbdev_default_export, nbdev_export, nbdev_export_and_show, nbdev_export_internal,
nbdev_hide, nbdev_hide_input, nbdev_hide_output, nbdev_default_class_level,
nbdev_collapse_input, nbdev_collapse_output, needs_local_scope(nbdev_add2all)]
for fn in fns: register_line_magic(fn)
try:
for flag in Config().get('tst_flags', '').split('|'): _new_test_flag_fn(flag)
except: pass # do not fail if we can't find config
`%nbdev_show_doc name_1, name_2, title_level=3`.
To show doc for a class and some of its members and specify class level:
`%nbdev_show_doc MyClass . __init__, my_method, default_cls_level=3`
To show doc for a class and all of its "public" members:
`%nbdev_show_doc MyClass *`"""
names, wild_names, kwargs = parse_nbdev_show_doc(line, local_ns)
for k,v in kwargs.items():
if not 1 <= v <= 6:
print(f'UsageError: Invalid {k} "{v}". Usage `%nbdev_show_doc name_1 {k}=[int between 1 and 6]`')
if not names:
print(f'UsageError: List of names is missing. Usage `%nbdev_show_doc name_1, name_2`')
for name in names: show_doc(eval(name,local_ns), name=name, **kwargs)
if IN_IPYTHON:
from IPython.core.magic import register_line_magic, needs_local_scope
register_line_magic(needs_local_scope(nbdev_show_doc))
# Cell
def md2html(md):
"Convert markdown `md` to HTML code"
import nbconvert
if nbconvert.__version__ < '5.5.0': return HTMLExporter().markdown2html(md)
else: return HTMLExporter().markdown2html(collections.defaultdict(lambda: collections.defaultdict(dict)), md)
# Cell
def get_doc_link(func):
mod = inspect.getmodule(func)
module = mod.__name__.replace('.', '/') + '.py'
try:
nbdev_mod = importlib.import_module(mod.__package__.split('.')[0] + '._nbdev')
try_pack = source_nb(func, mod=nbdev_mod)
def load_ipython_extension(ipython):
"""Load the ipython magic, when the module is called via ``%load_ext``."""
from IPython.core.magic import register_line_cell_magic, needs_local_scope
register_line_cell_magic(needs_local_scope(pytest))
except Exception as ex:
print(f'UsageError: {ex}')
def _new_test_flag_fn(flag):
"Create a new test flag function and magic"
# don't create "empty" test flags if tst_flags is not set, set to whitespace, has trailing | etc
if not flag.strip(): return
def _(line):
_validate_param(line, f'nbdev_{flag}_test', fixed_value='all')
_.__doc__ = f"""Put an `%nbdev_{flag}_test` magic on each "{flag}" test cell that you do not want to be run by default.
To apply this flag to all tests in a notebook, one cell should contain: `%nbdev_{flag}_test all`
These tests can be executed with the command line tool: `nbdev_test_nbs --flags {flag}`'."""
register_line_magic(f'nbdev_{flag}_test')(_)
if IN_IPYTHON:
from IPython.core.magic import register_line_magic, needs_local_scope
fns = [
nbdev_default_export, nbdev_export, nbdev_export_and_show,
nbdev_export_internal, nbdev_hide, nbdev_hide_input, nbdev_hide_output,
nbdev_default_class_level, nbdev_collapse_input, nbdev_collapse_output,
needs_local_scope(nbdev_add2all)
]
for fn in fns:
register_line_magic(fn)
for flag in Config().get('tst_flags', '').split('|'):
_new_test_flag_fn(flag)
