def execute_notebook(source_notebook, destination_notebook, infra_args, papermill_args, notebooks_args):
"""Execute notebook with papermill.
:param source_notebook: Source notebook file name
:type source_notebook: str
:param destination_notebook: Source notebook file name
:type source_nodestination_notebooktebook: str
:param infra_args: Infrastructure arguments
:type infra_args: dict
:param papermill_args: Papermill arguments
:type papermill_args: dict
:param notebooks_args: Notebook arguments
:type notebooks_args: dict
"""
import papermill as pm
# if kernel name is specified
kernel_name = papermill_args.get("kernel_name")
# if not specified try to get it from the notebook
if not kernel_name:
with open(source_notebook) as nbfile:
notebook = json.loads(nbfile.read())
try:
kernel_name = notebook.get("metadata").get("kernelspec").get("name")
except:
pass
# create a kernel spec if not installed
try:
if kernel_name:
from jupyter_client.kernelspec import KernelSpecManager
if not KernelSpecManager().get_all_specs().get(kernel_name):
# TODO: replace jupyter_client.kernelspec.KernelSpecManager logic
from ipykernel.kernelspec import install
install(kernel_name=kernel_name)
papermill_args["kernel_name"] = kernel_name
except:
pass
# execute notebook
pm.execute_notebook(
source_notebook,
destination_notebook,
parameters=notebooks_args,
**papermill_args
)
# TODO scrapbook
if infra_args.get("history"):
batchsize = 50
from azureml.core import Run
results = pm.read_notebook(destination_notebook).dataframe.set_index("name")["value"]
run = Run.get_context()
for key, value in results.items():
if isinstance(value, list):
for index in range(0, len(value), batchsize):
run.log_list(key, value[index:index + batchsize])
else:
run.log(key, value)
def render_ipynb(template_ipynb, rendered_ipynb, params=dict()):
"""Render the ipython notebook
Args:
template_ipynb: template ipython notebook where one cell defines the following metadata:
{"tags": ["parameters"]}
render_ipynb: output ipython notebook path
params: parameters used to execute the ipython notebook
"""
import jupyter_client
os.makedirs(os.path.dirname(rendered_ipynb), exist_ok=True)
kernel_name = os.environ.get("CONDA_DEFAULT_ENV", 'python3')
if kernel_name not in jupyter_client.kernelspec.find_kernel_specs():
logger.info(
f"Installing the ipython kernel for the current conda environment: {kernel_name}"
)
from ipykernel.kernelspec import install
install(user=True, kernel_name=kernel_name)
pm.execute_notebook(
template_ipynb, # input template
rendered_ipynb,
kernel_name=kernel_name, # default kernel
parameters=params)
jupyter_nbconvert(rendered_ipynb)
def install_native_kernel_spec(self, user=False):
"""DEPRECATED: Use ipykernel.kenelspec.install"""
warnings.warn(
"install_native_kernel_spec is deprecated." " Use ipykernel.kernelspec import install.", stacklevel=2
)
from ipykernel.kernelspec import install
install(self, user=user)
def install_native_kernel_spec(self, user=False):
"""DEPRECATED: Use ipykernel.kenelspec.install"""
warnings.warn(
"install_native_kernel_spec is deprecated."
" Use ipykernel.kernelspec import install.",
stacklevel=2)
from ipykernel.kernelspec import install
install(self, user=user)
def test_install():
system_jupyter_dir = tempfile.mkdtemp()
with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH',
[system_jupyter_dir]):
install()
assert_is_spec(os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME))
def test_install_user():
tmp = tempfile.mkdtemp()
with mock.patch.dict(os.environ, {'HOME': tmp}):
install(user=True)
data_dir = jupyter_data_dir()
assert_is_spec(os.path.join(data_dir, 'kernels', KERNEL_NAME))
def test_install_user():
tmp = tempfile.mkdtemp()
with mock.patch.dict(os.environ, {'HOME': tmp}):
install(user=True)
data_dir = jupyter_data_dir()
assert_is_spec(os.path.join(data_dir, 'kernels', KERNEL_NAME))
def test_install():
system_jupyter_dir = tempfile.mkdtemp()
with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH',
[system_jupyter_dir]):
install()
assert_is_spec(os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME))
def start(self):
try:
from ipykernel import kernelspec
except ImportError:
print("ipykernel not available, can't install its spec.", file=sys.stderr)
self.exit(1)
try:
kernelspec.install(self.kernel_spec_manager, user=self.user)
except OSError as e:
self.exit(e)
def test_install_display_name_overrides_profile():
system_jupyter_dir = tempfile.mkdtemp()
with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH',
[system_jupyter_dir]):
install(display_name="Display", profile="Test")
spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json")
with open(spec) as f:
spec = json.load(f)
assert spec["display_name"] == "Display"
def start(self):
try:
from ipykernel import kernelspec
except ImportError:
print("ipykernel not available, can't install its spec.",
file=sys.stderr)
self.exit(1)
try:
kernelspec.install(self.kernel_spec_manager, user=self.user)
except OSError as e:
self.exit(e)
def test_install_display_name_overrides_profile():
system_jupyter_dir = tempfile.mkdtemp()
with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH',
[system_jupyter_dir]):
install(display_name="Display", profile="Test")
spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json")
with open(spec) as f:
spec = json.load(f)
nt.assert_equal(spec["display_name"], "Display")
def setup():
"""setup temporary env for tests"""
global tmp
tmp = tempfile.mkdtemp()
patchers[:] = [
patch.dict(os.environ, {'HOME': tmp}),
]
for p in patchers:
p.start()
# install IPython in the temp home:
install(user=True)
def init_kernel_specs(self):
"""Check that the IPython kernel is present, if available"""
try:
self.kernel_spec_manager.get_kernel_spec(NATIVE_KERNEL_NAME)
except NoSuchKernel:
try:
from ipykernel.kernelspec import install
except ImportError:
self.log.warn("IPython kernel not available")
else:
self.log.warn("Installing IPython kernel spec")
install(kernel_spec_manager=self.kernel_spec_manager, user=True)
def test_install_profile():
system_jupyter_dir = tempfile.mkdtemp()
with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH',
[system_jupyter_dir]):
install(profile="Test")
spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json")
with open(spec) as f:
spec = json.load(f)
assert spec["display_name"].endswith(" [profile=Test]")
nt.assert_equal(spec["argv"][-2:], ["--profile", "Test"])
def test_install_profile():
system_jupyter_dir = tempfile.mkdtemp()
with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH',
[system_jupyter_dir]):
install(profile="Test")
spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json")
with open(spec) as f:
spec = json.load(f)
nt.assert_true(spec["display_name"].endswith(" [profile=Test]"))
nt.assert_equal(spec["argv"][-2:], ["--profile", "Test"])
def setup():
"""setup temporary env for tests"""
global tmp
tmp = tempfile.mkdtemp()
patchers[:] = [
patch.dict(os.environ, {'HOME': tmp}),
]
for p in patchers:
p.start()
# install IPython in the temp home:
install(user=True)
def start(self):
try:
from ipykernel import kernelspec
except ImportError:
print("ipykernel not available, can't install its spec.", file=sys.stderr)
self.exit(1)
try:
kernelspec.install(self.kernel_spec_manager, user=self.user)
except OSError as e:
if e.errno == errno.EACCES:
print(e, file=sys.stderr)
if not self.user:
print("Perhaps you want to install with `sudo` or `--user`?", file=sys.stderr)
self.exit(1)
self.exit(e)
def setup():
"""setup temporary env for tests"""
global tmp
tmp = tempfile.mkdtemp()
patchers[:] = [
patch.dict(os.environ, {
'HOME': tmp,
# Let tests work with --user install when HOME is changed:
'PYTHONPATH': os.pathsep.join(sys.path),
}),
]
for p in patchers:
p.start()
# install IPython in the temp home:
install(user=True)
def test_install_env(tmp_path, env):
# python 3.5 // tmp_path must be converted to str
with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH',
[str(tmp_path)]):
install(env=env)
spec = tmp_path / 'kernels' / KERNEL_NAME / "kernel.json"
with spec.open() as f:
spec = json.load(f)
if env:
assert len(env) == len(spec['env'])
for k, v in env.items():
assert spec['env'][k] == v
else:
assert 'env' not in spec
def start(self):
self.log.warn("`jupyter kernelspec install-self` is DEPRECATED as of 4.0."
" You probably want `ipython kernel install` to install the IPython kernelspec.")
try:
from ipykernel import kernelspec
except ImportError:
print("ipykernel not available, can't install its spec.", file=sys.stderr)
self.exit(1)
try:
kernelspec.install(self.kernel_spec_manager, user=self.user)
except OSError as e:
if e.errno == errno.EACCES:
print(e, file=sys.stderr)
if not self.user:
print("Perhaps you want to install with `sudo` or `--user`?", file=sys.stderr)
self.exit(1)
self.exit(e)
def start(self):
try:
from ipykernel import kernelspec
except ImportError:
print("ipykernel not available, can't install its spec.",
file=sys.stderr)
self.exit(1)
try:
kernelspec.install(self.kernel_spec_manager, user=self.user)
except OSError as e:
if e.errno == errno.EACCES:
print(e, file=sys.stderr)
if not self.user:
print(
"Perhaps you want to install with `sudo` or `--user`?",
file=sys.stderr)
self.exit(1)
self.exit(e)
def setup():
"""setup temporary env for tests"""
global tmp
tmp = tempfile.mkdtemp()
patchers[:] = [
patch.dict(
os.environ,
{
'HOME': tmp,
# Let tests work with --user install when HOME is changed:
'PYTHONPATH': os.pathsep.join(sys.path),
}),
]
for p in patchers:
p.start()
# install IPython in the temp home:
install(user=True)
def create_kernel(kernel, env, lang, proc, img, cmd):
tmp_install = kernelspec.install(kernel_name=env, display_name=kernel, prefix='/tmp/')
kernel_template['language'] = lang
kernel_template['metadata']['process_proxy']['class_name'] = proc
kernel_template['metadata']['process_proxy']['config']['image_name'] = img
kernel_template['env']['KERNEL_USERNAME'] = environ.get('KERNEL_USERNAME')
kernel_template['env']['KERNEL_ENV'] = env
kernel_template['argv'][1] = cmd
tmp_kernel = path.join(tmp_install, 'kernel.json')
# write our modified kernel to our temp dir
with open(tmp_kernel, 'w') as k:
k.write(json.dumps(tmp_kernel))
install_out = subprocess.check_output(["/opt/conda/bin/jupyter", "kernelspec", "install", "--prefix=/opt/conda", tmp_install])
src = '/opt/conda/share/jupyter/kernels/python_tf_gpu_docker/scripts'
dst = '/opt/conda/share/jupyter/kernels/{env}/scripts'.format(env=env)
try:
shutil.copytree(src, dst)
except OSError as e:
print(e)
raise e
def install_ipykernel(self):
install(user=True)
print('done')
def __enter__(self):
ipyk.install(user=True,
kernel_name=self.kernel,
display_name=self.kernel)
return self
