def execute_notebook(source_notebook, destination_notebook, infra_args, papermill_args, notebooks_args): """Execute notebook with papermill. :param source_notebook: Source notebook file name :type source_notebook: str :param destination_notebook: Source notebook file name :type source_nodestination_notebooktebook: str :param infra_args: Infrastructure arguments :type infra_args: dict :param papermill_args: Papermill arguments :type papermill_args: dict :param notebooks_args: Notebook arguments :type notebooks_args: dict """ import papermill as pm # if kernel name is specified kernel_name = papermill_args.get("kernel_name") # if not specified try to get it from the notebook if not kernel_name: with open(source_notebook) as nbfile: notebook = json.loads(nbfile.read()) try: kernel_name = notebook.get("metadata").get("kernelspec").get("name") except: pass # create a kernel spec if not installed try: if kernel_name: from jupyter_client.kernelspec import KernelSpecManager if not KernelSpecManager().get_all_specs().get(kernel_name): # TODO: replace jupyter_client.kernelspec.KernelSpecManager logic from ipykernel.kernelspec import install install(kernel_name=kernel_name) papermill_args["kernel_name"] = kernel_name except: pass # execute notebook pm.execute_notebook( source_notebook, destination_notebook, parameters=notebooks_args, **papermill_args ) # TODO scrapbook if infra_args.get("history"): batchsize = 50 from azureml.core import Run results = pm.read_notebook(destination_notebook).dataframe.set_index("name")["value"] run = Run.get_context() for key, value in results.items(): if isinstance(value, list): for index in range(0, len(value), batchsize): run.log_list(key, value[index:index + batchsize]) else: run.log(key, value)
def render_ipynb(template_ipynb, rendered_ipynb, params=dict()): """Render the ipython notebook Args: template_ipynb: template ipython notebook where one cell defines the following metadata: {"tags": ["parameters"]} render_ipynb: output ipython notebook path params: parameters used to execute the ipython notebook """ import jupyter_client os.makedirs(os.path.dirname(rendered_ipynb), exist_ok=True) kernel_name = os.environ.get("CONDA_DEFAULT_ENV", 'python3') if kernel_name not in jupyter_client.kernelspec.find_kernel_specs(): logger.info( f"Installing the ipython kernel for the current conda environment: {kernel_name}" ) from ipykernel.kernelspec import install install(user=True, kernel_name=kernel_name) pm.execute_notebook( template_ipynb, # input template rendered_ipynb, kernel_name=kernel_name, # default kernel parameters=params) jupyter_nbconvert(rendered_ipynb)
def install_native_kernel_spec(self, user=False): """DEPRECATED: Use ipykernel.kenelspec.install""" warnings.warn( "install_native_kernel_spec is deprecated." " Use ipykernel.kernelspec import install.", stacklevel=2 ) from ipykernel.kernelspec import install install(self, user=user)
def install_native_kernel_spec(self, user=False): """DEPRECATED: Use ipykernel.kenelspec.install""" warnings.warn( "install_native_kernel_spec is deprecated." " Use ipykernel.kernelspec import install.", stacklevel=2) from ipykernel.kernelspec import install install(self, user=user)
def test_install(): system_jupyter_dir = tempfile.mkdtemp() with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH', [system_jupyter_dir]): install() assert_is_spec(os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME))
def test_install_user(): tmp = tempfile.mkdtemp() with mock.patch.dict(os.environ, {'HOME': tmp}): install(user=True) data_dir = jupyter_data_dir() assert_is_spec(os.path.join(data_dir, 'kernels', KERNEL_NAME))
def start(self): try: from ipykernel import kernelspec except ImportError: print("ipykernel not available, can't install its spec.", file=sys.stderr) self.exit(1) try: kernelspec.install(self.kernel_spec_manager, user=self.user) except OSError as e: self.exit(e)
def test_install_display_name_overrides_profile(): system_jupyter_dir = tempfile.mkdtemp() with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH', [system_jupyter_dir]): install(display_name="Display", profile="Test") spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json") with open(spec) as f: spec = json.load(f) assert spec["display_name"] == "Display"
def test_install_display_name_overrides_profile(): system_jupyter_dir = tempfile.mkdtemp() with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH', [system_jupyter_dir]): install(display_name="Display", profile="Test") spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json") with open(spec) as f: spec = json.load(f) nt.assert_equal(spec["display_name"], "Display")
def setup(): """setup temporary env for tests""" global tmp tmp = tempfile.mkdtemp() patchers[:] = [ patch.dict(os.environ, {'HOME': tmp}), ] for p in patchers: p.start() # install IPython in the temp home: install(user=True)
def init_kernel_specs(self): """Check that the IPython kernel is present, if available""" try: self.kernel_spec_manager.get_kernel_spec(NATIVE_KERNEL_NAME) except NoSuchKernel: try: from ipykernel.kernelspec import install except ImportError: self.log.warn("IPython kernel not available") else: self.log.warn("Installing IPython kernel spec") install(kernel_spec_manager=self.kernel_spec_manager, user=True)
def test_install_profile(): system_jupyter_dir = tempfile.mkdtemp() with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH', [system_jupyter_dir]): install(profile="Test") spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json") with open(spec) as f: spec = json.load(f) assert spec["display_name"].endswith(" [profile=Test]") nt.assert_equal(spec["argv"][-2:], ["--profile", "Test"])
def test_install_profile(): system_jupyter_dir = tempfile.mkdtemp() with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH', [system_jupyter_dir]): install(profile="Test") spec = os.path.join(system_jupyter_dir, 'kernels', KERNEL_NAME, "kernel.json") with open(spec) as f: spec = json.load(f) nt.assert_true(spec["display_name"].endswith(" [profile=Test]")) nt.assert_equal(spec["argv"][-2:], ["--profile", "Test"])
def start(self): try: from ipykernel import kernelspec except ImportError: print("ipykernel not available, can't install its spec.", file=sys.stderr) self.exit(1) try: kernelspec.install(self.kernel_spec_manager, user=self.user) except OSError as e: if e.errno == errno.EACCES: print(e, file=sys.stderr) if not self.user: print("Perhaps you want to install with `sudo` or `--user`?", file=sys.stderr) self.exit(1) self.exit(e)
def setup(): """setup temporary env for tests""" global tmp tmp = tempfile.mkdtemp() patchers[:] = [ patch.dict(os.environ, { 'HOME': tmp, # Let tests work with --user install when HOME is changed: 'PYTHONPATH': os.pathsep.join(sys.path), }), ] for p in patchers: p.start() # install IPython in the temp home: install(user=True)
def test_install_env(tmp_path, env): # python 3.5 // tmp_path must be converted to str with mock.patch('jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH', [str(tmp_path)]): install(env=env) spec = tmp_path / 'kernels' / KERNEL_NAME / "kernel.json" with spec.open() as f: spec = json.load(f) if env: assert len(env) == len(spec['env']) for k, v in env.items(): assert spec['env'][k] == v else: assert 'env' not in spec
def start(self): self.log.warn("`jupyter kernelspec install-self` is DEPRECATED as of 4.0." " You probably want `ipython kernel install` to install the IPython kernelspec.") try: from ipykernel import kernelspec except ImportError: print("ipykernel not available, can't install its spec.", file=sys.stderr) self.exit(1) try: kernelspec.install(self.kernel_spec_manager, user=self.user) except OSError as e: if e.errno == errno.EACCES: print(e, file=sys.stderr) if not self.user: print("Perhaps you want to install with `sudo` or `--user`?", file=sys.stderr) self.exit(1) self.exit(e)
def start(self): try: from ipykernel import kernelspec except ImportError: print("ipykernel not available, can't install its spec.", file=sys.stderr) self.exit(1) try: kernelspec.install(self.kernel_spec_manager, user=self.user) except OSError as e: if e.errno == errno.EACCES: print(e, file=sys.stderr) if not self.user: print( "Perhaps you want to install with `sudo` or `--user`?", file=sys.stderr) self.exit(1) self.exit(e)
def setup(): """setup temporary env for tests""" global tmp tmp = tempfile.mkdtemp() patchers[:] = [ patch.dict( os.environ, { 'HOME': tmp, # Let tests work with --user install when HOME is changed: 'PYTHONPATH': os.pathsep.join(sys.path), }), ] for p in patchers: p.start() # install IPython in the temp home: install(user=True)
def create_kernel(kernel, env, lang, proc, img, cmd): tmp_install = kernelspec.install(kernel_name=env, display_name=kernel, prefix='/tmp/') kernel_template['language'] = lang kernel_template['metadata']['process_proxy']['class_name'] = proc kernel_template['metadata']['process_proxy']['config']['image_name'] = img kernel_template['env']['KERNEL_USERNAME'] = environ.get('KERNEL_USERNAME') kernel_template['env']['KERNEL_ENV'] = env kernel_template['argv'][1] = cmd tmp_kernel = path.join(tmp_install, 'kernel.json') # write our modified kernel to our temp dir with open(tmp_kernel, 'w') as k: k.write(json.dumps(tmp_kernel)) install_out = subprocess.check_output(["/opt/conda/bin/jupyter", "kernelspec", "install", "--prefix=/opt/conda", tmp_install]) src = '/opt/conda/share/jupyter/kernels/python_tf_gpu_docker/scripts' dst = '/opt/conda/share/jupyter/kernels/{env}/scripts'.format(env=env) try: shutil.copytree(src, dst) except OSError as e: print(e) raise e
def install_ipykernel(self): install(user=True) print('done')
def __enter__(self): ipyk.install(user=True, kernel_name=self.kernel, display_name=self.kernel) return self