def default_config(self): c = Config({ "NbConvertBase": { "display_data_priority": [ "application/vnd.jupyter.widget-view+json", "application/javascript", "text/html", "text/markdown", "image/svg+xml", "text/latex", "image/png", "image/jpeg", "text/plain", ] }, "HighlightMagicsPreprocessor": { "enabled": True }, }) c.merge(super().default_config) return c
def interact(session=False, apk=None): """ Start an interactive shell :param session: :param apk: :return: """ if session: CONF["SESSION"] = Session(export_ipython=True) if apk: print("Loading apk {}...".format(os.path.basename(apk))) print("Please be patient, this might take a while.") # TODO we can export fancy aliases for those as well... a, d, dx = AnalyzeAPK(apk) cfg = Config() ipshell = InteractiveShellEmbed( config=cfg, banner1="{} started".format(_version_string)) init_print_colors() ipshell()
def test_source_with_drs4_pedestal(): from ctapipe_io_lst import LSTEventSource config = Config({ 'LSTEventSource': { 'pointing_information': False, 'LSTR0Corrections': { 'drs4_pedestal_path': test_drs4_pedestal_path, }, }, }) source = LSTEventSource( input_url=test_r0_path, config=config, ) assert source.r0_r1_calibrator.drs4_pedestal_path.tel[1] == test_drs4_pedestal_path.absolute() with source: for event in source: assert event.r1.tel[1].waveform is not None
def test_already_gain_selected(): from ctapipe_io_lst import LSTEventSource config = Config({ 'LSTEventSource': { 'pointing_information': False, 'LSTR0Corrections': { 'drs4_pedestal_path': test_drs4_pedestal_path, 'drs4_time_calibration_path': test_time_calib_path, 'calibration_path': test_calib_path, }, }, }) source = LSTEventSource(test_r0_gainselected_path, config=config) reference_source = LSTEventSource(test_r0_path, config=config) with source, reference_source: for event, reference_event in zip(source, reference_source): assert np.all(event.r1.tel[1].waveform == reference_event.r1.tel[1].waveform) assert event.count == 199
def render_track(track, track_cfg): meta = utils.get_track_meta(track, track_cfg) cfg = Config() cfg.Exporter.preprocessors = [ 'lesson_preprocessor.LearnLessonPreprocessor' ] exporter = NotebookExporter(config=cfg) resources = {'track_meta': meta, 'track_cfg': track_cfg} outdir = os.path.join(track, track_cfg['tag'], 'rendered') os.makedirs(outdir, exist_ok=True) for nb_meta in meta.notebooks: in_path = os.path.join(track, 'raw', nb_meta.filename) resources['lesson'] = nb_meta.lesson resources['nb_meta'] = nb_meta if CLEAN: clean(in_path) nb, _ = exporter.from_filename(in_path, resources) out_path = os.path.join(outdir, nb_meta.filename) with open(out_path, 'w') as f: f.write(nb)
def test_exclude_markdown(self): no_md = { "TemplateExporter": { "exclude_output": False, "exclude_input": False, "exclude_input_prompt": False, "exclude_output_prompt": False, "exclude_markdown": True, "exclude_code_cell": False, } } c_no_md = Config(no_md) exporter_no_md = TemplateExporter(config=c_no_md) exporter_no_md.template_file = 'python' nb_no_md, resources_no_md = exporter_no_md.from_filename( self._get_notebook()) assert not resources_no_md['global_content_filter']['include_markdown'] assert "First import NumPy and Matplotlib" not in nb_no_md
def on_config(self, config): c = Config() if self.config["execute"]: if self.config["preamble"]: default_preprocessors = MarkdownExporter.default_preprocessors.default_args[ 0] default_preprocessors.insert( default_preprocessors.index( "nbconvert.preprocessors.ExecutePreprocessor"), "nbconvert_utils.ExecuteWithPreamble", ) c.default_preprocessors = default_preprocessors c.ExecutePreprocessor.timeout = self.config["timeout"] c.ExecuteWithPreamble.enabled = True c.ExecuteWithPreamble.preamble_scripts = [ self.config["preamble"] ] else: c.Executor.enabled = True config["notebook_exporter"] = MarkdownExporter(config=c) return config
def test_exclude_output_stdin(self): no_output_stdin = { "TemplateExporter": { "exclude_output": False, "exclude_input": False, "exclude_input_prompt": False, "exclude_output_prompt": True, "exclude_markdown": False, "exclude_code_cell": False, "exclude_output_stdin": True, } } c_no_output_stdin = Config(no_output_stdin) exporter_no_output_prompt = HTMLExporter(config=c_no_output_stdin) nb_no_output_stdin, resources_no_output_stdin = exporter_no_output_prompt.from_filename( self._get_notebook("notebook3.ipynb")) assert not resources_no_output_stdin["global_content_filter"][ "include_output_stdin"] assert "test input: input value" not in nb_no_output_stdin
def test_get_pvc_manifest(): c = Config() c.KubeSpawner.pvc_name_template = "user-{username}" c.KubeSpawner.storage_extra_labels = {"user": "******"} c.KubeSpawner.storage_selector = {"matchLabels": {"user": "******"}} spawner = KubeSpawner(config=c, _mock=True) manifest = spawner.get_pvc_manifest() assert isinstance(manifest, V1PersistentVolumeClaim) assert manifest.metadata.name == "user-mock-5fname" assert manifest.metadata.labels == { "user": "******", "hub.jupyter.org/username": "******", "app": "jupyterhub", "component": "singleuser-storage", "heritage": "jupyterhub", } assert manifest.spec.selector == {"matchLabels": {"user": "******"}}
def default_config(self): c = Config({ 'NbConvertBase': { 'display_data_priority': [ 'application/vnd.jupyter.widget-view+json', 'application/javascript', 'text/html', 'text/markdown', 'image/svg+xml', 'text/latex', 'image/png', 'image/jpeg', 'text/plain' ] }, 'HighlightMagicsPreprocessor': { 'enabled': True } }) c.merge(super().default_config) return c
def test_deprecated_config(): """Deprecated config is handled correctly""" with pytest.warns(DeprecationWarning): c = Config() # both set, non-deprecated wins c.KubeSpawner.singleuser_fs_gid = 5 c.KubeSpawner.fs_gid = 10 # only deprecated set, should still work c.KubeSpawner.hub_connect_ip = '10.0.1.1' c.KubeSpawner.singleuser_extra_pod_config = extra_pod_config = {"key": "value"} c.KubeSpawner.image_spec = 'abc:123' c.KubeSpawner.image_pull_secrets = 'k8s-secret-a' spawner = KubeSpawner(hub=Hub(), config=c, _mock=True) assert spawner.hub.connect_ip == '10.0.1.1' assert spawner.fs_gid == 10 assert spawner.extra_pod_config == extra_pod_config # deprecated access gets the right values, too assert spawner.singleuser_fs_gid == spawner.fs_gid assert spawner.singleuser_extra_pod_config == spawner.extra_pod_config assert spawner.image == 'abc:123' assert spawner.image_pull_secrets[0]["name"] == 'k8s-secret-a'
def convert_ipynb_to_html(notebook_file, html_file): """ Convert the given `notebook_file` to HTML and write it to `html_file`. """ # set a high timeout for datasets with a large number of features report_config = Config({ 'ExecutePreprocessor': { 'enabled': True, 'timeout': 600 }, 'HTMLExporter': { 'template_path': [template_path], 'template_file': 'report.tpl' } }) exportHtml = HTMLExporter(config=report_config) output, resources = exportHtml.from_filename(notebook_file) open(html_file, mode='w', encoding='utf-8').write(output)
def init_exporter(extract_images, execute, **exporter_config): """ Returns an initialized exporter. """ config = Config(InteractExporter=exporter_config) preprocessors = [] if extract_images: # Use ExtractOutputPreprocessor to extract the images to separate files preprocessors.append( 'nbconvert.preprocessors.ExtractOutputPreprocessor' ) if execute: # Use the NbiExecutePreprocessor to correctly generate widget output # for interact() calls. preprocessors.append('nbinteract.preprocessors.NbiExecutePreprocessor') config.InteractExporter.preprocessors = preprocessors exporter = InteractExporter(config=config) return exporter
def test_include_output_stdin(self): output_stdin = { "TemplateExporter": { "exclude_output": False, "exclude_input": False, "exclude_input_prompt": False, "exclude_output_prompt": True, "exclude_markdown": False, "exclude_code_cell": False, "exclude_output_stdin": False, } } c_output_stdin = Config(output_stdin) exporter_output_stdin = HTMLExporter(config=c_output_stdin) nb_output_stdin, resources_output_stdin = exporter_output_stdin.from_filename( self._get_notebook('notebook3.ipynb')) assert resources_output_stdin['global_content_filter'][ 'include_output_stdin'] assert "test input: input value" in nb_output_stdin
def default_config(self): c = super().default_config c.merge( Config({ 'ExecutePreprocessor': { 'enabled': True }, 'TagRemovePreprocessor': { 'enabled': True, 'remove_all_outputs_tags': {'silent'}, 'remove_cell_tags': {'exclude', 'run_only'}, 'remove_input_tags': {'hidden', 'output_only'} }, 'RemoveMagicPreprocessor': { 'enabled': True }, 'HighlightMagicsPreprocessor': { 'enabled': False } })) return c
def execute_notebook(notebook, execute=True, to_html=False, kernel_name='venv'): # Get start time start_time = time.time() # Initialize preprocess ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) # Execute if execute: ep.preprocess(notebook, {'metadata': {'path': 'app/static/library'}}) # Convert to HTML if to_html: c = Config() c.HTMLExporter.preprocessors = ['nbconvert.preprocessors.ExtractOutputPreprocessor'] html_exporter_with_figs = HTMLExporter(config=c) notebook = html_exporter_with_figs.from_notebook_node(notebook)[0] # Return return notebook, round(time.time() - start_time)
def test_jupyterhub_base_config(setup_jupyterhub_db, setup_jupyterhub_config_base): """ Ensure all environment variables for the base config are set. """ c = Config() c.JupyterHub.base_url = os.environ.get("JUPYTERHUB_BASE_URL") # JupyterHub postgres settings c.JupyterHub.db_url = "postgresql://{user}:{password}@{host}:{port}/{db}".format( user=os.environ.get("POSTGRES_JUPYTERHUB_USER"), password=os.environ.get("POSTGRES_JUPYTERHUB_PASSWORD"), host=os.environ.get("POSTGRES_JUPYTERHUB_HOST"), port=os.environ.get("POSTGRES_JUPYTERHUB_PORT"), db=os.environ.get("POSTGRES_JUPYTERHUB_DB"), ) c.JupyterHub.shutdown_on_logout = bool( strtobool(os.environ.get("JUPYTERHUB_SHUTDOWN_ON_LOGOUT"))) c.Authenticator.admin_users = {os.environ.get("JUPYTERHUB_ADMIN_USER")} c.Spawner.image = os.environ.get("DOCKER_END_USER_IMAGE") c.Spawner.cpu_limit = float(os.environ.get("SPAWNER_CPU_LIMIT")) c.Spawner.mem_limit = os.environ.get("SPAWNER_MEM_LIMIT") c.KubeSpawner.network_name = os.environ.get("DOCKER_NETWORK_NAME") docker_spawn_command = os.environ.get("DOCKER_SPAWN_CMD") exchange_dir = os.environ.get("EXCHANGE_DIR") notebook_dir = os.environ.get("DOCKER_NOTEBOOK_DIR") assert c.Authenticator.admin_users == {"admin0"} assert c.KubeSpawner.network_name == "test-network" assert (c.JupyterHub.db_url == "postgresql://*****:*****@jupyterhub-db:5432/jupyterhub") assert c.JupyterHub.shutdown_on_logout == True # noqa: E712 assert c.Spawner.cpu_limit == 0.5 assert c.Spawner.mem_limit == "2G" assert docker_spawn_command == "single_user_test.sh" assert exchange_dir == "/path/to/exchange" assert notebook_dir == "/home/saturn"
def convert_ipynb_to_html(notebook_file, html_file): """ Convert given Jupyter notebook (``.ipynb``) to HTML file. Parameters ---------- notebook_file : str Path to input Jupyter notebook file. html_file : str Path to output HTML file. Note ---- This function is also exposed as the :ref:`render_notebook <render_notebook>` command-line utility. """ # `nbconvert` uses `asyncio` which uses an entirely default # implemention of the event loop on Windows for Cpython 3.8 # which breaks the report generation unless we include the # following workaround if (sys.version_info[0] == 3 and sys.version_info[1] >= 8 and sys.platform.startswith('win')): asyncio.set_event_loop_policy( asyncio.WindowsSelectorEventLoopPolicy()) # set a high timeout for datasets with a large number of features report_config = Config({ 'ExecutePreprocessor': { 'enabled': True, 'timeout': 3600 }, 'HTMLExporter': { "template_name": "classic", "template_file": join(template_path, 'report.tpl') } }) exportHtml = HTMLExporter(config=report_config) output, _ = exportHtml.from_filename(notebook_file) open(html_file, mode='w', encoding='utf-8').write(output)
def get_html_from_filepath(filepath): """Convert ipython notebook to html Return: html content of the converted notebook """ config = Config({ 'CSSHTMLHeaderTransformer': { 'enabled': True, 'highlight_class': '.highlight-ipynb' } }) # Remove prompts from rendered notebook (requires nbconvert>=5.1.0) from jinja2 import DictLoader loader = DictLoader({ 'noprompt': """ {%- extends 'basic.tpl' -%} {% block in_prompt -%} {%- endblock in_prompt %} {% block output_area_prompt %} {%- endblock output_area_prompt %} """ }) exporter = HTMLExporter(config=config, template_file='noprompt', extra_loaders=[loader], filters={'highlight2html': custom_highlighter}) content, info = exporter.from_filename(filepath) if BeautifulSoup: soup = BeautifulSoup(content, 'html.parser') for i in soup.findAll('div', {'class': 'input'}): if i.findChildren()[1].find(text='#ignore') is not None: i.extract() content = soup.decode(formatter=None) return content, info
def convert(filename, options): """ Take as input a filename for a Jupyter Notebook (and a variety of options) and write an HTML file that is a representation of that notebook. """ # Open the Jupyter notebook (basename, ext) = os.path.splitext(filename) response = open(filename,"r").read() if options.verbose: print('Reading "%s"' % filename) notebook = nbformat.reads(response, as_version=4) # Configure the HTMLExporter to use the preprocessors cfg = Config() cfg.ExecutePreprocessor.enabled = True cfg.ExecutePreprocessor.kernel_name = options.kernel cfg.ExecutePreprocessor.timeout = options.timeout cfg.VerboseExecutePreprocessor.enabled = True cfg.VerboseExecutePreprocessor.verbose = options.verbose cfg.AddCitationsPreprocessor.enabled = True cfg.AddCitationsPreprocessor.verbose = options.verbose cfg.AddCitationsPreprocessor.csl = options.csl cfg.AddCitationsPreprocessor.csl_path = options.csl_path cfg.AddCitationsPreprocessor.bibliography = options.bib cfg.AddCitationsPreprocessor.header = options.header cfg.HTMLExporter.preprocessors = [VerboseExecutePreprocessor(config=cfg), AddCitationsPreprocessor(config=cfg)] # Convert the notebook to HTML html_exporter = HTMLExporter(config=cfg) if options.verbose: print('Converting "%s" to HTML' % filename) (body, resources) = html_exporter.from_notebook_node(notebook) # Output writer = FilesWriter() if options.verbose: print('Writing "%s.html"' % basename) writer.write(body, resources, notebook_name=basename)
async def test_jupyterhub_auth_user(monkeypatch): from jupyterhub.tests.utils import add_user jhub_api_token = uuid.uuid4().hex jhub_bind_url = "http://127.0.0.1:%i/@/space%%20word/" % random_port() hub_config = Config() hub_config.JupyterHub.services = [{ "name": "dask-gateway", "api_token": jhub_api_token }] hub_config.JupyterHub.bind_url = jhub_bind_url class MockHub(hub_mocking.MockHub): def init_logging(self): pass hub = MockHub(config=hub_config) # Configure gateway config = configure_dask_gateway(jhub_api_token, jhub_bind_url) async with temp_gateway(config=config) as g: async with temp_hub(hub): # Create a new jupyterhub user alice, and get the api token u = add_user(hub.db, name="alice") api_token = u.new_api_token() hub.db.commit() # Configure auth with incorrect api token auth = JupyterHubAuth(api_token=uuid.uuid4().hex) async with g.gateway_client(auth=auth) as gateway: # Auth fails with bad token with pytest.raises(Exception): await gateway.list_clusters() # Auth works with correct token auth.api_token = api_token await gateway.list_clusters()
def export_notebook(nb, format, config, template): """ exports a notebook in a particular format Parameters ---------- nb: nbformat.notebooknode.NotebookNode format: str the nbconvert exporter class prefix ('Latex', 'HTML',...) config : dict configuration for the nbconvert exporter template : str the Jinja template for the conversion Returns ------- export: tuple (body, resources) extension: str the file extension of the exported format (e.g. .tex) """ jinja_template = DictLoader({'my_template': template}) c = Config() for key, val in config.items(): # TODO should probably not use exec, need to think of another way exec('c.{0} = val'.format(key)) in globals(), locals() if not hasattr(nbconvert, format + 'Exporter'): logging.error('the export format is not recognised: {}'.format(format)) raise ValueError( 'the export format is not recognised: {}'.format(format)) class MyExporter(getattr(nbconvert, format + 'Exporter')): """override the default template""" template_file = 'my_template' logging.info('running nbconvert') exporter = MyExporter(config=c, extra_loaders=[jinja_template]) return exporter.from_notebook_node(nb), exporter.file_extension
async def test_basic_auth_password(tmpdir): config = Config() config.DaskGateway.temp_dir = str(tmpdir) config.DaskGateway.authenticator_class = ( "dask_gateway_server.auth.DummyAuthenticator") config.DummyAuthenticator.password = "******" async with temp_gateway(config=config) as gateway_proc: auth = BasicAuth() async with Gateway( address=gateway_proc.public_urls.connect_url, proxy_address=gateway_proc.gateway_urls.connect_url, asynchronous=True, auth=auth, ) as gateway: with pytest.raises(Exception): await gateway.list_clusters() auth.password = "******" await gateway.list_clusters()
async def test_jupyterhub_auth_service(monkeypatch): jhub_api_token = uuid.uuid4().hex jhub_service_token = uuid.uuid4().hex jhub_bind_url = "http://127.0.0.1:%i/@/space%%20word/" % random_port() hub_config = Config() hub_config.JupyterHub.services = [ { "name": "dask-gateway", "api_token": jhub_api_token }, { "name": "any-service", "api_token": jhub_service_token }, ] hub_config.JupyterHub.bind_url = jhub_bind_url class MockHub(hub_mocking.MockHub): def init_logging(self): pass hub = MockHub(config=hub_config) # Configure gateway config = configure_dask_gateway(jhub_api_token, jhub_bind_url) async with temp_gateway(config=config) as g: async with temp_hub(hub): # Configure auth with incorrect api token auth = JupyterHubAuth(api_token=uuid.uuid4().hex) async with g.gateway_client(auth=auth) as gateway: # Auth fails with bad token with pytest.raises(Exception): await gateway.list_clusters() # Auth works with service token auth.api_token = jhub_api_token await gateway.list_clusters()
async def test_upgrade(tmpdir, hub_version): db_url = os.getenv('JUPYTERHUB_TEST_DB_URL') if db_url: db_url += '_upgrade_' + hub_version.replace('.', '') else: db_url = 'sqlite:///jupyterhub.sqlite' tmpdir.chdir() # use persistent temp env directory # to reuse across multiple runs env_dir = os.path.join(tempfile.gettempdir(), 'test-hub-upgrade-%s' % hub_version) generate_old_db(env_dir, hub_version, db_url) cfg = Config() cfg.JupyterHub.db_url = db_url tokenapp = NewToken(config=cfg) tokenapp.initialize(['admin']) with raises(SystemExit): tokenapp.start() if 'sqlite' in db_url: fname = db_url.split(':///')[1] sqlite_files = glob(os.path.join(str(tmpdir), 'jupyterhub.sqlite*')) assert len(sqlite_files) == 1 upgradeapp = UpgradeDB(config=cfg) upgradeapp.initialize([]) upgradeapp.start() # check that backup was created: if 'sqlite' in db_url: sqlite_files = glob(os.path.join(str(tmpdir), 'jupyterhub.sqlite*')) assert len(sqlite_files) == 2 # run tokenapp again, it should work tokenapp.start()
def EnterPython(self, args=None): if os.sep == '\\': eof = 'Ctrl-Z plus Return' else: eof = 'Ctrl-D (i.e. EOF)' hdtv.ui.msg("Python {}. Return to hdtv with {}.".format( platform.python_version(), eof), end='') #self.fPyMode = True from traitlets.config import Config c = Config() c.InteractiveShell.confirm_exit = False c.TerminalInteractiveShell.simple_prompt = False c.TerminalInteractiveShell.colors = 'LightBG' c.TerminalInteractiveShell.autoindent = True c.TerminalIPythonApp.display_banner = False c.InteractiveShell.ipython_dir = "/tmp" from IPython import start_ipython start_ipython([], config=c, user_ns=self.cmds)
def clean(): _delete_generated_files() config = Config() config.NotebookExporter.preprocessors = [ClearOutputPreprocessor()] exporter = NotebookExporter(config=config) for notebook in get_files("ipynb"): with open(notebook, encoding="utf8") as nb_file: nb = nbformat.read(nb_file, as_version=4) if not nb.cells[-1]["source"]: nb.cells.pop() format_code_cells(nb) for cell_id, cell in enumerate(nb.cells): cell["id"] = f"{notebook.stem}-{cell_id}" ipynb, _ = exporter.from_notebook_node(nb) with open(f"{notebook.stem}.ipynb", "w", encoding="utf8") as writable: writable.write(ipynb)
def generate_feedback(self, assignment_id, student_id=None): """Run ``nbgrader feedback`` for a particular assignment and student. Arguments --------- assignment_id: string The name of the assignment student_id: string The name of the student (optional). If not provided, then generate feedback from autograded submissions. Returns ------- result: dict A dictionary with the following keys (error and log may or may not be present): - success (bool): whether or not the operation completed successfully - error (string): formatted traceback - log (string): captured log output """ # Because we may be using HTMLExporter.template_file in other # parts of the the UI, we need to make sure that the template # is explicitply 'feedback.tpl` here: c = Config() c.HTMLExporter.template_file = 'feedback.tpl' if student_id is not None: with temp_attrs(self.coursedir, assignment_id=assignment_id, student_id=student_id): app = Feedback(coursedir=self.coursedir, parent=self) app.update_config(c) return capture_log(app) else: with temp_attrs(self.coursedir, assignment_id=assignment_id): app = Feedback(coursedir=self.coursedir, parent=self) app.update_config(c) return capture_log(app)
def launch_ipython(argv=None, ipython_app=None): """ Launch IPython from this interpreter with custom args if needed. Chimera magic commands are also enabled automatically. """ try: if ipython_app is None: from IPython.terminal.ipapp import TerminalIPythonApp as ipython_app from traitlets.config import Config except ImportError: sys.exit("ERROR: IPython not installed in this environment. " "Try with `conda install ipython`") else: # launch_new_instance(argv) app = ipython_app() c = Config() code = ["from pychimera import enable_chimera_inline", "enable_chimera_inline()"] c.InteractiveShellApp.exec_lines = code app.update_config(c) app.initialize(argv) app.start()
def nb2md(nb_path): """Convert a notebook and return markdown We use a template that removed all code cells because if the body is to big (javascript and stuff) it takes to long to read and parse """ config = Config() template = os.path.join(THIS_DIR, "assets", "md-no-codecell.tpl") template_file = os.path.basename(template) extra_loaders = [] extra_loaders.append(jinja2.FileSystemLoader([os.path.dirname(template)])) exporter = MarkdownExporter( config=config, template_file=template_file, extra_loaders=extra_loaders, # filters=filters, # preprocessors=preprocessors_, ) body, resources = exporter.from_filename(nb_path) return body