def blast_it(func, *args, **kwargs): try: initialise_base_dirs() result = func(*args, **kwargs) return result finally: _cleanup_dirs()
def docker_compose_entrypoint(): """ Sadness. This is required because of https://github.com/jupyter/jupyter_client/issues/154 Otherwise we will get "RuntimeError: Kernel died before replying to kernel_info" The suggested fix to use sh -c "command" does not work for our use-case, sadly. Examples -------- notebooker_execute --report-name watchdog_checks --mongo-host mktdatad Received a request to run a report with the following parameters: ['/users/is/jbannister/pyenvs/notebooker/bin/python', '-m', 'notebooker.execute_notebook', '--report-name', 'watchdog_checks', '--mongo-host', 'mktdatad'] ... notebooker_execute Received a request to run a report with the following parameters: ['/users/is/jbannister/pyenvs/notebooker/bin/python', '-m', 'notebooker.execute_notebook'] ValueError: Error! Please provide a --report-name. """ args_to_execute = [sys.executable, "-m", __name__] + sys.argv[1:] logger.info( "Received a request to run a report with the following parameters:") logger.info(args_to_execute) try: subprocess.Popen(args_to_execute).wait() finally: _cleanup_dirs()
def sanity_check(template_dir): logger.info("Starting sanity check") os.environ["PY_TEMPLATE_DIR"] = template_dir try: for template_name in templates._all_templates(): logger.info( "========================[ Sanity checking {} ]========================" .format(template_name)) # Test conversion to ipynb - this will throw if stuff goes wrong generate_ipynb_from_py(filesystem.get_template_dir(), template_name, warn_on_local=False) # Test that each template has parameters as expected nb = templates.template_name_to_notebook_node(template_name, warn_on_local=False) param_idx = templates._get_parameters_cell_idx(nb) if param_idx is None: logger.warning( 'Template {} does not have a "parameters"-tagged cell.'. format(template_name)) # Test that we can generate a preview from the template preview = templates._get_preview(template_name, warn_on_local=False) # Previews in HTML are gigantic since they include all jupyter css and js. assert len( preview ) > 1000, "Preview was not properly generated for {}".format( template_name) logger.info( "========================[ {} PASSED ]========================" .format(template_name)) finally: filesystem._cleanup_dirs()
def setup_test(template_dir): try: with TemporaryDirectory() as tmpdir: app = create_app() web_config = WebappConfig() web_config.PY_TEMPLATE_BASE_DIR = template_dir web_config.CACHE_DIR = tmpdir app = setup_app(app, web_config) with app.app_context(): yield finally: filesystem._cleanup_dirs(web_config)
def _cleanup_on_exit(): global all_report_refresher if "pytest" in sys.modules or not all_report_refresher: return os.environ["NOTEBOOKER_APP_STOPPING"] = "1" _cancel_all_jobs() _cleanup_dirs() if all_report_refresher: # Wait until it terminates. logger.info('Stopping "report hunter" thread.') all_report_refresher.join() # Allow all clients looking for task results to get the bad news... time.sleep(2)
def test_execution_of_templates(template_name): try: _run_checks( "job_id_{}".format(str(uuid.uuid4())[:6]), datetime.datetime.now(), template_name, template_name, get_output_dir(), get_template_dir(), {}, generate_pdf_output=False, ) finally: _cleanup_dirs()
def test_generate_ipynb_from_py(): python_dir = tempfile.mkdtemp() try: set_cache("latest_sha", "fake_sha_early") os.mkdir(python_dir + "/extra_path") with open(os.path.join(python_dir, "extra_path", "test_report.py"), "w") as f: f.write("#hello world\n") report_path = os.sep.join(["extra_path", "test_report"]) with mock.patch("notebooker.utils.conversion._git_pull_templates") as pull: conversion.python_template_dir = lambda *a, **kw: python_dir pull.return_value = "fake_sha_early" conversion.generate_ipynb_from_py(python_dir, report_path) pull.return_value = "fake_sha_later" conversion.generate_ipynb_from_py(python_dir, report_path) conversion.generate_ipynb_from_py(python_dir, report_path) assert get_cache("latest_sha") == "fake_sha_later" expected_filename = _output_ipynb_name(report_path) expected_ipynb_path = os.path.join(python_dir, "fake_sha_early", expected_filename) assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!" expected_ipynb_path = os.path.join(python_dir, "fake_sha_later", expected_filename) assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!" with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4: with mock.patch("notebooker.utils.conversion.pkg_resources.resource_filename") as resource_filename: conversion.python_template_dir = lambda *a, **kw: None uuid4.return_value = "uuid" resource_filename.return_value = python_dir + "/extra_path/test_report.py" conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report") expected_ipynb_path = os.path.join(python_dir, "uuid", expected_filename) assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!" with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4: conversion.python_template_dir = lambda *a, **kw: python_dir conversion.NOTEBOOKER_DISABLE_GIT = True uuid4.return_value = "uuid_nogit" conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report") expected_ipynb_path = os.path.join(python_dir, "uuid_nogit", expected_filename) assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!" finally: _cleanup_dirs() shutil.rmtree(python_dir)
def regression_test(template_dir): logger.info("Starting regression test") os.environ["PY_TEMPLATE_DIR"] = template_dir try: attempted_templates, failed_templates = [], set() for template_name in templates._all_templates(): logger.info( "============================[ Testing {} ]============================" .format(template_name)) try: attempted_templates.append(template_name) _run_checks( "job_id_{}".format(str(uuid.uuid4())[:6]), datetime.datetime.now(), template_name, template_name, filesystem.get_output_dir(), filesystem.get_template_dir(), {}, generate_pdf_output=False, ) logger.info( "===============================[ SUCCESS ]==============================" ) except Exception: failed_templates.add(template_name) logger.info( "===============================[ FAILED ]===============================" ) logger.exception( "Failed to execute template {}".format(template_name)) for template in attempted_templates: logger.info("{}: {}".format( template, "FAILED" if template in failed_templates else "PASSED")) if len(failed_templates) > 0: raise NotebookRunException( "The following templates failed to execute with no parameters:\n{}" .format("\n".join(failed_templates))) finally: filesystem._cleanup_dirs()
def setup_and_cleanup_notebooker_filesystem(webapp_config): try: initialise_base_dirs(webapp_config=webapp_config) yield finally: _cleanup_dirs(webapp_config)