def blast_it(func, *args, **kwargs):
try:
initialise_base_dirs()
result = func(*args, **kwargs)
return result
finally:
_cleanup_dirs()
def docker_compose_entrypoint():
"""
Sadness. This is required because of https://github.com/jupyter/jupyter_client/issues/154
Otherwise we will get "RuntimeError: Kernel died before replying to kernel_info"
The suggested fix to use sh -c "command" does not work for our use-case, sadly.
Examples
--------
notebooker_execute --report-name watchdog_checks --mongo-host mktdatad
Received a request to run a report with the following parameters:
['/users/is/jbannister/pyenvs/notebooker/bin/python', '-m', 'notebooker.execute_notebook', '--report-name', 'watchdog_checks', '--mongo-host', 'mktdatad']
...
notebooker_execute
Received a request to run a report with the following parameters:
['/users/is/jbannister/pyenvs/notebooker/bin/python', '-m', 'notebooker.execute_notebook']
ValueError: Error! Please provide a --report-name.
"""
args_to_execute = [sys.executable, "-m", __name__] + sys.argv[1:]
logger.info(
"Received a request to run a report with the following parameters:")
logger.info(args_to_execute)
try:
subprocess.Popen(args_to_execute).wait()
finally:
_cleanup_dirs()
def sanity_check(template_dir):
logger.info("Starting sanity check")
os.environ["PY_TEMPLATE_DIR"] = template_dir
try:
for template_name in templates._all_templates():
logger.info(
"========================[ Sanity checking {} ]========================"
.format(template_name))
# Test conversion to ipynb - this will throw if stuff goes wrong
generate_ipynb_from_py(filesystem.get_template_dir(),
template_name,
warn_on_local=False)
# Test that each template has parameters as expected
nb = templates.template_name_to_notebook_node(template_name,
warn_on_local=False)
param_idx = templates._get_parameters_cell_idx(nb)
if param_idx is None:
logger.warning(
'Template {} does not have a "parameters"-tagged cell.'.
format(template_name))
# Test that we can generate a preview from the template
preview = templates._get_preview(template_name,
warn_on_local=False)
# Previews in HTML are gigantic since they include all jupyter css and js.
assert len(
preview
) > 1000, "Preview was not properly generated for {}".format(
template_name)
logger.info(
"========================[ {} PASSED ]========================"
.format(template_name))
finally:
filesystem._cleanup_dirs()
def setup_test(template_dir):
try:
with TemporaryDirectory() as tmpdir:
app = create_app()
web_config = WebappConfig()
web_config.PY_TEMPLATE_BASE_DIR = template_dir
web_config.CACHE_DIR = tmpdir
app = setup_app(app, web_config)
with app.app_context():
yield
finally:
filesystem._cleanup_dirs(web_config)
def _cleanup_on_exit():
global all_report_refresher
if "pytest" in sys.modules or not all_report_refresher:
return
os.environ["NOTEBOOKER_APP_STOPPING"] = "1"
_cancel_all_jobs()
_cleanup_dirs()
if all_report_refresher:
# Wait until it terminates.
logger.info('Stopping "report hunter" thread.')
all_report_refresher.join()
# Allow all clients looking for task results to get the bad news...
time.sleep(2)
def test_execution_of_templates(template_name):
try:
_run_checks(
"job_id_{}".format(str(uuid.uuid4())[:6]),
datetime.datetime.now(),
template_name,
template_name,
get_output_dir(),
get_template_dir(),
{},
generate_pdf_output=False,
)
finally:
_cleanup_dirs()
def test_generate_ipynb_from_py():
python_dir = tempfile.mkdtemp()
try:
set_cache("latest_sha", "fake_sha_early")
os.mkdir(python_dir + "/extra_path")
with open(os.path.join(python_dir, "extra_path", "test_report.py"), "w") as f:
f.write("#hello world\n")
report_path = os.sep.join(["extra_path", "test_report"])
with mock.patch("notebooker.utils.conversion._git_pull_templates") as pull:
conversion.python_template_dir = lambda *a, **kw: python_dir
pull.return_value = "fake_sha_early"
conversion.generate_ipynb_from_py(python_dir, report_path)
pull.return_value = "fake_sha_later"
conversion.generate_ipynb_from_py(python_dir, report_path)
conversion.generate_ipynb_from_py(python_dir, report_path)
assert get_cache("latest_sha") == "fake_sha_later"
expected_filename = _output_ipynb_name(report_path)
expected_ipynb_path = os.path.join(python_dir, "fake_sha_early", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
expected_ipynb_path = os.path.join(python_dir, "fake_sha_later", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
with mock.patch("notebooker.utils.conversion.pkg_resources.resource_filename") as resource_filename:
conversion.python_template_dir = lambda *a, **kw: None
uuid4.return_value = "uuid"
resource_filename.return_value = python_dir + "/extra_path/test_report.py"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
conversion.python_template_dir = lambda *a, **kw: python_dir
conversion.NOTEBOOKER_DISABLE_GIT = True
uuid4.return_value = "uuid_nogit"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid_nogit", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
finally:
_cleanup_dirs()
shutil.rmtree(python_dir)
def regression_test(template_dir):
logger.info("Starting regression test")
os.environ["PY_TEMPLATE_DIR"] = template_dir
try:
attempted_templates, failed_templates = [], set()
for template_name in templates._all_templates():
logger.info(
"============================[ Testing {} ]============================"
.format(template_name))
try:
attempted_templates.append(template_name)
_run_checks(
"job_id_{}".format(str(uuid.uuid4())[:6]),
datetime.datetime.now(),
template_name,
template_name,
filesystem.get_output_dir(),
filesystem.get_template_dir(),
{},
generate_pdf_output=False,
)
logger.info(
"===============================[ SUCCESS ]=============================="
)
except Exception:
failed_templates.add(template_name)
logger.info(
"===============================[ FAILED ]==============================="
)
logger.exception(
"Failed to execute template {}".format(template_name))
for template in attempted_templates:
logger.info("{}: {}".format(
template,
"FAILED" if template in failed_templates else "PASSED"))
if len(failed_templates) > 0:
raise NotebookRunException(
"The following templates failed to execute with no parameters:\n{}"
.format("\n".join(failed_templates)))
finally:
filesystem._cleanup_dirs()
def setup_and_cleanup_notebooker_filesystem(webapp_config):
try:
initialise_base_dirs(webapp_config=webapp_config)
yield
finally:
_cleanup_dirs(webapp_config)
