def test_server_examples(server_example, example, config, bokeh_server):
# mitigate some weird interaction isolated to simple ids, py2.7,
# "push_session" server usage, and TravisCI
if six.PY2: os.environ['BOKEH_SIMPLE_IDS'] = 'no'
app = build_single_handler_application(example.path)
doc = app.create_document()
if six.PY2: del os.environ['BOKEH_SIMPLE_IDS']
# remove all next-tick, periodic, and timeout callbacks
for session_callback in doc.session_callbacks:
if isinstance(session_callback, NextTickCallback):
doc.remove_next_tick_callback(session_callback)
elif isinstance(session_callback, PeriodicCallback):
doc.remove_periodic_callback(session_callback)
elif isinstance(session_callback, TimeoutCallback):
doc.remove_timeout_callback(session_callback)
else:
raise RuntimeError('Unhandled callback type', type(session_callback))
session_id = basename(example.path)
push_session(doc, session_id=session_id)
if example.no_js:
if not config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, config.option.verbose)
def test_server_examples(server_example, example, config, report,
bokeh_server):
app = build_single_handler_application(example.path)
doc = app.create_document()
# remove all next-tick, periodic, and timeout callbacks
for session_callback in doc.session_callbacks:
if isinstance(session_callback, NextTickCallback):
doc.remove_next_tick_callback(session_callback)
elif isinstance(session_callback, PeriodicCallback):
doc.remove_periodic_callback(session_callback)
elif isinstance(session_callback, TimeoutCallback):
doc.remove_timeout_callback(session_callback)
else:
raise RuntimeError('Unhandled callback type',
type(session_callback))
session_id = basename(example.path)
push_session(doc, session_id=session_id)
if example.no_js:
if not config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(
example, "http://localhost:5006/?bokeh-session-id=%s" % session_id,
config.option.verbose)
def test_file_examples(file_example, example, report, config) -> None:
if config.option.verbose:
print()
(status, duration, out, err) = _run_example(example)
info("Example run in %s" % white("%.3fs" % duration))
for line in out.split("\n"):
if len(line) == 0 or line.startswith("Wrote "):
continue
info(line, label="PY")
for line in err.split("\n"):
if len(line) == 0:
continue
warn(line, label="PY")
assert status != "timeout", "%s timed out" % example.relpath
assert status == 0, "%s failed to run (exit code %s)" % (example.relpath,
status)
if example.no_js:
if not config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(example, "file://%s.html" % example.path_no_ext,
report, config.option.verbose)
def test_server_examples(server_example, example, report, bokeh_server):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
app = build_single_handler_application(example.path)
doc = app.create_document()
# remove all next-tick, periodic, and timeout callbacks
for session_callback in doc.session_callbacks:
if isinstance(session_callback, NextTickCallback):
doc.remove_next_tick_callback(session_callback)
elif isinstance(session_callback, PeriodicCallback):
doc.remove_periodic_callback(session_callback)
elif isinstance(session_callback, TimeoutCallback):
doc.remove_timeout_callback(session_callback)
else:
raise RuntimeError('Unhandled callback type',
type(session_callback))
session_id = basename(example.path)
push_session(doc, session_id=session_id)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(
example, "http://localhost:5006/?bokeh-session-id=%s" % session_id,
'server')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def test_server_examples(server_example: Example, example: Example, report: List[Example], config: _pytest.config.Config, bokeh_server: str) -> None:
if config.option.verbose:
print()
app = build_single_handler_application(example.path)
doc = app.create_document()
# remove all next-tick, periodic, and timeout callbacks
for session_callback in doc.session_callbacks:
if isinstance(session_callback, NextTickCallback):
doc.remove_next_tick_callback(session_callback)
elif isinstance(session_callback, PeriodicCallback):
doc.remove_periodic_callback(session_callback)
elif isinstance(session_callback, TimeoutCallback):
doc.remove_timeout_callback(session_callback)
else:
raise RuntimeError('Unhandled callback type', type(session_callback))
session_id = ID(basename(example.path))
push_session(doc, session_id=session_id)
if example.no_js:
if not config.option.no_js:
warn(f"skipping bokehjs for {example.relpath}")
else:
_run_in_browser(example, f"http://localhost:5006/?bokeh-session-id={session_id}", report, config.option.verbose)
def test_server_examples(server_example, example, config, report, bokeh_server):
# mitigate some weird interaction isolated to simple ids, py2.7,
# "push_session" server usage, and TravisCI
if six.PY2: os.environ['BOKEH_SIMPLE_IDS'] = 'no'
app = build_single_handler_application(example.path)
doc = app.create_document()
if six.PY2: del os.environ['BOKEH_SIMPLE_IDS']
# remove all next-tick, periodic, and timeout callbacks
for session_callback in doc.session_callbacks:
if isinstance(session_callback, NextTickCallback):
doc.remove_next_tick_callback(session_callback)
elif isinstance(session_callback, PeriodicCallback):
doc.remove_periodic_callback(session_callback)
elif isinstance(session_callback, TimeoutCallback):
doc.remove_timeout_callback(session_callback)
else:
raise RuntimeError('Unhandled callback type', type(session_callback))
session_id = basename(example.path)
push_session(doc, session_id=session_id)
if example.no_js:
if not config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, config.option.verbose)
def _run_in_browser(example: Example, url: str, report: List[Example], verbose: bool = False) -> None:
start = time.time()
result = run_in_chrome(url)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result["success"]
timeout = result["timeout"]
errors = result["errors"]
image = result["image"]
example.store_img(image["data"])
report.append(example)
no_errors = len(errors) == 0
if timeout:
warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish"))
if verbose:
_print_webengine_output(result)
assert success, f"{example.relpath} failed to load"
assert no_errors, f"{example.relpath} failed with {len(errors)} errors"
def test_js_examples(js_example, example, config, report) -> None:
if example.no_js:
if not config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(example, "file://%s" % example.path,
config.option.verbose)
def _assert_snapshot(example, url, example_type):
screenshot_path = example.img_path
width = 1000
height = 2000 if example_type == 'notebook' else 1000
local_wait = 100
global_wait = 15000
start = time.time()
result = get_screenshot(url, screenshot_path, local_wait, global_wait,
width, height)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result['success']
timeout = result['timeout']
errors = result['errors']
no_errors = len(errors) == 0
if timeout:
warn(
"%s %s" %
(red("TIMEOUT:"), "bokehjs did not finish in %s ms" % global_wait))
if pytest.config.option.verbose:
_print_webengine_output(result)
assert success, "%s failed to load" % example.relpath
assert no_errors, "%s failed with %d errors" % (example.relpath,
len(errors))
def _run_in_browser(example, url, verbose=False):
start = time.time()
result = run_in_chrome(url)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result["success"]
timeout = result["timeout"]
errors = result["errors"]
state = result["state"]
image = result["image"]
no_errors = len(errors) == 0
if timeout:
warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish"))
if verbose:
_print_webengine_output(result)
assert success, "%s failed to load" % example.relpath
has_image = image is not None
has_state = state is not None
has_baseline = example.has_baseline
baseline_ok = True
if not has_state:
fail("no state data was produced for comparison with the baseline")
else:
new_baseline = _create_baseline(state)
example.store_baseline(new_baseline)
if not has_baseline:
fail("%s baseline doesn't exist" % example.baseline_path)
else:
result = example.diff_baseline()
if result is not None:
baseline_ok = False
fail("BASELINE DOESN'T MATCH (make sure to update baselines before running tests):")
for line in result.split("\n"):
fail(line)
example.store_img(image["data"])
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
elif not has_image:
fail("no image data was produced for comparison with the reference image")
assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
assert has_state, "%s didn't produce state data" % example.relpath
assert has_baseline, "%s doesn't have a baseline" % example.relpath
assert baseline_ok, "%s's baseline differs" % example.relpath
def pytest_runtest_call(item):
if 'example' in item.fixturenames:
if item.config.option.verbose:
print()
global _warned
if not _warned and item.config.option.no_js:
_warned = True
warn("All examples will skip js rendering and image diff (under --no-js flag)")
def test_js_examples(js_example, example, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(example, "file://%s" % example.path)
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def has_no_console_errors(selenium):
''' Helper function to detect console errors.
'''
logs = selenium.get_log('browser')
severe_errors = [l for l in logs if l.get('level') == 'SEVERE']
non_network_errors = [l for l in severe_errors if l.get('type') != 'network']
if len(non_network_errors) == 0:
return True
else:
pytest.fail('Console errors: %s' % non_network_errors)
if len(non_network_errors) == 0 and len(severe_errors) != 0:
warn("There were severe network errors (this may or may not have affected your test): %s" % severe_errors)
canvas = selenium.find_element_by_tag_name('canvas')
wait_for_canvas_resize(canvas, selenium)
def _get_pdiff(example):
img_path, ref_path, diff_path = example.img_path, example.ref_path, example.diff_path
trace("generated image: " + img_path)
ref = example.fetch_ref()
if not ref:
warn("reference image %s doesn't exist" % example.ref_url)
else:
_store_binary(ref_path, ref)
trace("saved reference: " + ref_path)
example.pixels = image_diff(diff_path, img_path, ref_path)
if example.pixels != 0:
comment = white("%.02f%%" % example.pixels) + " of pixels"
warn("generated and reference images differ: %s" % comment)
else:
ok("generated and reference images match")
def test_file_examples(file_example, example, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
(status, duration, out, err) = _run_example(example)
info("Example run in %s" % white("%.3fs" % duration))
for line in out.split("\n"):
if len(line) == 0 or line.startswith("Wrote "):
continue
info(line, label="PY")
for line in err.split("\n"):
if len(line) == 0:
continue
warn(line, label="PY")
assert status != "timeout", "%s timed out" % example.relpath
assert status == 0, "%s failed to run (exit code %s)" % (example.relpath,
status)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "file://%s.html" % example.path_no_ext,
'file')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def test_file_examples(file_example, example, config, report):
(status, duration, out, err) = _run_example(example)
info("Example run in %s" % white("%.3fs" % duration))
for line in out.split("\n"):
if len(line) == 0 or line.startswith("Wrote "):
continue
info(line, label="PY")
for line in err.split("\n"):
if len(line) == 0:
continue
warn(line, label="PY")
assert status != "timeout", "%s timed out" % example.relpath
assert status == 0, "%s failed to run (exit code %s)" % (example.relpath, status)
if example.no_js:
if not config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(example, "file://%s.html" % example.path_no_ext, config.option.verbose)
def _run_in_browser(example, url, verbose=False):
start = time.time()
result = run_in_chrome(url)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result["success"]
timeout = result["timeout"]
errors = result["errors"]
no_errors = len(errors) == 0
if timeout:
warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish"))
if verbose:
_print_webengine_output(result)
assert success, "%s failed to load" % example.relpath
assert no_errors, "%s failed with %d errors" % (example.relpath,
len(errors))
def _get_pdiff(example):
img_path, ref_path, diff_path = example.img_path, example.ref_path, example.diff_path
trace("generated image: " + img_path)
ref = example.fetch_ref()
if not ref:
warn("reference image %s doesn't exist" % example.ref_url)
else:
ref_dir = dirname(ref_path)
if not exists(ref_dir):
os.makedirs(ref_dir)
with open(ref_path, "wb") as f:
f.write(ref)
trace("saved reference: " + ref_path)
example.pixels = image_diff(diff_path, img_path, ref_path)
if example.pixels != 0:
comment = white("%.02f%%" % example.pixels) + " of pixels"
warn("generated and reference images differ: %s" % comment)
else:
ok("generated and reference images match")
def _run_in_browser(example, url):
start = time.time()
result = run_in_chrome(url)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result["success"]
timeout = result["timeout"]
errors = result["errors"]
image = result["image"]
_store_binary(example.img_path, b64decode(image["data"]))
no_errors = len(errors) == 0
if timeout:
warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish"))
if pytest.config.option.verbose:
_print_webengine_output(result)
assert success, "%s failed to load" % example.relpath
assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def _run_in_browser(example, url, verbose=False):
start = time.time()
result = run_in_chrome(url)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result["success"]
timeout = result["timeout"]
errors = result["errors"]
state = result["state"]
image = result["image"]
no_errors = len(errors) == 0
if timeout:
warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish"))
if verbose:
_print_webengine_output(result)
assert success, "%s failed to load" % example.relpath
has_image = image is not None
has_state = state is not None
has_baseline = example.has_baseline
baseline_ok = True
if not has_state:
fail("no state data was produced for comparison with the baseline")
else:
new_baseline = _create_baseline(state)
example.store_baseline(new_baseline)
if not has_baseline:
fail("%s baseline doesn't exist" % example.baseline_path)
else:
result = example.diff_baseline()
if result is not None:
baseline_ok = False
fail("BASELINE DOESN'T MATCH (make sure to update baselines before running tests):")
for line in result.split("\n"):
fail(line)
example.store_img(image["data"])
ref = example.fetch_ref()
if not ref:
warn("reference image %s doesn't exist" % example.ref_url)
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
elif not has_image:
fail("no image data was produced for comparison with the reference image")
elif ref:
pixels = example.image_diff()
if pixels != 0:
comment = white("%.02f%%" % pixels) + " of pixels"
warn("generated and reference images differ: %s" % comment)
else:
ok("generated and reference images match")
assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
assert has_state, "%s didn't produce state data" % example.relpath
assert has_baseline, "%s doesn't have a baseline" % example.relpath
assert baseline_ok, "%s's baseline differs" % example.relpath
def test_js_examples(js_example, example, config, report):
if example.no_js:
if not config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_run_in_browser(example, "file://%s" % example.path, config.option.verbose)