def test_file_examples(file_example, example, report, config, bokeh_server) -> None: if config.option.verbose: print() (status, duration, out, err) = _run_example(example) info("Example run in %s" % white("%.3fs" % duration)) for line in out.split("\n"): if len(line) == 0 or line.startswith("Wrote "): continue info(line, label="PY") for line in err.split("\n"): if len(line) == 0: continue warn(line, label="PY") assert status != "timeout", "%s timed out" % example.relpath assert status == 0, "%s failed to run (exit code %s)" % (example.relpath, status) if example.no_js: if not config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _run_in_browser(example, "file://%s.html" % example.path_no_ext, report, config.option.verbose)
def _run_in_browser(example, url, report, verbose=False): start = time.time() result = run_in_chrome(url) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result["success"] timeout = result["timeout"] errors = result["errors"] image = result["image"] example.store_img(image["data"]) report.append(example) no_errors = len(errors) == 0 if timeout: warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish")) if verbose: _print_webengine_output(result) assert success, "%s failed to load" % example.relpath assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def _assert_snapshot(example, url, example_type): screenshot_path = example.img_path width = 1000 height = 2000 if example_type == 'notebook' else 1000 local_wait = 100 global_wait = 15000 start = time.time() result = get_screenshot(url, screenshot_path, local_wait, global_wait, width, height) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result['success'] timeout = result['timeout'] errors = result['errors'] no_errors = len(errors) == 0 if timeout: warn( "%s %s" % (red("TIMEOUT:"), "bokehjs did not finish in %s ms" % global_wait)) if pytest.config.option.verbose: _print_webengine_output(result) assert success, "%s failed to load" % example.relpath assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def test_file_examples(file_example, example, report): if example.is_skip: pytest.skip("skipping %s" % example.relpath) (status, duration, out, err) = _run_example(example) info("Example run in %s" % white("%.3fs" % duration)) for line in out.split("\n"): if len(line) == 0 or line.startswith("Wrote "): continue info(line, label="PY") for line in err.split("\n"): if len(line) == 0: continue warn(line, label="PY") assert status != "timeout", "%s timed out" % example.relpath assert status == 0, "%s failed to run (exit code %s)" % (example.relpath, status) if example.no_js: if not pytest.config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _run_in_browser(example, "file://%s.html" % example.path_no_ext) if example.no_diff: warn("skipping image diff for %s" % example.relpath) else: _get_pdiff(example)
def _run_in_browser(example, url, verbose=False): start = time.time() result = run_in_chrome(url) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result["success"] timeout = result["timeout"] errors = result["errors"] state = result["state"] image = result["image"] no_errors = len(errors) == 0 if timeout: warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish")) if verbose: _print_webengine_output(result) assert success, "%s failed to load" % example.relpath has_image = image is not None has_state = state is not None has_baseline = example.has_baseline baseline_ok = True if not has_state: fail("no state data was produced for comparison with the baseline") else: new_baseline = _create_baseline(state) example.store_baseline(new_baseline) if not has_baseline: fail("%s baseline doesn't exist" % example.baseline_path) else: result = example.diff_baseline() if result is not None: baseline_ok = False fail("BASELINE DOESN'T MATCH (make sure to update baselines before running tests):") for line in result.split("\n"): fail(line) example.store_img(image["data"]) if example.no_diff: warn("skipping image diff for %s" % example.relpath) elif not has_image: fail("no image data was produced for comparison with the reference image") assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors)) assert has_state, "%s didn't produce state data" % example.relpath assert has_baseline, "%s doesn't have a baseline" % example.relpath assert baseline_ok, "%s's baseline differs" % example.relpath
def _print_webengine_output(result): errors = result['errors'] messages = result['messages'] for message in messages: level = message['level'] text = message['text'] url = message['url'] line = message['line'] col = message['col'] msg = "{%s} %s:%s:%s %s" % (level, url, line, col, text) info(msg, label="JS") for error in errors: for line in error['text'].split("\n"): fail(line, label="JS")
def _print_webengine_output(result: JSResult) -> None: errors = result['errors'] messages = result['messages'] for message in messages: level = message['level'] text = message['text'] url = message['url'] line = message['line'] col = message['col'] msg = "{%s} %s:%s:%s %s" % (level, url, line, col, text) info(msg, label="JS") for error in errors: _url = error["url"] if _url is not None: fail(f"@{_url}", label="JS") for _line in error['text'].split("\n"): fail(_line, label="JS")
def test_file_examples(file_example, example, config, report): (status, duration, out, err) = _run_example(example) info("Example run in %s" % white("%.3fs" % duration)) for line in out.split("\n"): if len(line) == 0 or line.startswith("Wrote "): continue info(line, label="PY") for line in err.split("\n"): if len(line) == 0: continue warn(line, label="PY") assert status != "timeout", "%s timed out" % example.relpath assert status == 0, "%s failed to run (exit code %s)" % (example.relpath, status) if example.no_js: if not config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _run_in_browser(example, "file://%s.html" % example.path_no_ext, config.option.verbose)
def _run_in_browser(example, url): start = time.time() result = run_in_chrome(url) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result["success"] timeout = result["timeout"] errors = result["errors"] image = result["image"] _store_binary(example.img_path, b64decode(image["data"])) no_errors = len(errors) == 0 if timeout: warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish")) if pytest.config.option.verbose: _print_webengine_output(result) assert success, "%s failed to load" % example.relpath assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def _run_in_browser(example, url, verbose=False): start = time.time() result = run_in_chrome(url) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result["success"] timeout = result["timeout"] errors = result["errors"] state = result["state"] image = result["image"] no_errors = len(errors) == 0 if timeout: warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish")) if verbose: _print_webengine_output(result) assert success, "%s failed to load" % example.relpath has_image = image is not None has_state = state is not None has_baseline = example.has_baseline baseline_ok = True if not has_state: fail("no state data was produced for comparison with the baseline") else: new_baseline = _create_baseline(state) example.store_baseline(new_baseline) if not has_baseline: fail("%s baseline doesn't exist" % example.baseline_path) else: result = example.diff_baseline() if result is not None: baseline_ok = False fail("BASELINE DOESN'T MATCH (make sure to update baselines before running tests):") for line in result.split("\n"): fail(line) example.store_img(image["data"]) ref = example.fetch_ref() if not ref: warn("reference image %s doesn't exist" % example.ref_url) if example.no_diff: warn("skipping image diff for %s" % example.relpath) elif not has_image: fail("no image data was produced for comparison with the reference image") elif ref: pixels = example.image_diff() if pixels != 0: comment = white("%.02f%%" % pixels) + " of pixels" warn("generated and reference images differ: %s" % comment) else: ok("generated and reference images match") assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors)) assert has_state, "%s didn't produce state data" % example.relpath assert has_baseline, "%s doesn't have a baseline" % example.relpath assert baseline_ok, "%s's baseline differs" % example.relpath