def test_server_examples(server_example, example, report, bokeh_server): if example.is_skip: pytest.skip("skipping %s" % example.relpath) app = build_single_handler_application(example.path) doc = app.create_document() # remove all next-tick, periodic, and timeout callbacks for session_callback in doc.session_callbacks: if isinstance(session_callback, NextTickCallback): doc.remove_next_tick_callback(session_callback) elif isinstance(session_callback, PeriodicCallback): doc.remove_periodic_callback(session_callback) elif isinstance(session_callback, TimeoutCallback): doc.remove_timeout_callback(session_callback) else: raise RuntimeError('Unhandled callback type', type(session_callback)) session_id = basename(example.path) push_session(doc, session_id=session_id) if example.no_js: if not pytest.config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _assert_snapshot(example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, 'server') if example.no_diff: warn("skipping image diff for %s" % example.relpath) else: _get_pdiff(example)
def _get_pdiff(example): img_path, ref_path, diff_path = example.img_path, example.ref_path, example.diff_path trace("generated image: " + img_path) ref = example.fetch_ref() if not ref: warn("reference image %s doesn't exist" % example.ref_url) else: ref_dir = dirname(ref_path) if not exists(ref_dir): os.makedirs(ref_dir) with open(ref_path, "wb") as f: f.write(ref) trace("saved reference: " + ref_path) example.pixels = image_diff(diff_path, img_path, ref_path) if example.pixels != 0: comment = "dimensions don't match" if example.pixels == -1 else white( "%.02f%%" % example.pixels) + " of pixels" warn("generated and reference images differ: %s" % comment) else: ok("generated and reference images match")
def _print_phantomjs_output(result): errors = result['errors'] messages = result['messages'] resources = result['resources'] for message in messages: msg = message['msg'] line = message.get('line') source = message.get('source') if source is None: write(msg) elif line is None: write("%s: %s" % (source, msg)) else: write("%s:%s: %s" % (source, line, msg)) # Process resources for resource in resources: url = resource['url'] if url.endswith(".png"): ok("%s: %s (%s)" % (url, yellow(resource['status']), resource['statusText'])) else: warn("Resource error:: %s: %s (%s)" % (url, red(resource['status']), resource['statusText'])) # You can have a successful test, and still have errors reported, so not failing here. for error in errors: warn("%s: %s" % (red("PhatomJS Error: "), error['msg'])) for item in error['trace']: write(" %s: %d" % (item['file'], item['line']))
def _assert_snapshot(example, url, example_type): screenshot_path = example.img_path height = 2000 if example_type == 'notebook' else 1000 wait = 30000 start = time.time() result = get_phantomjs_screenshot(url, screenshot_path, 1000, wait, 1000, height) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result['success'] timeout = result['timeout'] errors = result['errors'] resources = result['resources'] no_errors = len(errors) == 0 no_resources = len(resources) == 0 if timeout: warn("%s: %s" % (red("TIMEOUT: "), "bokehjs did not finish in %s ms" % wait)) if pytest.config.option.verbose: _print_phantomjs_output(result) assert success, "%s failed to load" % example.relpath assert no_resources, "%s failed with %d missing resources" % ( example.relpath, len(resources)) assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def _assert_snapshot(example, url, example_type): screenshot_path = example.img_path width = 1000 height = 2000 if example_type == 'notebook' else 1000 local_wait = 100 global_wait = 15000 start = time.time() result = get_screenshot(url, screenshot_path, local_wait, global_wait, width, height) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result['success'] timeout = result['timeout'] errors = result['errors'] no_errors = len(errors) == 0 if timeout: warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish in %s ms" % global_wait)) if pytest.config.option.verbose: _print_webengine_output(result) assert success, "%s failed to load" % example.relpath assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def test_server_examples(server_example, example, report, bokeh_server): if example.is_skip: pytest.skip("skipping %s" % example.relpath) app = build_single_handler_application(example.path) doc = app.create_document() # remove all periodic and timeout callbacks for session_callback in list(doc._session_callbacks.keys()): doc._remove_session_callback(session_callback) session_id = "session_id" push_session(doc, session_id=session_id) if example.no_js: if not pytest.config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _assert_snapshot(example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, 'server') if example.no_diff: warn("skipping image diff for %s" % example.relpath) else: _get_pdiff(example)
def _assert_snapshot(example, url, example_type): screenshot_path = example.img_path height = 2000 if example_type == 'notebook' else 1000 wait = 30000 start = time.time() result = get_phantomjs_screenshot(url, screenshot_path, 1000, wait, 1000, height) end = time.time() info("Example rendered in %s" % white("%.3fs" % (end - start))) success = result['success'] timeout = result['timeout'] errors = result['errors'] resources = result['resources'] no_errors = len(errors) == 0 no_resources = len(resources) == 0 if timeout: warn("%s: %s" % (red("TIMEOUT: "), "bokehjs did not finish in %s ms" % wait)) if pytest.config.option.verbose: _print_phantomjs_output(result) assert success, "%s failed to load" % example.relpath assert no_resources, "%s failed with %d missing resources" % (example.relpath, len(resources)) assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def test_server_examples(server_example, example, report, bokeh_server): if example.is_skip: pytest.skip("skipping %s" % example.relpath) app = build_single_handler_application(example.path) doc = app.create_document() # remove all periodic and timeout callbacks for session_callback in list(doc._session_callbacks.keys()): doc._remove_session_callback(session_callback) session_id = "session_id" push_session(doc, session_id=session_id) if example.no_js: if not pytest.config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _assert_snapshot( example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, 'server') if example.no_diff: warn("skipping image diff for %s" % example.relpath) else: _get_pdiff(example)
def test_js_examples(js_example, example, report): if example.is_skip: pytest.skip("skipping %s" % example.relpath) if example.no_js: if not pytest.config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _assert_snapshot(example, "file://%s" % example.path, 'js') if example.no_diff: warn("skipping image diff for %s" % example.relpath) else: _get_pdiff(example)
def has_no_console_errors(selenium): """ Helper function to detect console errors. """ logs = selenium.get_log('browser') severe_errors = [l for l in logs if l.get('level') == 'SEVERE'] non_network_errors = [l for l in severe_errors if l.get('type') != 'network'] if len(non_network_errors) == 0: return True else: pytest.fail('Console errors: %s' % non_network_errors) if len(non_network_errors) == 0 and len(severe_errors) != 0: warn("There were severe network errors (this may or may not have affected your test): %s" % severe_errors) canvas = selenium.find_element_by_tag_name('canvas') wait_for_canvas_resize(canvas, selenium)
def has_no_console_errors(selenium): """ Helper function to detect console errors. """ logs = selenium.get_log("browser") severe_errors = [l for l in logs if l.get("level") == "SEVERE"] non_network_errors = [l for l in severe_errors if l.get("type") != "network"] if len(non_network_errors) == 0: return True else: pytest.fail("Console errors: %s" % non_network_errors) if len(non_network_errors) == 0 and len(severe_errors) != 0: warn("There were severe network errors (this may or may not have affected your test): %s" % severe_errors) canvas = selenium.find_element_by_tag_name("canvas") wait_for_canvas_resize(canvas, selenium)
def _run_example(example, log_file): example_path = join(example_dir, example) code = """\ filename = '%s' import random random.seed(1) import numpy as np np.random.seed(1) with open(filename, 'rb') as example: exec(compile(example.read(), filename, 'exec')) """ % example_path cmd = ["python", "-c", code] cwd = dirname(example_path) env = os.environ.copy() env['BOKEH_RESOURCES'] = 'relative' env['BOKEH_BROWSER'] = 'none' class Timeout(Exception): pass def alarm_handler(sig, frame): raise Timeout signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(10) try: proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log_file, stderr=log_file) return proc.wait() except Timeout: warn("Timeout - Example timed out when attempting to run") proc.kill() return 0 finally: signal.alarm(0)
def test_server_examples(server_example, example, bokeh_server, report): if example.is_skip: pytest.skip("skipping %s" % example.relpath) # Note this is currently broken - server uses random sessions but we're # calling for "default" here - this has been broken for a while. # https://github.com/bokeh/bokeh/issues/3897 url = '%s/?bokeh-session-id=%s' % (bokeh_server, example.name) assert _run_example(example) == 0, 'Example did not run' if example.no_js: if not pytest.config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _assert_snapshot(example, url, 'server') if example.no_diff: warn("skipping image diff for %s" % example.relpath) else: _get_pdiff(example)
def _run_example(example, log_file): example_path = join(example_dir, example) code = """\ __file__ = filename = '%s' import random random.seed(1) import numpy as np np.random.seed(1) with open(filename, 'rb') as example: exec(compile(example.read(), filename, 'exec')) """ % example_path cmd = ["python", "-c", code] cwd = dirname(example_path) env = os.environ.copy() env['BOKEH_RESOURCES'] = 'relative' env['BOKEH_BROWSER'] = 'none' class Timeout(Exception): pass def alarm_handler(sig, frame): raise Timeout signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(10) try: proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log_file, stderr=log_file) return proc.wait() except Timeout: warn("Timeout - Example timed out when attempting to run") proc.kill() return 0 finally: signal.alarm(0)
def test_file_examples(file_example, example, report): if example.is_skip: pytest.skip("skipping %s" % example.relpath) (status, duration, out, err) = _run_example(example) info("Example run in %s" % white("%.3fs" % duration)) for line in out.split("\n"): if len(line) == 0 or line.startswith("Wrote "): continue info(line, label="PY") for line in err.split("\n"): if len(line) == 0: continue warn(line, label="PY") assert status != "timeout", "%s timed out" % example.relpath assert status == 0, "%s failed to run (exit code %s)" % (example.relpath, status) if example.no_js: if not pytest.config.option.no_js: warn("skipping bokehjs for %s" % example.relpath) else: _assert_snapshot(example, "file://%s.html" % example.path_no_ext, 'file') if example.no_diff: warn("skipping image diff for %s" % example.relpath) else: _get_pdiff(example)
def _get_pdiff(example, diff): test_png, ref_png, diff_png = get_example_pngs(example, diff) info("generated image: " + test_png) retrieved_reference_image = _get_reference_image_from_s3(example, diff) if retrieved_reference_image: ref_png_path = dirname(ref_png) if not exists(ref_png_path): os.makedirs(ref_png_path) with open(ref_png, "wb") as f: f.write(retrieved_reference_image) info("saved reference: " + ref_png) code = process_image_diff(diff_png, test_png, ref_png) if code != 0: warn("generated and reference images differ") warn("diff: " + diff_png) else: ok("generated and reference images match")
def _get_pdiff(example): img_path, ref_path, diff_path = example.img_path, example.ref_path, example.diff_path trace("generated image: " + img_path) ref = example.fetch_ref() if not ref: warn("reference image %s doesn't exist" % example.ref_url) else: ref_dir = dirname(ref_path) if not exists(ref_dir): os.makedirs(ref_dir) with open(ref_path, "wb") as f: f.write(ref) trace("saved reference: " + ref_path) example.pixels = image_diff(diff_path, img_path, ref_path) if example.pixels != 0: comment = "dimensions don't match" if example.pixels == -1 else white("%.02f%%" % example.pixels) + " of pixels" warn("generated and reference images differ: %s" % comment) else: ok("generated and reference images match")