def test_server_examples(server_example, example, report, bokeh_server):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
app = build_single_handler_application(example.path)
doc = app.create_document()
# remove all next-tick, periodic, and timeout callbacks
for session_callback in doc.session_callbacks:
if isinstance(session_callback, NextTickCallback):
doc.remove_next_tick_callback(session_callback)
elif isinstance(session_callback, PeriodicCallback):
doc.remove_periodic_callback(session_callback)
elif isinstance(session_callback, TimeoutCallback):
doc.remove_timeout_callback(session_callback)
else:
raise RuntimeError('Unhandled callback type', type(session_callback))
session_id = basename(example.path)
push_session(doc, session_id=session_id)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, 'server')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def _get_pdiff(example):
img_path, ref_path, diff_path = example.img_path, example.ref_path, example.diff_path
trace("generated image: " + img_path)
ref = example.fetch_ref()
if not ref:
warn("reference image %s doesn't exist" % example.ref_url)
else:
ref_dir = dirname(ref_path)
if not exists(ref_dir):
os.makedirs(ref_dir)
with open(ref_path, "wb") as f:
f.write(ref)
trace("saved reference: " + ref_path)
example.pixels = image_diff(diff_path, img_path, ref_path)
if example.pixels != 0:
comment = "dimensions don't match" if example.pixels == -1 else white(
"%.02f%%" % example.pixels) + " of pixels"
warn("generated and reference images differ: %s" % comment)
else:
ok("generated and reference images match")
def _print_phantomjs_output(result):
errors = result['errors']
messages = result['messages']
resources = result['resources']
for message in messages:
msg = message['msg']
line = message.get('line')
source = message.get('source')
if source is None:
write(msg)
elif line is None:
write("%s: %s" % (source, msg))
else:
write("%s:%s: %s" % (source, line, msg))
# Process resources
for resource in resources:
url = resource['url']
if url.endswith(".png"):
ok("%s: %s (%s)" % (url, yellow(resource['status']), resource['statusText']))
else:
warn("Resource error:: %s: %s (%s)" % (url, red(resource['status']), resource['statusText']))
# You can have a successful test, and still have errors reported, so not failing here.
for error in errors:
warn("%s: %s" % (red("PhatomJS Error: "), error['msg']))
for item in error['trace']:
write(" %s: %d" % (item['file'], item['line']))
def _assert_snapshot(example, url, example_type):
screenshot_path = example.img_path
height = 2000 if example_type == 'notebook' else 1000
wait = 30000
start = time.time()
result = get_phantomjs_screenshot(url, screenshot_path, 1000, wait, 1000,
height)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result['success']
timeout = result['timeout']
errors = result['errors']
resources = result['resources']
no_errors = len(errors) == 0
no_resources = len(resources) == 0
if timeout:
warn("%s: %s" %
(red("TIMEOUT: "), "bokehjs did not finish in %s ms" % wait))
if pytest.config.option.verbose:
_print_phantomjs_output(result)
assert success, "%s failed to load" % example.relpath
assert no_resources, "%s failed with %d missing resources" % (
example.relpath, len(resources))
assert no_errors, "%s failed with %d errors" % (example.relpath,
len(errors))
def test_server_examples(server_example, example, report, bokeh_server):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
app = build_single_handler_application(example.path)
doc = app.create_document()
# remove all next-tick, periodic, and timeout callbacks
for session_callback in doc.session_callbacks:
if isinstance(session_callback, NextTickCallback):
doc.remove_next_tick_callback(session_callback)
elif isinstance(session_callback, PeriodicCallback):
doc.remove_periodic_callback(session_callback)
elif isinstance(session_callback, TimeoutCallback):
doc.remove_timeout_callback(session_callback)
else:
raise RuntimeError('Unhandled callback type', type(session_callback))
session_id = basename(example.path)
push_session(doc, session_id=session_id)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, 'server')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def _assert_snapshot(example, url, example_type):
screenshot_path = example.img_path
width = 1000
height = 2000 if example_type == 'notebook' else 1000
local_wait = 100
global_wait = 15000
start = time.time()
result = get_screenshot(url, screenshot_path, local_wait, global_wait, width, height)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result['success']
timeout = result['timeout']
errors = result['errors']
no_errors = len(errors) == 0
if timeout:
warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish in %s ms" % global_wait))
if pytest.config.option.verbose:
_print_webengine_output(result)
assert success, "%s failed to load" % example.relpath
assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def _assert_snapshot(example, url, example_type):
screenshot_path = example.img_path
width = 1000
height = 2000 if example_type == 'notebook' else 1000
local_wait = 100
global_wait = 15000
start = time.time()
result = get_screenshot(url, screenshot_path, local_wait, global_wait, width, height)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result['success']
timeout = result['timeout']
errors = result['errors']
no_errors = len(errors) == 0
if timeout:
warn("%s %s" % (red("TIMEOUT:"), "bokehjs did not finish in %s ms" % global_wait))
if pytest.config.option.verbose:
_print_webengine_output(result)
assert success, "%s failed to load" % example.relpath
assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def test_server_examples(server_example, example, report, bokeh_server):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
app = build_single_handler_application(example.path)
doc = app.create_document()
# remove all periodic and timeout callbacks
for session_callback in list(doc._session_callbacks.keys()):
doc._remove_session_callback(session_callback)
session_id = "session_id"
push_session(doc, session_id=session_id)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "http://localhost:5006/?bokeh-session-id=%s" % session_id, 'server')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def _assert_snapshot(example, url, example_type):
screenshot_path = example.img_path
height = 2000 if example_type == 'notebook' else 1000
wait = 30000
start = time.time()
result = get_phantomjs_screenshot(url, screenshot_path, 1000, wait, 1000, height)
end = time.time()
info("Example rendered in %s" % white("%.3fs" % (end - start)))
success = result['success']
timeout = result['timeout']
errors = result['errors']
resources = result['resources']
no_errors = len(errors) == 0
no_resources = len(resources) == 0
if timeout:
warn("%s: %s" % (red("TIMEOUT: "), "bokehjs did not finish in %s ms" % wait))
if pytest.config.option.verbose:
_print_phantomjs_output(result)
assert success, "%s failed to load" % example.relpath
assert no_resources, "%s failed with %d missing resources" % (example.relpath, len(resources))
assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
def test_server_examples(server_example, example, report, bokeh_server):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
app = build_single_handler_application(example.path)
doc = app.create_document()
# remove all periodic and timeout callbacks
for session_callback in list(doc._session_callbacks.keys()):
doc._remove_session_callback(session_callback)
session_id = "session_id"
push_session(doc, session_id=session_id)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(
example, "http://localhost:5006/?bokeh-session-id=%s" % session_id,
'server')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def test_js_examples(js_example, example, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "file://%s" % example.path, 'js')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def test_js_examples(js_example, example, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "file://%s" % example.path, 'js')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def has_no_console_errors(selenium):
"""
Helper function to detect console errors.
"""
logs = selenium.get_log('browser')
severe_errors = [l for l in logs if l.get('level') == 'SEVERE']
non_network_errors = [l for l in severe_errors if l.get('type') != 'network']
if len(non_network_errors) == 0:
return True
else:
pytest.fail('Console errors: %s' % non_network_errors)
if len(non_network_errors) == 0 and len(severe_errors) != 0:
warn("There were severe network errors (this may or may not have affected your test): %s" % severe_errors)
canvas = selenium.find_element_by_tag_name('canvas')
wait_for_canvas_resize(canvas, selenium)
def has_no_console_errors(selenium):
"""
Helper function to detect console errors.
"""
logs = selenium.get_log("browser")
severe_errors = [l for l in logs if l.get("level") == "SEVERE"]
non_network_errors = [l for l in severe_errors if l.get("type") != "network"]
if len(non_network_errors) == 0:
return True
else:
pytest.fail("Console errors: %s" % non_network_errors)
if len(non_network_errors) == 0 and len(severe_errors) != 0:
warn("There were severe network errors (this may or may not have affected your test): %s" % severe_errors)
canvas = selenium.find_element_by_tag_name("canvas")
wait_for_canvas_resize(canvas, selenium)
def _run_example(example, log_file):
example_path = join(example_dir, example)
code = """\
filename = '%s'
import random
random.seed(1)
import numpy as np
np.random.seed(1)
with open(filename, 'rb') as example:
exec(compile(example.read(), filename, 'exec'))
""" % example_path
cmd = ["python", "-c", code]
cwd = dirname(example_path)
env = os.environ.copy()
env['BOKEH_RESOURCES'] = 'relative'
env['BOKEH_BROWSER'] = 'none'
class Timeout(Exception):
pass
def alarm_handler(sig, frame):
raise Timeout
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(10)
try:
proc = subprocess.Popen(cmd,
cwd=cwd,
env=env,
stdout=log_file,
stderr=log_file)
return proc.wait()
except Timeout:
warn("Timeout - Example timed out when attempting to run")
proc.kill()
return 0
finally:
signal.alarm(0)
def test_server_examples(server_example, example, bokeh_server, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
# Note this is currently broken - server uses random sessions but we're
# calling for "default" here - this has been broken for a while.
# https://github.com/bokeh/bokeh/issues/3897
url = '%s/?bokeh-session-id=%s' % (bokeh_server, example.name)
assert _run_example(example) == 0, 'Example did not run'
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, url, 'server')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def test_server_examples(server_example, example, bokeh_server, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
# Note this is currently broken - server uses random sessions but we're
# calling for "default" here - this has been broken for a while.
# https://github.com/bokeh/bokeh/issues/3897
url = '%s/?bokeh-session-id=%s' % (bokeh_server, example.name)
assert _run_example(example) == 0, 'Example did not run'
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, url, 'server')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def _run_example(example, log_file):
example_path = join(example_dir, example)
code = """\
__file__ = filename = '%s'
import random
random.seed(1)
import numpy as np
np.random.seed(1)
with open(filename, 'rb') as example:
exec(compile(example.read(), filename, 'exec'))
""" % example_path
cmd = ["python", "-c", code]
cwd = dirname(example_path)
env = os.environ.copy()
env['BOKEH_RESOURCES'] = 'relative'
env['BOKEH_BROWSER'] = 'none'
class Timeout(Exception):
pass
def alarm_handler(sig, frame):
raise Timeout
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(10)
try:
proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log_file, stderr=log_file)
return proc.wait()
except Timeout:
warn("Timeout - Example timed out when attempting to run")
proc.kill()
return 0
finally:
signal.alarm(0)
def test_file_examples(file_example, example, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
(status, duration, out, err) = _run_example(example)
info("Example run in %s" % white("%.3fs" % duration))
for line in out.split("\n"):
if len(line) == 0 or line.startswith("Wrote "):
continue
info(line, label="PY")
for line in err.split("\n"):
if len(line) == 0:
continue
warn(line, label="PY")
assert status != "timeout", "%s timed out" % example.relpath
assert status == 0, "%s failed to run (exit code %s)" % (example.relpath,
status)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "file://%s.html" % example.path_no_ext,
'file')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def test_file_examples(file_example, example, report):
if example.is_skip:
pytest.skip("skipping %s" % example.relpath)
(status, duration, out, err) = _run_example(example)
info("Example run in %s" % white("%.3fs" % duration))
for line in out.split("\n"):
if len(line) == 0 or line.startswith("Wrote "):
continue
info(line, label="PY")
for line in err.split("\n"):
if len(line) == 0:
continue
warn(line, label="PY")
assert status != "timeout", "%s timed out" % example.relpath
assert status == 0, "%s failed to run (exit code %s)" % (example.relpath, status)
if example.no_js:
if not pytest.config.option.no_js:
warn("skipping bokehjs for %s" % example.relpath)
else:
_assert_snapshot(example, "file://%s.html" % example.path_no_ext, 'file')
if example.no_diff:
warn("skipping image diff for %s" % example.relpath)
else:
_get_pdiff(example)
def _get_pdiff(example, diff):
test_png, ref_png, diff_png = get_example_pngs(example, diff)
info("generated image: " + test_png)
retrieved_reference_image = _get_reference_image_from_s3(example, diff)
if retrieved_reference_image:
ref_png_path = dirname(ref_png)
if not exists(ref_png_path):
os.makedirs(ref_png_path)
with open(ref_png, "wb") as f:
f.write(retrieved_reference_image)
info("saved reference: " + ref_png)
code = process_image_diff(diff_png, test_png, ref_png)
if code != 0:
warn("generated and reference images differ")
warn("diff: " + diff_png)
else:
ok("generated and reference images match")
def _get_pdiff(example):
img_path, ref_path, diff_path = example.img_path, example.ref_path, example.diff_path
trace("generated image: " + img_path)
ref = example.fetch_ref()
if not ref:
warn("reference image %s doesn't exist" % example.ref_url)
else:
ref_dir = dirname(ref_path)
if not exists(ref_dir):
os.makedirs(ref_dir)
with open(ref_path, "wb") as f:
f.write(ref)
trace("saved reference: " + ref_path)
example.pixels = image_diff(diff_path, img_path, ref_path)
if example.pixels != 0:
comment = "dimensions don't match" if example.pixels == -1 else white("%.02f%%" % example.pixels) + " of pixels"
warn("generated and reference images differ: %s" % comment)
else:
ok("generated and reference images match")
