def setUp(self):
"""Bind to the LNT test instance."""
_, instance_path, shared_inputs = sys.argv
app = lnt.server.ui.app.App.create_standalone(instance_path)
app.testing = True
self.client = app.test_client()
self.shared_inputs = shared_inputs
def sanity_check_instance(instance_path):
# Create an application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Create a test client.
client = app.test_client()
# Fetch the index page.
index = client.get('/')
# Visit all the test suites.
test_suite_link_rex = re.compile(""" <a href="(.*)">(.*)</a><br>""")
test_suite_list_start = index.data.index("<h3>Test Suites</h3>")
test_suite_list_end = index.data.index("</div>", test_suite_list_start)
for ln in index.data[test_suite_list_start:test_suite_list_end].split("\n"):
# Ignore non-matching lines.
print >>sys.stderr,ln
m = test_suite_link_rex.match(ln)
if not m:
continue
# We found a test suite link...
link,name = m.groups()
logging.info("visiting test suite %r", name)
# Get the test suite overview page.
overview = client.get(os.path.join("/", link))
assert "LNT : %s - Recent Activity" % (name,) in overview.data
def setUp(self):
"""Bind to the LNT test instance."""
_, instance_path = sys.argv
self.instance_path = instance_path
app = lnt.server.ui.app.App.create_standalone(instance_path)
app.testing = True
self.client = app.test_client()
def setUp(self):
"""Bind to the LNT test instance."""
instance_path = sys.argv[1]
app = lnt.server.ui.app.App.create_standalone(instance_path)
app.testing = True
app.config['WTF_CSRF_ENABLED'] = False
self.client = app.test_client()
def sanity_check_instance(instance_path):
# Create an application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Create a test client.
client = app.test_client()
# Fetch the index page.
index = client.get('/')
# Visit all the test suites.
test_suite_link_rex = re.compile(""" <a href="(.*)">(.*)</a><br>""")
data = index.get_data(as_text=True)
test_suite_list_start = data.index("<h3>Test Suites</h3>")
test_suite_list_end = data.index("</div>", test_suite_list_start)
for ln in data[test_suite_list_start:test_suite_list_end].split("\n"):
# Ignore non-matching lines.
print(ln, file=sys.stderr)
m = test_suite_link_rex.match(ln)
if not m:
continue
# We found a test suite link...
link, name = m.groups()
logging.info("visiting test suite %r", name)
# Get the test suite overview page.
overview = client.get(os.path.join("/", link))
assert "LNT : %s - Recent Activity" % (name, ) in overview.data
def setUp(self):
"""Bind to the LNT test instance."""
_, instance_path, shared_inputs = sys.argv
app = lnt.server.ui.app.App.create_standalone(instance_path)
app.testing = True
self.client = app.test_client()
self.shared_inputs = shared_inputs
def setUp(self):
"""Bind to the LNT test instance."""
instance_path = sys.argv[1]
app = lnt.server.ui.app.App.create_standalone(instance_path)
app.testing = True
app.config['WTF_CSRF_ENABLED'] = False
self.client = app.test_client()
def main():
instance_path = sys.argv[1]
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
app.config['WTF_CSRF_ENABLED'] = False
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_html(client, '/')
# Get the V4 overview page.
check_html(client, '/v4/nts/')
# Get a machine overview page.
check_html(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
check_json(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_html(client, '/v4/nts/all_orders')
# Get an order page.
check_html(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
# Check that we can promote a baseline, then demote.
form_data = dict(name="foo_baseline",
description="foo_descrimport iption",
prmote=True)
r = client.post('/v4/nts/order/3', data=form_data)
# We should redirect to the last page and flash.
assert r.status_code == HTTP_REDIRECT
# Try with redirect.
r = client.post('/v4/nts/order/3', data=form_data, follow_redirects=True)
assert r.status_code == HTTP_OK
# Should see baseline displayed in page body.
assert "Baseline - foo_baseline" in r.data
# Now demote it.
data2 = dict(name="foo_baseline",
description="foo_description",
update=False,
promote=False,
demote=True)
r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
assert r.status_code == HTTP_OK
# Baseline should no longer be shown in page baseline.
assert "Baseline - foo_baseline" not in r.data
# Leave a baseline in place for the rest of the tests.
client.post('/v4/nts/order/3', data=form_data)
check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT)
with app.test_client() as c:
c.get('/v4/nts/set_baseline/1')
session.get('baseline-default-nts') == 1
# Get a run result page (and associated views).
check_html(client, '/v4/nts/1')
check_json(client, '/v4/nts/1?json=true')
check_html(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/9999/text_report',
expected_code=HTTP_NOT_FOUND)
# Get a graph page. This has been changed to redirect.
check_redirect(client, '/v4/nts/1/graph?test.3=2',
'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')
# Get a run that contains generic producer information
check_producer_label(
client, '/v4/nts/7',
['Current', '152293', '2012-05-10T16:28:23', '0:00:35', 'Producer'])
check_producer_label(
client, '/v4/nts/8',
['Current', '152294', '2012-05-11T16:28:23', '0:00:35', 'Producer'])
# Get a run that contains Buildbot producer information
check_producer_label(client, '/v4/nts/7', [
'Previous', '152292', '2012-05-01T16:28:23', '0:00:35',
'some-builder #987'
])
check_producer_label(client, '/v4/nts/9', [
'Current', '152295', '2012-05-12T16:28:23', '0:00:35',
'some-builder #999'
])
# Get the new graph page.
check_html(client, '/v4/nts/graph?plot.0=1.3.2')
# Don't crash when requesting non-existing data
check_html(client, '/v4/nts/graph?plot.9999=1.3.2')
check_code(client,
'/v4/nts/graph?plot.0=9999.3.2',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/graph?plot.0=1.3.9999',
expected_code=HTTP_NOT_FOUND)
check_json(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
# Get the mean graph page.
check_html(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client,
'/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check baselines work.
check_html(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
# Check some variations of the daily report work.
check_html(client, '/v4/nts/daily_report/2012/4/12')
check_html(client, '/v4/nts/daily_report/2012/4/11')
check_html(client, '/v4/nts/daily_report/2012/4/13')
check_html(client, '/v4/nts/daily_report/2012/4/10')
check_html(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
'/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
# Don't crash when using a parameter that happens to have the same name as
# a flask URL variable.
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
check_html(client, '/v4/nts/daily_report/1999/4/12')
check_code(client,
'/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/daily_report/2012/13/12',
expected_code=HTTP_BAD_REQUEST)
check_code(client,
'/v4/nts/daily_report/2012/4/32',
expected_code=HTTP_BAD_REQUEST)
# check ?filter-machine-regex= filter
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2',
1)
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine',
2)
check_nr_machines_reported(
client,
'/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
# Don't crash on an invalid regular expression:
# FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
# ignoring the invalid regex.
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
# check that a regression seen between 2 consecutive runs that are
# more than a day apart gets reported
result_table_20120504 = get_results_table(
client, '/v4/nts/daily_report/2012/5/04', "Execution Time")
check_table_content(
result_table_20120504,
[["test1", ""], ["", "machine2", "1.000", "-", "900.00%", ""],
["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]])
check_table_links(
result_table_20120504,
[[], ["/db_default/v4/nts/graph?plot.0=2.4.2&highlight_run=6"], [],
["/db_default/v4/nts/graph?plot.0=2.5.2&highlight_run=6"]])
check_body_nr_tests_table(client, '/v4/nts/daily_report/2012/5/04',
[['machine2', '2', '0', '1']])
# Check that a failing result does not show up in the spark line
# as a dot with value 0.
result_table_20120513 = get_results_table(
client, '/v4/nts/daily_report/2012/5/13?num_days=3', "Execution Time")
check_table_content(result_table_20120513, [
["test6", ""],
["", "machine2", "1.000", "FAIL", "PASS", ""],
["test_hash1", ""],
["", "machine2", "1.000", '-', '20.00%', ""],
["test_hash2", ""],
["", "machine2", "1.000", '-', '20.00%', ""],
["test_mhash_on_run", ""],
["", "machine2", "1.000", '-', '20.00%', ""],
])
check_table_links(
result_table_20120513,
[[], ['/db_default/v4/nts/graph?plot.0=2.6.2&highlight_run=9'], [],
['/db_default/v4/nts/graph?plot.0=2.7.2&highlight_run=9'], [],
['/db_default/v4/nts/graph?plot.0=2.8.2&highlight_run=9'], [],
['/db_default/v4/nts/graph?plot.0=2.9.2&highlight_run=9']])
sparkline_test6_xml = \
get_sparkline(result_table_20120513, "test6", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
assert 2 == nr_sample_points, \
"Expected 2 sample points, found %d" % nr_sample_points
# Check that a different background color is used in the sparkline
# when the hash values recorded are different. At the same time,
# check that no background color is drawn on missing hash values,
# using a sequence of (hash1, no hash, hash2) over 3 consecutive
# days.
sparkline_hash1_xml = \
get_sparkline(result_table_20120513, "test_hash1", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash1_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color3 is not None
assert color1 != color3
assert color2 is None
# Check that the same background color is used in the sparkline
# when the hash values recorded are the same, using a
# (hash1, hash2, hash1) sequence.
sparkline_hash2_xml = \
get_sparkline(result_table_20120513, "test_hash2", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash2_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color1 == color3
assert color1 != color2
assert color2 is not None
# Check that we don't crash if a single run produces multiple
# samples with different hash values for the same run. This could
# happen e.g. when the compiler under test doesn't produce
# object code deterministically.
sparkline_mhashonrun_xml = get_sparkline(result_table_20120513,
"test_mhash_on_run", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
assert 4 == nr_sample_points, \
"Expected 4 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color2 is None
assert color1 != color3
# Check some variations of the latest runs report work.
check_html(client, '/v4/nts/latest_runs_report')
check_redirect(client, '/db_default/submitRun',
'/db_default/v4/nts/submitRun')
check_html(client, '/db_default/v4/nts/submitRun')
check_html(client, '/v4/nts/global_status')
check_html(client, '/v4/nts/recent_activity')
# Now check the compile report
# Get the V4 overview page.
check_html(client, '/v4/compile/')
# Get a machine overview page.
check_html(client, '/v4/compile/machine/1')
check_html(client, '/v4/compile/machine/2')
check_code(client,
'/v4/compile/machine/2/latest',
expected_code=HTTP_REDIRECT)
# Don't crash when requesting non-existing data
check_code(client,
'/v4/compile/machine/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
# Check the compare machine form gives correct redirects.
resp = check_code(client,
'/v4/nts/machine/2/compare?compare_to_id=3',
expected_code=HTTP_REDIRECT)
assert resp.headers[
'Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4"
resp = check_code(client,
'/v4/nts/machine/3/compare?compare_to_id=2',
expected_code=HTTP_REDIRECT)
assert resp.headers[
'Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"
# Get the order summary page.
check_html(client, '/v4/compile/all_orders')
# Get an order page.
check_html(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_html(client, '/v4/compile/1')
check_html(client, '/v4/compile/2')
check_html(client, '/v4/compile/3')
check_html(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
check_html(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_html(client, '/v4/compile/graph?plot.3=2.3.9')
# Get the mean graph page.
check_html(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_html(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_html(client, '/v4/compile/daily_report/2014/6/4')
check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1',
'/v4/nts/regressions/1')
check_html(client, '/v4/nts/regressions/')
check_html(client, '/v4/nts/regressions/?machine_filter=machine2')
check_html(client, '/v4/nts/regressions/?machine_filter=machine0')
check_html(client, '/v4/nts/regressions/1')
check_json(client, '/v4/nts/regressions/1?json=True')
# Check 404 is issues for inexistent Code
check_code(client,
'v4/nts/profile/9999/9999',
expected_code=HTTP_NOT_FOUND)
# Profile Viewer Ajax functions
# Check profiles page is responsive with expected IDs
check_code(client, 'v4/nts/profile/10/10')
# Check ajax call
functions = check_json(
client, 'v4/nts/profile/ajax/getFunctions?runid=10&testid=10')
number_of_functions = len(functions)
first_function_name = functions[0][0]
assert 1 == number_of_functions
assert "fn1" == first_function_name
top_level_counters = check_json(
client, 'v4/nts/profile/ajax/getTopLevelCounters?runids=10&testid=10')
assert "cycles" in top_level_counters
assert "branch-misses" in top_level_counters
code_for_fn = check_json(
client,
'v4/nts/profile/ajax/getCodeForFunction?runid=10&testid=10&f=fn1')
lines_in_function = len(code_for_fn)
assert 2 == lines_in_function
# Make sure the new option does not break anything
check_html(
client,
'/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update'
)
check_json(
client,
'/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update'
)
check_html(client,
'/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
check_json(
client,
'/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
app.testing = False
error_page = check_html(client, '/explode', expected_code=500)
assert "integer division or modulo by zero" in error_page.data
error_page = check_html(client, '/gone', expected_code=404)
assert "test" in error_page.data
check_html(client, '/sleep?timeout=0', expected_code=200)
check_html(client, '/db_default/summary_report')
check_html(client, '/rules')
check_html(client, '/log')
resp = check_code(client, '/__health')
assert resp.data == "Ok"
resp = check_code(client, '/ping')
assert resp.data == "pong"
# Check we can convert a sample into a graph page.
graph_to_sample = check_code(
client,
'/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar',
expected_code=HTTP_REDIRECT)
assert graph_to_sample.headers[
'Location'] == "http://localhost/db_default/v4/nts/graph?foo=bar&plot.0=2.6.0"
# Check that is we ask for a sample or invalid field, we explode with 400s.
check_code(
client,
'/db_default/v4/nts/graph_for_sample/10000/compile_time?foo=bar',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/db_default/v4/nts/graph_for_sample/10/not_a_metric?foo=bar',
expected_code=HTTP_BAD_REQUEST)
# check get_geomean_comparison_result with empty unchanged_tests
check_html(client, '/v4/nts/11')
check_html(client, '/v4/nts/12')
def main():
_,instance_path = sys.argv
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_code(client, '/')
# Get the V4 overview page.
check_code(client, '/v4/nts/')
# Get a machine overview page.
check_code(client, '/v4/nts/machine/1')
# Get a machine overview page in JSON format.
check_code(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_code(client, '/v4/nts/all_orders')
# Get an order page.
check_code(client, '/v4/nts/order/3')
# Get a run result page (and associated views).
check_code(client, '/v4/nts/1')
check_code(client, '/v4/nts/1?json=true')
check_code(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Get a graph page. This has been changed to redirect.
check_code(client, '/v4/nts/1/graph?test.87=2', expected_code=302)
# Get the new graph page.
check_code(client, '/v4/nts/graph?plot.0=1.87.2')
# Get the mean graph page.
check_code(client, '/v4/nts/graph?mean=1.2')
# Check some variations of the daily report work.
check_code(client, '/v4/nts/daily_report/2012/4/12')
check_code(client, '/v4/nts/daily_report/2012/4/11')
check_code(client, '/v4/nts/daily_report/2012/4/13')
check_code(client, '/v4/nts/daily_report/2012/4/10')
check_code(client, '/v4/nts/daily_report/2012/4/14')
# Now check the compile report
# Get the V4 overview page.
check_code(client, '/v4/compile/')
# Get a machine overview page.
check_code(client, '/v4/compile/machine/1')
check_code(client, '/v4/compile/machine/2')
# Get the order summary page.
check_code(client, '/v4/compile/all_orders')
# Get an order page.
check_code(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_code(client, '/v4/compile/1')
check_code(client, '/v4/compile/2')
check_code(client, '/v4/compile/3')
check_code(client, '/v4/compile/4')
check_code(client, '/v4/compile/10', expected_code=404) # This page should not be there.
check_code(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_code(client, '/v4/compile/graph?plot.38=2.38.9')
# Get the mean graph page.
check_code(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_code(client, '/v4/compile/daily_report/2014/6/4')
def action_view_comparison(name, args):
"""view a report comparison using a temporary server"""
import lnt.server.instance
import lnt.server.ui.app
import lnt.server.db.migrate
parser = OptionParser("%s [options] <report A> <report B>" % (name,))
parser.add_option("", "--hostname", dest="hostname", type=str,
help="host interface to use [%default]",
default='localhost')
parser.add_option("", "--port", dest="port", type=int, metavar="N",
help="local port to use [%default]", default=8000)
parser.add_option("", "--dry-run", dest="dry_run",
help="Do a dry run through the comparison. [%default]"
" [%default]", action="store_true", default=False)
(opts, args) = parser.parse_args(args)
if len(args) != 2:
parser.error("invalid number of arguments")
report_a_path, report_b_path = args
# Set up the default logger.
logger = logging.getLogger("lnt")
logger.setLevel(logging.ERROR)
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'))
logger.addHandler(handler)
# Create a temporary directory to hold the instance.
tmpdir = tempfile.mkdtemp(suffix='lnt')
try:
# Create a temporary instance.
url = 'http://%s:%d' % (opts.hostname, opts.port)
db_path = os.path.join(tmpdir, 'data.db')
db_info = lnt.server.config.DBInfo(
'sqlite:///%s' % (db_path,), '0.4', None,
lnt.server.config.EmailConfig(False, '', '', []), "0")
# _(self, name, zorgURL, dbDir, tempDir,
# profileDir, secretKey, databases, blacklist):
config = lnt.server.config.Config('LNT', url, db_path, tmpdir,
None, None, {'default': db_info},
None)
instance = lnt.server.instance.Instance(None, config)
# Create the database.
lnt.server.db.migrate.update_path(db_path)
# Import the two reports.
with contextlib.closing(config.get_database('default')) as db:
import_and_report(
config, 'default', db, report_a_path,
'<auto>', commit=True)
import_and_report(
config, 'default', db, report_b_path,
'<auto>', commit=True)
# Dispatch another thread to start the webbrowser.
comparison_url = '%s/v4/nts/2?compare_to=1' % (url,)
note("opening comparison view: %s" % (comparison_url,))
if not opts.dry_run:
thread.start_new_thread(start_browser, (comparison_url, True))
# Run the webserver.
app = lnt.server.ui.app.App.create_with_instance(instance)
app.debug = True
if opts.dry_run:
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
response = client.get(comparison_url)
assert response.status_code == 200, "Page did not return 200."
else:
app.run(opts.hostname, opts.port, use_reloader=False)
finally:
shutil.rmtree(tmpdir)
def main():
_, instance_path = sys.argv
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
app.config['WTF_CSRF_ENABLED'] = False
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_code(client, '/')
# Rules the index page.
check_code(client, '/rules')
# Get the V4 overview page.
check_code(client, '/v4/nts/')
# Get a machine overview page.
check_code(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
check_code(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_code(client, '/v4/nts/all_orders')
# Get an order page.
check_code(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
# Check that we can promote a baseline, then demote.
form_data = dict(name="foo_baseline",
description="foo_descrimport iption",
prmote=True)
r = client.post('/v4/nts/order/3', data=form_data)
# We should redirect to the last page and flash.
assert r.status_code == HTTP_REDIRECT
# Try with redirect.
r = client.post('/v4/nts/order/3',
data=form_data,
follow_redirects=True)
assert r.status_code == HTTP_OK
# Should see baseline displayed in page body.
assert "Baseline - foo_baseline" in r.data
# Now demote it.
data2 = dict(name="foo_baseline",
description="foo_description",
update=False,
promote=False,
demote=True)
r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
assert r.status_code == HTTP_OK
# Baseline should no longer be shown in page baseline.
assert "Baseline - foo_baseline" not in r.data
# Leave a baseline in place for the rest of the tests.
client.post('/v4/nts/order/3', data=form_data)
check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT)
with app.test_client() as c:
c.get('/v4/nts/set_baseline/1')
session.get('baseline-default-nts') == 1
# Get a run result page (and associated views).
check_code(client, '/v4/nts/1')
check_code(client, '/v4/nts/1?json=true')
check_code(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999?json=true',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/report',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/text_report',
expected_code=HTTP_NOT_FOUND)
# Get a graph page. This has been changed to redirect.
check_redirect(client, '/v4/nts/1/graph?test.3=2',
'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')
# Get the new graph page.
check_code(client, '/v4/nts/graph?plot.0=1.3.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
# Get the mean graph page.
check_code(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check baselines work.
check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
# Check some variations of the daily report work.
check_code(client, '/v4/nts/daily_report/2012/4/12')
check_code(client, '/v4/nts/daily_report/2012/4/11')
check_code(client, '/v4/nts/daily_report/2012/4/13')
check_code(client, '/v4/nts/daily_report/2012/4/10')
check_code(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
'/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
# Don't crash when using a parameter that happens to have the same name as
# a flask URL variable.
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/daily_report/1999/4/12')
check_code(client, '/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/daily_report/2012/13/12',
expected_code=HTTP_BAD_REQUEST)
check_code(client, '/v4/nts/daily_report/2012/4/32',
expected_code=HTTP_BAD_REQUEST)
# check ?filter-machine-regex= filter
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
# Don't crash on an invalid regular expression:
# FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
# ignoring the invalid regex.
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
# check that a regression seen between 2 consecutive runs that are
# more than a day apart gets reported
result_table_20120504 = get_results_table(
client, '/v4/nts/daily_report/2012/5/04', "execution_time")
check_table_content(result_table_20120504,
[["test1", ""],
["", "machine2", "1.0000", "-", "900.00%", ""],
["test2", ""],
["", "machine2", "FAIL", "-", "PASS", ""]])
check_body_nr_tests_table(
client, '/v4/nts/daily_report/2012/5/04',
[['machine2', '2', '0', '1']])
# Check that a failing result does not show up in the spark line
# as a dot with value 0.
result_table_20120513 = get_results_table(
client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time")
check_table_content(result_table_20120513,
[["test6", ""],
["", "machine2", "1.0000", "FAIL", "PASS", ""],
["test_hash1", ""],
["", "machine2", "1.0000", '-', '20.00%', ""],
["test_hash2", ""],
["", "machine2", "1.0000", '-', '20.00%', ""],
["test_mhash_on_run", ""],
["", "machine2", "1.0000", '-', '20.00%', ""], ])
sparkline_test6_xml = \
get_sparkline(result_table_20120513, "test6", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
assert 2 == nr_sample_points, \
"Expected 2 sample points, found %d" % nr_sample_points
# Check that a different background color is used in the sparkline
# when the hash values recorded are different. At the same time,
# check that no background color is drawn on missing hash values,
# using a sequence of (hash1, no hash, hash2) over 3 consecutive
# days.
sparkline_hash1_xml = \
get_sparkline(result_table_20120513, "test_hash1", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash1_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color3 is not None
assert color1 != color3
assert color2 is None
# Check that the same background color is used in the sparkline
# when the hash values recorded are the same, using a
# (hash1, hash2, hash1) sequence.
sparkline_hash2_xml = \
get_sparkline(result_table_20120513, "test_hash2", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash2_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color1 == color3
assert color1 != color2
assert color2 is not None
# Check that we don't crash if a single run produces multiple
# samples with different hash values for the same run. This could
# happen e.g. when the compiler under test doesn't produce
# object code deterministically.
sparkline_mhashonrun_xml = get_sparkline(
result_table_20120513, "test_mhash_on_run", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
assert 4 == nr_sample_points, \
"Expected 4 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color2 is None
assert color1 != color3
# Now check the compile report
# Get the V4 overview page.
check_code(client, '/v4/compile/')
# Get a machine overview page.
check_code(client, '/v4/compile/machine/1')
check_code(client, '/v4/compile/machine/2')
check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT)
# Don't crash when requesting non-existing data
check_code(client, '/v4/compile/machine/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
# Check the compare machine form gives correct redirects.
resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT)
assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4"
resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT)
assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"
# Get the order summary page.
check_code(client, '/v4/compile/all_orders')
# Get an order page.
check_code(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_code(client, '/v4/compile/1')
check_code(client, '/v4/compile/2')
check_code(client, '/v4/compile/3')
check_code(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_code(client, '/v4/compile/graph?plot.3=2.3.9')
# Get the mean graph page.
check_code(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_code(client, '/v4/compile/daily_report/2014/6/4')
check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
check_code(client, '/v4/nts/regressions/')
check_code(client, '/v4/nts/regressions/?machine_filter=machine2')
check_code(client, '/v4/nts/regressions/?machine_filter=machine0')
check_code(client, '/v4/nts/regressions/1')
check_json(client, '/v4/nts/regressions/1?json=True')
# Make sure the new option does not break anything
check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
def main():
_,instance_path = sys.argv
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_code(client, '/')
# Get the V4 overview page.
check_code(client, '/v4/nts/')
# Get a machine overview page.
check_code(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
check_code(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_code(client, '/v4/nts/all_orders')
# Get an order page.
check_code(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
# Get a run result page (and associated views).
check_code(client, '/v4/nts/1')
check_code(client, '/v4/nts/1?json=true')
check_code(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999?json=true',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/report',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/text_report',
expected_code=HTTP_NOT_FOUND)
# Get a graph page. This has been changed to redirect.
check_redirect(client, '/v4/nts/1/graph?test.87=2',
'v4/nts/graph\?plot\.0=1\.87\.2&highlight_run=1$')
# Get the new graph page.
check_code(client, '/v4/nts/graph?plot.0=1.87.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?plot.9999=1.87.2')
check_code(client, '/v4/nts/graph?plot.0=9999.87.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.87.9999',
expected_code=HTTP_NOT_FOUND)
# Get the mean graph page.
check_code(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check some variations of the daily report work.
check_code(client, '/v4/nts/daily_report/2012/4/12')
check_code(client, '/v4/nts/daily_report/2012/4/11')
check_code(client, '/v4/nts/daily_report/2012/4/13')
check_code(client, '/v4/nts/daily_report/2012/4/10')
check_code(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
'/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
# Don't crash when using a parameter that happens to have the same name as
# a flask URL variable.
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/daily_report/1999/4/12')
check_code(client, '/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/daily_report/2012/13/12',
expected_code=HTTP_BAD_REQUEST)
check_code(client, '/v4/nts/daily_report/2012/4/32',
expected_code=HTTP_BAD_REQUEST)
# check ?filter-machine-regex= filter
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
# Don't crash on an invalid regular expression:
# FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
# ignoring the invalid regex.
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
# check that a regression seen between 2 consecutive runs that are
# more than a day apart gets reported
check_body_result_table(client, '/v4/nts/daily_report/2012/5/04',
"execution_time",
[["test1", ""],
["", "machine2", "1.0000", "-", "900.00%", ""],
["test2", ""],
["", "machine2", "FAIL", "-", "PASS", ""]])
# Now check the compile report
# Get the V4 overview page.
check_code(client, '/v4/compile/')
# Get a machine overview page.
check_code(client, '/v4/compile/machine/1')
check_code(client, '/v4/compile/machine/2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/compile/machine/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
# Get the order summary page.
check_code(client, '/v4/compile/all_orders')
# Get an order page.
check_code(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_code(client, '/v4/compile/1')
check_code(client, '/v4/compile/2')
check_code(client, '/v4/compile/3')
check_code(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_code(client, '/v4/compile/graph?plot.38=2.38.9')
# Get the mean graph page.
check_code(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_code(client, '/v4/compile/daily_report/2014/6/4')
def main():
_, instance_path = sys.argv
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_code(client, '/')
# Rules the index page.
check_code(client, '/rules')
# Get the V4 overview page.
check_code(client, '/v4/nts/')
# Get a machine overview page.
check_code(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
check_code(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_code(client, '/v4/nts/all_orders')
# Get an order page.
check_code(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
# Get a run result page (and associated views).
check_code(client, '/v4/nts/1')
check_code(client, '/v4/nts/1?json=true')
check_code(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999?json=true',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/report',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/text_report',
expected_code=HTTP_NOT_FOUND)
# Get a graph page. This has been changed to redirect.
check_redirect(client, '/v4/nts/1/graph?test.3=2',
'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')
# Get the new graph page.
check_code(client, '/v4/nts/graph?plot.0=1.3.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
# Get the mean graph page.
check_code(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check baselines work.
check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
# Check some variations of the daily report work.
check_code(client, '/v4/nts/daily_report/2012/4/12')
check_code(client, '/v4/nts/daily_report/2012/4/11')
check_code(client, '/v4/nts/daily_report/2012/4/13')
check_code(client, '/v4/nts/daily_report/2012/4/10')
check_code(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
'/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
# Don't crash when using a parameter that happens to have the same name as
# a flask URL variable.
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/daily_report/1999/4/12')
check_code(client, '/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/daily_report/2012/13/12',
expected_code=HTTP_BAD_REQUEST)
check_code(client, '/v4/nts/daily_report/2012/4/32',
expected_code=HTTP_BAD_REQUEST)
# check ?filter-machine-regex= filter
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
# Don't crash on an invalid regular expression:
# FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
# ignoring the invalid regex.
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
# check that a regression seen between 2 consecutive runs that are
# more than a day apart gets reported
result_table_20120504 = get_results_table(
client, '/v4/nts/daily_report/2012/5/04', "execution_time")
check_table_content(result_table_20120504,
[["test1", ""],
["", "machine2", "1.0000", "-", "900.00%", ""],
["test2", ""],
["", "machine2", "FAIL", "-", "PASS", ""]])
check_body_nr_tests_table(
client, '/v4/nts/daily_report/2012/5/04',
[['machine2', '2', '0', '1']])
# Check that a failing result does not show up in the spark line
# as a dot with value 0.
result_table_20120513 = get_results_table(
client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time")
check_table_content(result_table_20120513,
[["test6", ""],
["", "machine2", "1.0000", "FAIL", "PASS", ""],
["test_hash1", ""],
["", "machine2", "1.0000", '-', '20.00%', ""],
["test_hash2", ""],
["", "machine2", "1.0000", '-', '20.00%', ""],
["test_mhash_on_run", ""],
["", "machine2", "1.0000", '-', '20.00%', ""], ])
sparkline_test6_xml = \
get_sparkline(result_table_20120513, "test6", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
assert 2 == nr_sample_points, \
"Expected 2 sample points, found %d" % nr_sample_points
# Check that a different background color is used in the sparkline
# when the hash values recorded are different. At the same time,
# check that no background color is drawn on missing hash values,
# using a sequence of (hash1, no hash, hash2) over 3 consecutive
# days.
sparkline_hash1_xml = \
get_sparkline(result_table_20120513, "test_hash1", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash1_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color3 is not None
assert color1 != color3
assert color2 is None
# Check that the same background color is used in the sparkline
# when the hash values recorded are the same, using a
# (hash1, hash2, hash1) sequence.
sparkline_hash2_xml = \
get_sparkline(result_table_20120513, "test_hash2", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash2_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color1 == color3
assert color1 != color2
assert color2 is not None
# Check that we don't crash if a single run produces multiple
# samples with different hash values for the same run. This could
# happen e.g. when the compiler under test doesn't produce
# object code deterministically.
sparkline_mhashonrun_xml = get_sparkline(
result_table_20120513, "test_mhash_on_run", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
assert 4 == nr_sample_points, \
"Expected 4 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color2 is None
assert color1 != color3
# Now check the compile report
# Get the V4 overview page.
check_code(client, '/v4/compile/')
# Get a machine overview page.
check_code(client, '/v4/compile/machine/1')
check_code(client, '/v4/compile/machine/2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/compile/machine/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
# Get the order summary page.
check_code(client, '/v4/compile/all_orders')
# Get an order page.
check_code(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_code(client, '/v4/compile/1')
check_code(client, '/v4/compile/2')
check_code(client, '/v4/compile/3')
check_code(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_code(client, '/v4/compile/graph?plot.3=2.3.9')
# Get the mean graph page.
check_code(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_code(client, '/v4/compile/daily_report/2014/6/4')
check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
check_code(client, '/v4/nts/regressions/')
check_code(client, '/v4/nts/regressions/1')
check_json(client, '/v4/nts/regressions/1?json=True')
def action_view_comparison(report_a, report_b, hostname, port, dry_run,
testsuite):
"""view a report comparison using a temporary server"""
from .common import init_logger
from lnt.util import logger
from lnt.util.ImportData import import_and_report
import contextlib
import lnt.server.db.migrate
import lnt.server.instance
import lnt.server.ui.app
import logging
import os
import shutil
import tempfile
import thread
init_logger(logging.ERROR)
# Create a temporary directory to hold the instance.
tmpdir = tempfile.mkdtemp(suffix='lnt')
try:
# Create a temporary instance.
url = 'http://%s:%d' % (hostname, port)
db_path = os.path.join(tmpdir, 'data.db')
db_info = lnt.server.config.DBInfo(
'sqlite:///%s' % (db_path,), None,
lnt.server.config.EmailConfig(False, '', '', []), "0")
# _(self, name, zorgURL, dbDir, tempDir,
# profileDir, secretKey, databases, blacklist):
config = lnt.server.config.Config('LNT', url, db_path, tmpdir,
None, "Not secret key.",
{'default': db_info}, None,
None)
instance = lnt.server.instance.Instance(None, config)
# Create the database.
lnt.server.db.migrate.update_path(db_path)
# Import the two reports.
with contextlib.closing(config.get_database('default')) as db:
session = db.make_session()
import_and_report(config, 'default', db, session, report_a,
'<auto>', testsuite, select_machine='match')
import_and_report(config, 'default', db, session, report_b,
'<auto>', testsuite, select_machine='match')
# Dispatch another thread to start the webbrowser.
comparison_url = '%s/v4/nts/2?compare_to=1' % (url,)
logger.info("opening comparison view: %s" % (comparison_url,))
if not dry_run:
thread.start_new_thread(_start_browser, (comparison_url, True))
# Run the webserver.
app = lnt.server.ui.app.App.create_with_instance(instance)
app.debug = True
if dry_run:
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
response = client.get(comparison_url)
assert response.status_code == 200, "Page did not return 200."
else:
app.run(hostname, port, use_reloader=False)
finally:
shutil.rmtree(tmpdir)
def main():
_, instance_path = sys.argv
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_code(client, '/')
# Rules the index page.
check_code(client, '/rules')
# Get the V4 overview page.
check_code(client, '/v4/nts/')
# Get a machine overview page.
check_code(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
check_code(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_code(client, '/v4/nts/all_orders')
# Get an order page.
check_code(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
# Get a run result page (and associated views).
check_code(client, '/v4/nts/1')
check_code(client, '/v4/nts/1?json=true')
check_code(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999?json=true',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/report',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/text_report',
expected_code=HTTP_NOT_FOUND)
# Get a graph page. This has been changed to redirect.
check_redirect(client, '/v4/nts/1/graph?test.3=2',
'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')
# Get the new graph page.
check_code(client, '/v4/nts/graph?plot.0=1.3.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
# Get the mean graph page.
check_code(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check some variations of the daily report work.
check_code(client, '/v4/nts/daily_report/2012/4/12')
check_code(client, '/v4/nts/daily_report/2012/4/11')
check_code(client, '/v4/nts/daily_report/2012/4/13')
check_code(client, '/v4/nts/daily_report/2012/4/10')
check_code(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
'/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
# Don't crash when using a parameter that happens to have the same name as
# a flask URL variable.
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/daily_report/1999/4/12')
check_code(client, '/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/daily_report/2012/13/12',
expected_code=HTTP_BAD_REQUEST)
check_code(client, '/v4/nts/daily_report/2012/4/32',
expected_code=HTTP_BAD_REQUEST)
# check ?filter-machine-regex= filter
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
# Don't crash on an invalid regular expression:
# FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
# ignoring the invalid regex.
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
# check that a regression seen between 2 consecutive runs that are
# more than a day apart gets reported
check_body_result_table(client, '/v4/nts/daily_report/2012/5/04',
"execution_time",
[["test1", ""],
["", "machine2", "1.0000", "-", "900.00%", ""],
["test2", ""],
["", "machine2", "FAIL", "-", "PASS", ""]])
# Check that a failing result does not show up in the spark line
# as a dot with value 0.
check_body_result_table(client,
'/v4/nts/daily_report/2012/5/13?num_days=3',
"execution_time",
[["test6", ""],
["", "machine2", "1.0000", "FAIL", "PASS", ""]])
sparkline_xml = get_sparkline(client,
'/v4/nts/daily_report/2012/5/13?num_days=3',
"execution_time", "test6", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_xml))
assert 2 == nr_sample_points, \
"Expected 2 sample points, found %d" % nr_sample_points
# Now check the compile report
# Get the V4 overview page.
check_code(client, '/v4/compile/')
# Get a machine overview page.
check_code(client, '/v4/compile/machine/1')
check_code(client, '/v4/compile/machine/2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/compile/machine/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
# Get the order summary page.
check_code(client, '/v4/compile/all_orders')
# Get an order page.
check_code(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_code(client, '/v4/compile/1')
check_code(client, '/v4/compile/2')
check_code(client, '/v4/compile/3')
check_code(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_code(client, '/v4/compile/graph?plot.3=2.3.9')
# Get the mean graph page.
check_code(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_code(client, '/v4/compile/daily_report/2014/6/4')
check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
check_code(client, '/v4/nts/regressions/')
check_code(client, '/v4/nts/regressions/1')
def main():
_, instance_path = sys.argv
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_code(client, '/')
# Rules the index page.
check_code(client, '/rules')
# Get the V4 overview page.
check_code(client, '/v4/nts/')
# Get a machine overview page.
check_code(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
check_code(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_code(client, '/v4/nts/all_orders')
# Get an order page.
check_code(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
# Get a run result page (and associated views).
check_code(client, '/v4/nts/1')
check_code(client, '/v4/nts/1?json=true')
check_code(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/9999/text_report',
expected_code=HTTP_NOT_FOUND)
# Get a graph page. This has been changed to redirect.
check_redirect(client, '/v4/nts/1/graph?test.3=2',
'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')
# Get the new graph page.
check_code(client, '/v4/nts/graph?plot.0=1.3.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
check_code(client,
'/v4/nts/graph?plot.0=9999.3.2',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/graph?plot.0=1.3.9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
# Get the mean graph page.
check_code(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client,
'/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check baselines work.
check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
# Check some variations of the daily report work.
check_code(client, '/v4/nts/daily_report/2012/4/12')
check_code(client, '/v4/nts/daily_report/2012/4/11')
check_code(client, '/v4/nts/daily_report/2012/4/13')
check_code(client, '/v4/nts/daily_report/2012/4/10')
check_code(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
'/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
# Don't crash when using a parameter that happens to have the same name as
# a flask URL variable.
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/daily_report/1999/4/12')
check_code(client,
'/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client,
'/v4/nts/daily_report/2012/13/12',
expected_code=HTTP_BAD_REQUEST)
check_code(client,
'/v4/nts/daily_report/2012/4/32',
expected_code=HTTP_BAD_REQUEST)
# check ?filter-machine-regex= filter
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2',
1)
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine',
2)
check_nr_machines_reported(
client,
'/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
# Don't crash on an invalid regular expression:
# FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
# ignoring the invalid regex.
check_nr_machines_reported(
client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
# check that a regression seen between 2 consecutive runs that are
# more than a day apart gets reported
result_table_20120504 = get_results_table(
client, '/v4/nts/daily_report/2012/5/04', "execution_time")
check_table_content(
result_table_20120504,
[["test1", ""], ["", "machine2", "1.0000", "-", "900.00%", ""],
["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]])
check_body_nr_tests_table(client, '/v4/nts/daily_report/2012/5/04',
[['machine2', '2', '0', '1']])
# Check that a failing result does not show up in the spark line
# as a dot with value 0.
result_table_20120513 = get_results_table(
client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time")
check_table_content(result_table_20120513, [
["test6", ""],
["", "machine2", "1.0000", "FAIL", "PASS", ""],
["test_hash1", ""],
["", "machine2", "1.0000", '-', '20.00%', ""],
["test_hash2", ""],
["", "machine2", "1.0000", '-', '20.00%', ""],
["test_mhash_on_run", ""],
["", "machine2", "1.0000", '-', '20.00%', ""],
])
sparkline_test6_xml = \
get_sparkline(result_table_20120513, "test6", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
assert 2 == nr_sample_points, \
"Expected 2 sample points, found %d" % nr_sample_points
# Check that a different background color is used in the sparkline
# when the hash values recorded are different. At the same time,
# check that no background color is drawn on missing hash values,
# using a sequence of (hash1, no hash, hash2) over 3 consecutive
# days.
sparkline_hash1_xml = \
get_sparkline(result_table_20120513, "test_hash1", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash1_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color3 is not None
assert color1 != color3
assert color2 is None
# Check that the same background color is used in the sparkline
# when the hash values recorded are the same, using a
# (hash1, hash2, hash1) sequence.
sparkline_hash2_xml = \
get_sparkline(result_table_20120513, "test_hash2", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash2_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color1 == color3
assert color1 != color2
assert color2 is not None
# Check that we don't crash if a single run produces multiple
# samples with different hash values for the same run. This could
# happen e.g. when the compiler under test doesn't produce
# object code deterministically.
sparkline_mhashonrun_xml = get_sparkline(result_table_20120513,
"test_mhash_on_run", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
assert 4 == nr_sample_points, \
"Expected 4 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color2 is None
assert color1 != color3
# Now check the compile report
# Get the V4 overview page.
check_code(client, '/v4/compile/')
# Get a machine overview page.
check_code(client, '/v4/compile/machine/1')
check_code(client, '/v4/compile/machine/2')
# Don't crash when requesting non-existing data
check_code(client,
'/v4/compile/machine/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
# Get the order summary page.
check_code(client, '/v4/compile/all_orders')
# Get an order page.
check_code(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_code(client, '/v4/compile/1')
check_code(client, '/v4/compile/2')
check_code(client, '/v4/compile/3')
check_code(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_code(client, '/v4/compile/graph?plot.3=2.3.9')
# Get the mean graph page.
check_code(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_code(client, '/v4/compile/daily_report/2014/6/4')
check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1',
'/v4/nts/regressions/1')
check_code(client, '/v4/nts/regressions/')
check_code(client, '/v4/nts/regressions/1')
check_json(client, '/v4/nts/regressions/1?json=True')
def action_view_comparison(report_a, report_b, hostname, port, dry_run,
testsuite):
"""view a report comparison using a temporary server"""
from .common import init_logger
from lnt.util import logger
from lnt.util.ImportData import import_and_report
import contextlib
import lnt.server.db.migrate
import lnt.server.instance
import lnt.server.ui.app
import logging
import os
import shutil
import sys
import tempfile
import thread
import time
import urllib
import webbrowser
init_logger(logging.ERROR)
# Create a temporary directory to hold the instance.
tmpdir = tempfile.mkdtemp(suffix='lnt')
try:
# Create a temporary instance.
url = 'http://%s:%d' % (hostname, port)
db_path = os.path.join(tmpdir, 'data.db')
db_info = lnt.server.config.DBInfo(
'sqlite:///%s' % (db_path, ), None,
lnt.server.config.EmailConfig(False, '', '', []), "0")
# _(self, name, zorgURL, dbDir, tempDir,
# profileDir, secretKey, databases, blacklist):
config = lnt.server.config.Config('LNT', url, db_path, tmpdir, None,
"Not secret key.",
{'default': db_info}, None, None)
instance = lnt.server.instance.Instance(None, config)
# Create the database.
lnt.server.db.migrate.update_path(db_path)
# Import the two reports.
with contextlib.closing(config.get_database('default')) as db:
session = db.make_session()
r = import_and_report(config,
'default',
db,
session,
report_a,
'<auto>',
testsuite,
select_machine='match')
import_and_report(config,
'default',
db,
session,
report_b,
'<auto>',
testsuite,
select_machine='match')
# Dispatch another thread to start the webbrowser.
comparison_url = '%s/v4/nts/2?compare_to=1' % (url, )
logger.info("opening comparison view: %s" % (comparison_url, ))
if not dry_run:
thread.start_new_thread(_start_browser, (comparison_url, True))
# Run the webserver.
app = lnt.server.ui.app.App.create_with_instance(instance)
app.debug = True
if dry_run:
# Don't catch out exceptions.
app.testing = True
# Create a test client.
client = app.test_client()
response = client.get(comparison_url)
assert response.status_code == 200, "Page did not return 200."
else:
app.run(hostname, port, use_reloader=False)
finally:
shutil.rmtree(tmpdir)
def main():
instance_path = sys.argv[1]
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
# Don't catch out exceptions.
app.testing = True
app.config['WTF_CSRF_ENABLED'] = False
# Create a test client.
client = app.test_client()
# Fetch the index page.
check_html(client, '/')
# Get the V4 overview page.
check_html(client, '/v4/nts/')
# Get a machine overview page.
check_html(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
check_json(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
check_html(client, '/v4/nts/all_orders')
# Get an order page.
check_html(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
# Check that we can promote a baseline, then demote.
form_data = dict(name="foo_baseline",
description="foo_descrimport iption",
prmote=True)
r = client.post('/v4/nts/order/3', data=form_data)
# We should redirect to the last page and flash.
assert r.status_code == HTTP_REDIRECT
# Try with redirect.
r = client.post('/v4/nts/order/3',
data=form_data,
follow_redirects=True)
assert r.status_code == HTTP_OK
# Should see baseline displayed in page body.
assert "Baseline - foo_baseline" in r.data
# Now demote it.
data2 = dict(name="foo_baseline",
description="foo_description",
update=False,
promote=False,
demote=True)
r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
assert r.status_code == HTTP_OK
# Baseline should no longer be shown in page baseline.
assert "Baseline - foo_baseline" not in r.data
# Leave a baseline in place for the rest of the tests.
client.post('/v4/nts/order/3', data=form_data)
check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT)
with app.test_client() as c:
c.get('/v4/nts/set_baseline/1')
session.get('baseline-default-nts') == 1
# Get a run result page (and associated views).
check_html(client, '/v4/nts/1')
check_json(client, '/v4/nts/1?json=true')
check_html(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999?json=true',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/report',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/9999/text_report',
expected_code=HTTP_NOT_FOUND)
# Get a graph page. This has been changed to redirect.
check_redirect(client, '/v4/nts/1/graph?test.3=2',
'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')
# Get a run that contains generic producer information
check_producer_label(client, '/v4/nts/7',
['Current', '152293', '2012-05-10T16:28:23',
'0:00:35', 'Producer'])
check_producer_label(client, '/v4/nts/8',
['Current', '152294', '2012-05-11T16:28:23',
'0:00:35', 'Producer'])
# Get a run that contains Buildbot producer information
check_producer_label(client, '/v4/nts/7',
['Previous', '152292', '2012-05-01T16:28:23',
'0:00:35', 'some-builder #987'])
check_producer_label(client, '/v4/nts/9',
['Current', '152295', '2012-05-12T16:28:23',
'0:00:35', 'some-builder #999'])
# Get the new graph page.
check_html(client, '/v4/nts/graph?plot.0=1.3.2')
# Don't crash when requesting non-existing data
check_html(client, '/v4/nts/graph?plot.9999=1.3.2')
check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
expected_code=HTTP_NOT_FOUND)
check_json(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
# Get the mean graph page.
check_html(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check baselines work.
check_html(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
# Check some variations of the daily report work.
check_html(client, '/v4/nts/daily_report/2012/4/12')
check_html(client, '/v4/nts/daily_report/2012/4/11')
check_html(client, '/v4/nts/daily_report/2012/4/13')
check_html(client, '/v4/nts/daily_report/2012/4/10')
check_html(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
'/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
# Don't crash when using a parameter that happens to have the same name as
# a flask URL variable.
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
check_html(client, '/v4/nts/daily_report/1999/4/12')
check_code(client, '/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/daily_report/2012/13/12',
expected_code=HTTP_BAD_REQUEST)
check_code(client, '/v4/nts/daily_report/2012/4/32',
expected_code=HTTP_BAD_REQUEST)
# check ?filter-machine-regex= filter
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
# Don't crash on an invalid regular expression:
# FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
# ignoring the invalid regex.
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
# check that a regression seen between 2 consecutive runs that are
# more than a day apart gets reported
result_table_20120504 = get_results_table(
client, '/v4/nts/daily_report/2012/5/04', "Execution Time")
check_table_content(result_table_20120504,
[["test1", ""],
["", "machine2", "1.000", "-", "900.00%", ""],
["test2", ""],
["", "machine2", "FAIL", "-", "PASS", ""]])
check_table_links(result_table_20120504,
[[],
["/db_default/v4/nts/graph?plot.0=2.4.2&highlight_run=6"],
[],
["/db_default/v4/nts/graph?plot.0=2.5.2&highlight_run=6"]])
check_body_nr_tests_table(
client, '/v4/nts/daily_report/2012/5/04',
[['machine2', '2', '0', '1']])
# Check that a failing result does not show up in the spark line
# as a dot with value 0.
result_table_20120513 = get_results_table(
client, '/v4/nts/daily_report/2012/5/13?num_days=3', "Execution Time")
check_table_content(result_table_20120513,
[["test6", ""],
["", "machine2", "1.000", "FAIL", "PASS", ""],
["test_hash1", ""],
["", "machine2", "1.000", '-', '20.00%', ""],
["test_hash2", ""],
["", "machine2", "1.000", '-', '20.00%', ""],
["test_mhash_on_run", ""],
["", "machine2", "1.000", '-', '20.00%', ""], ])
check_table_links(result_table_20120513,
[[],
['/db_default/v4/nts/graph?plot.0=2.6.2&highlight_run=9'],
[],
['/db_default/v4/nts/graph?plot.0=2.7.2&highlight_run=9'],
[],
['/db_default/v4/nts/graph?plot.0=2.8.2&highlight_run=9'],
[],
['/db_default/v4/nts/graph?plot.0=2.9.2&highlight_run=9']])
sparkline_test6_xml = \
get_sparkline(result_table_20120513, "test6", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
assert 2 == nr_sample_points, \
"Expected 2 sample points, found %d" % nr_sample_points
# Check that a different background color is used in the sparkline
# when the hash values recorded are different. At the same time,
# check that no background color is drawn on missing hash values,
# using a sequence of (hash1, no hash, hash2) over 3 consecutive
# days.
sparkline_hash1_xml = \
get_sparkline(result_table_20120513, "test_hash1", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash1_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color3 is not None
assert color1 != color3
assert color2 is None
# Check that the same background color is used in the sparkline
# when the hash values recorded are the same, using a
# (hash1, hash2, hash1) sequence.
sparkline_hash2_xml = \
get_sparkline(result_table_20120513, "test_hash2", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
assert 3 == nr_sample_points, \
"Expected 3 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_hash2_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color1 is not None
assert color1 == color3
assert color1 != color2
assert color2 is not None
# Check that we don't crash if a single run produces multiple
# samples with different hash values for the same run. This could
# happen e.g. when the compiler under test doesn't produce
# object code deterministically.
sparkline_mhashonrun_xml = get_sparkline(
result_table_20120513, "test_mhash_on_run", "machine2")
nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
assert 4 == nr_sample_points, \
"Expected 4 sample points, found %d" % nr_sample_points
background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
assert len(background_colors) == 3
color1, color2, color3 = background_colors
assert color2 is None
assert color1 != color3
# Check some variations of the latest runs report work.
check_html(client, '/v4/nts/latest_runs_report')
check_redirect(client, '/db_default/submitRun',
'/db_default/v4/nts/submitRun')
check_html(client, '/db_default/v4/nts/submitRun')
check_html(client, '/v4/nts/global_status')
check_html(client, '/v4/nts/recent_activity')
# Now check the compile report
# Get the V4 overview page.
check_html(client, '/v4/compile/')
# Get a machine overview page.
check_html(client, '/v4/compile/machine/1')
check_html(client, '/v4/compile/machine/2')
check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT)
# Don't crash when requesting non-existing data
check_code(client, '/v4/compile/machine/9999',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
# Check the compare machine form gives correct redirects.
resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT)
assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4"
resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT)
assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"
# Get the order summary page.
check_html(client, '/v4/compile/all_orders')
# Get an order page.
check_html(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
check_html(client, '/v4/compile/1')
check_html(client, '/v4/compile/2')
check_html(client, '/v4/compile/3')
check_html(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
check_html(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
check_html(client, '/v4/compile/graph?plot.3=2.3.9')
# Get the mean graph page.
check_html(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
check_html(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
check_html(client, '/v4/compile/daily_report/2014/6/4')
check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
check_html(client, '/v4/nts/regressions/')
check_html(client, '/v4/nts/regressions/?machine_filter=machine2')
check_html(client, '/v4/nts/regressions/?machine_filter=machine0')
check_html(client, '/v4/nts/regressions/1')
check_json(client, '/v4/nts/regressions/1?json=True')
# Check 404 is issues for inexistent Code
check_code(client, 'v4/nts/profile/9999/9999', expected_code=HTTP_NOT_FOUND)
# Profile Viewer Ajax functions
# Check profiles page is responsive with expected IDs
check_code(client, 'v4/nts/profile/10/10')
# Check ajax call
functions = check_json(client, 'v4/nts/profile/ajax/getFunctions?runid=10&testid=10')
number_of_functions = len(functions)
first_function_name = functions[0][0]
assert 1 == number_of_functions
assert "fn1" == first_function_name
top_level_counters = check_json(client, 'v4/nts/profile/ajax/getTopLevelCounters?runids=10&testid=10')
assert "cycles" in top_level_counters
assert "branch-misses" in top_level_counters
code_for_fn = check_json(client, 'v4/nts/profile/ajax/getCodeForFunction?runid=10&testid=10&f=fn1')
lines_in_function = len(code_for_fn)
assert 2 == lines_in_function
# Make sure the new option does not break anything
check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
app.testing = False
error_page = check_html(client, '/explode', expected_code=500)
assert "integer division or modulo by zero" in error_page.data
error_page = check_html(client, '/gone', expected_code=404)
assert "test" in error_page.data
check_html(client, '/sleep?timeout=0', expected_code=200)
check_html(client, '/db_default/summary_report')
check_html(client, '/rules')
check_html(client, '/log')
resp = check_code(client, '/__health')
assert resp.data == "Ok"
resp = check_code(client, '/ping')
assert resp.data == "pong"
# Check we can convert a sample into a graph page.
graph_to_sample = check_code(client, '/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar',
expected_code=HTTP_REDIRECT)
assert graph_to_sample.headers['Location'] == "http://localhost/db_default/v4/nts/graph?foo=bar&plot.0=2.6.0"
# Check that is we ask for a sample or invalid field, we explode with 400s.
check_code(client, '/db_default/v4/nts/graph_for_sample/10000/compile_time?foo=bar',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/db_default/v4/nts/graph_for_sample/10/not_a_metric?foo=bar',
expected_code=HTTP_BAD_REQUEST)
# check get_geomean_comparison_result with empty unchanged_tests
check_html(client, '/v4/nts/11')
check_html(client, '/v4/nts/12')