def setUp(self): """Bind to the LNT test instance.""" _, instance_path, shared_inputs = sys.argv app = lnt.server.ui.app.App.create_standalone(instance_path) app.testing = True self.client = app.test_client() self.shared_inputs = shared_inputs
def sanity_check_instance(instance_path): # Create an application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Create a test client. client = app.test_client() # Fetch the index page. index = client.get('/') # Visit all the test suites. test_suite_link_rex = re.compile(""" <a href="(.*)">(.*)</a><br>""") test_suite_list_start = index.data.index("<h3>Test Suites</h3>") test_suite_list_end = index.data.index("</div>", test_suite_list_start) for ln in index.data[test_suite_list_start:test_suite_list_end].split("\n"): # Ignore non-matching lines. print >>sys.stderr,ln m = test_suite_link_rex.match(ln) if not m: continue # We found a test suite link... link,name = m.groups() logging.info("visiting test suite %r", name) # Get the test suite overview page. overview = client.get(os.path.join("/", link)) assert "LNT : %s - Recent Activity" % (name,) in overview.data
def setUp(self): """Bind to the LNT test instance.""" _, instance_path = sys.argv self.instance_path = instance_path app = lnt.server.ui.app.App.create_standalone(instance_path) app.testing = True self.client = app.test_client()
def setUp(self): """Bind to the LNT test instance.""" instance_path = sys.argv[1] app = lnt.server.ui.app.App.create_standalone(instance_path) app.testing = True app.config['WTF_CSRF_ENABLED'] = False self.client = app.test_client()
def sanity_check_instance(instance_path): # Create an application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Create a test client. client = app.test_client() # Fetch the index page. index = client.get('/') # Visit all the test suites. test_suite_link_rex = re.compile(""" <a href="(.*)">(.*)</a><br>""") data = index.get_data(as_text=True) test_suite_list_start = data.index("<h3>Test Suites</h3>") test_suite_list_end = data.index("</div>", test_suite_list_start) for ln in data[test_suite_list_start:test_suite_list_end].split("\n"): # Ignore non-matching lines. print(ln, file=sys.stderr) m = test_suite_link_rex.match(ln) if not m: continue # We found a test suite link... link, name = m.groups() logging.info("visiting test suite %r", name) # Get the test suite overview page. overview = client.get(os.path.join("/", link)) assert "LNT : %s - Recent Activity" % (name, ) in overview.data
def main(): instance_path = sys.argv[1] # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True app.config['WTF_CSRF_ENABLED'] = False # Create a test client. client = app.test_client() # Fetch the index page. check_html(client, '/') # Get the V4 overview page. check_html(client, '/v4/nts/') # Get a machine overview page. check_html(client, '/v4/nts/machine/1') # Check invalid machine gives error. check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND) # Get a machine overview page in JSON format. check_json(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_html(client, '/v4/nts/all_orders') # Get an order page. check_html(client, '/v4/nts/order/3') # Check invalid order gives error. check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND) # Check that we can promote a baseline, then demote. form_data = dict(name="foo_baseline", description="foo_descrimport iption", prmote=True) r = client.post('/v4/nts/order/3', data=form_data) # We should redirect to the last page and flash. assert r.status_code == HTTP_REDIRECT # Try with redirect. r = client.post('/v4/nts/order/3', data=form_data, follow_redirects=True) assert r.status_code == HTTP_OK # Should see baseline displayed in page body. assert "Baseline - foo_baseline" in r.data # Now demote it. data2 = dict(name="foo_baseline", description="foo_description", update=False, promote=False, demote=True) r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True) assert r.status_code == HTTP_OK # Baseline should no longer be shown in page baseline. assert "Baseline - foo_baseline" not in r.data # Leave a baseline in place for the rest of the tests. client.post('/v4/nts/order/3', data=form_data) check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT) with app.test_client() as c: c.get('/v4/nts/set_baseline/1') session.get('baseline-default-nts') == 1 # Get a run result page (and associated views). check_html(client, '/v4/nts/1') check_json(client, '/v4/nts/1?json=true') check_html(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Check invalid run numbers give errors. check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/text_report', expected_code=HTTP_NOT_FOUND) # Get a graph page. This has been changed to redirect. check_redirect(client, '/v4/nts/1/graph?test.3=2', 'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$') # Get a run that contains generic producer information check_producer_label( client, '/v4/nts/7', ['Current', '152293', '2012-05-10T16:28:23', '0:00:35', 'Producer']) check_producer_label( client, '/v4/nts/8', ['Current', '152294', '2012-05-11T16:28:23', '0:00:35', 'Producer']) # Get a run that contains Buildbot producer information check_producer_label(client, '/v4/nts/7', [ 'Previous', '152292', '2012-05-01T16:28:23', '0:00:35', 'some-builder #987' ]) check_producer_label(client, '/v4/nts/9', [ 'Current', '152295', '2012-05-12T16:28:23', '0:00:35', 'some-builder #999' ]) # Get the new graph page. check_html(client, '/v4/nts/graph?plot.0=1.3.2') # Don't crash when requesting non-existing data check_html(client, '/v4/nts/graph?plot.9999=1.3.2') check_code(client, '/v4/nts/graph?plot.0=9999.3.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.3.9999', expected_code=HTTP_NOT_FOUND) check_json(client, '/v4/nts/graph?plot.9999=1.3.2&json=True') # Get the mean graph page. check_html(client, '/v4/nts/graph?mean=1.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?mean=9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?mean=1.9999', expected_code=HTTP_NOT_FOUND) # Check baselines work. check_html(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3') # Check some variations of the daily report work. check_html(client, '/v4/nts/daily_report/2012/4/12') check_html(client, '/v4/nts/daily_report/2012/4/11') check_html(client, '/v4/nts/daily_report/2012/4/13') check_html(client, '/v4/nts/daily_report/2012/4/10') check_html(client, '/v4/nts/daily_report/2012/4/14') check_redirect(client, '/v4/nts/daily_report', '/v4/nts/daily_report/\d+/\d+/\d+$') check_redirect(client, '/v4/nts/daily_report?num_days=7', '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$') # Don't crash when using a parameter that happens to have the same name as # a flask URL variable. check_redirect(client, '/v4/nts/daily_report?day=15', '/v4/nts/daily_report/\d+/\d+/\d+$') # Don't crash when requesting non-existing data check_html(client, '/v4/nts/daily_report/1999/4/12') check_code(client, '/v4/nts/daily_report/-1/4/12', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/daily_report/2012/13/12', expected_code=HTTP_BAD_REQUEST) check_code(client, '/v4/nts/daily_report/2012/4/32', expected_code=HTTP_BAD_REQUEST) # check ?filter-machine-regex= filter check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0) # Don't crash on an invalid regular expression: # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently # ignoring the invalid regex. check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3) # check that a regression seen between 2 consecutive runs that are # more than a day apart gets reported result_table_20120504 = get_results_table( client, '/v4/nts/daily_report/2012/5/04', "Execution Time") check_table_content( result_table_20120504, [["test1", ""], ["", "machine2", "1.000", "-", "900.00%", ""], ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]]) check_table_links( result_table_20120504, [[], ["/db_default/v4/nts/graph?plot.0=2.4.2&highlight_run=6"], [], ["/db_default/v4/nts/graph?plot.0=2.5.2&highlight_run=6"]]) check_body_nr_tests_table(client, '/v4/nts/daily_report/2012/5/04', [['machine2', '2', '0', '1']]) # Check that a failing result does not show up in the spark line # as a dot with value 0. result_table_20120513 = get_results_table( client, '/v4/nts/daily_report/2012/5/13?num_days=3', "Execution Time") check_table_content(result_table_20120513, [ ["test6", ""], ["", "machine2", "1.000", "FAIL", "PASS", ""], ["test_hash1", ""], ["", "machine2", "1.000", '-', '20.00%', ""], ["test_hash2", ""], ["", "machine2", "1.000", '-', '20.00%', ""], ["test_mhash_on_run", ""], ["", "machine2", "1.000", '-', '20.00%', ""], ]) check_table_links( result_table_20120513, [[], ['/db_default/v4/nts/graph?plot.0=2.6.2&highlight_run=9'], [], ['/db_default/v4/nts/graph?plot.0=2.7.2&highlight_run=9'], [], ['/db_default/v4/nts/graph?plot.0=2.8.2&highlight_run=9'], [], ['/db_default/v4/nts/graph?plot.0=2.9.2&highlight_run=9']]) sparkline_test6_xml = \ get_sparkline(result_table_20120513, "test6", "machine2") nr_sample_points = len(extract_sample_points(sparkline_test6_xml)) assert 2 == nr_sample_points, \ "Expected 2 sample points, found %d" % nr_sample_points # Check that a different background color is used in the sparkline # when the hash values recorded are different. At the same time, # check that no background color is drawn on missing hash values, # using a sequence of (hash1, no hash, hash2) over 3 consecutive # days. sparkline_hash1_xml = \ get_sparkline(result_table_20120513, "test_hash1", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash1_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash1_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color3 is not None assert color1 != color3 assert color2 is None # Check that the same background color is used in the sparkline # when the hash values recorded are the same, using a # (hash1, hash2, hash1) sequence. sparkline_hash2_xml = \ get_sparkline(result_table_20120513, "test_hash2", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash2_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash2_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color1 == color3 assert color1 != color2 assert color2 is not None # Check that we don't crash if a single run produces multiple # samples with different hash values for the same run. This could # happen e.g. when the compiler under test doesn't produce # object code deterministically. sparkline_mhashonrun_xml = get_sparkline(result_table_20120513, "test_mhash_on_run", "machine2") nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml)) assert 4 == nr_sample_points, \ "Expected 4 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color2 is None assert color1 != color3 # Check some variations of the latest runs report work. check_html(client, '/v4/nts/latest_runs_report') check_redirect(client, '/db_default/submitRun', '/db_default/v4/nts/submitRun') check_html(client, '/db_default/v4/nts/submitRun') check_html(client, '/v4/nts/global_status') check_html(client, '/v4/nts/recent_activity') # Now check the compile report # Get the V4 overview page. check_html(client, '/v4/compile/') # Get a machine overview page. check_html(client, '/v4/compile/machine/1') check_html(client, '/v4/compile/machine/2') check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT) # Don't crash when requesting non-existing data check_code(client, '/v4/compile/machine/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND) # Check the compare machine form gives correct redirects. resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT) assert resp.headers[ 'Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4" resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT) assert resp.headers[ 'Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9" # Get the order summary page. check_html(client, '/v4/compile/all_orders') # Get an order page. check_html(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_html(client, '/v4/compile/1') check_html(client, '/v4/compile/2') check_html(client, '/v4/compile/3') check_html(client, '/v4/compile/4') check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND) check_html(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_html(client, '/v4/compile/graph?plot.3=2.3.9') # Get the mean graph page. check_html(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_html(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_html(client, '/v4/compile/daily_report/2014/6/4') check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1') check_html(client, '/v4/nts/regressions/') check_html(client, '/v4/nts/regressions/?machine_filter=machine2') check_html(client, '/v4/nts/regressions/?machine_filter=machine0') check_html(client, '/v4/nts/regressions/1') check_json(client, '/v4/nts/regressions/1?json=True') # Check 404 is issues for inexistent Code check_code(client, 'v4/nts/profile/9999/9999', expected_code=HTTP_NOT_FOUND) # Profile Viewer Ajax functions # Check profiles page is responsive with expected IDs check_code(client, 'v4/nts/profile/10/10') # Check ajax call functions = check_json( client, 'v4/nts/profile/ajax/getFunctions?runid=10&testid=10') number_of_functions = len(functions) first_function_name = functions[0][0] assert 1 == number_of_functions assert "fn1" == first_function_name top_level_counters = check_json( client, 'v4/nts/profile/ajax/getTopLevelCounters?runids=10&testid=10') assert "cycles" in top_level_counters assert "branch-misses" in top_level_counters code_for_fn = check_json( client, 'v4/nts/profile/ajax/getCodeForFunction?runid=10&testid=10&f=fn1') lines_in_function = len(code_for_fn) assert 2 == lines_in_function # Make sure the new option does not break anything check_html( client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update' ) check_json( client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update' ) check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2') check_json( client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true') app.testing = False error_page = check_html(client, '/explode', expected_code=500) assert "integer division or modulo by zero" in error_page.data error_page = check_html(client, '/gone', expected_code=404) assert "test" in error_page.data check_html(client, '/sleep?timeout=0', expected_code=200) check_html(client, '/db_default/summary_report') check_html(client, '/rules') check_html(client, '/log') resp = check_code(client, '/__health') assert resp.data == "Ok" resp = check_code(client, '/ping') assert resp.data == "pong" # Check we can convert a sample into a graph page. graph_to_sample = check_code( client, '/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar', expected_code=HTTP_REDIRECT) assert graph_to_sample.headers[ 'Location'] == "http://localhost/db_default/v4/nts/graph?foo=bar&plot.0=2.6.0" # Check that is we ask for a sample or invalid field, we explode with 400s. check_code( client, '/db_default/v4/nts/graph_for_sample/10000/compile_time?foo=bar', expected_code=HTTP_NOT_FOUND) check_code(client, '/db_default/v4/nts/graph_for_sample/10/not_a_metric?foo=bar', expected_code=HTTP_BAD_REQUEST) # check get_geomean_comparison_result with empty unchanged_tests check_html(client, '/v4/nts/11') check_html(client, '/v4/nts/12')
def main(): _,instance_path = sys.argv # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() # Fetch the index page. check_code(client, '/') # Get the V4 overview page. check_code(client, '/v4/nts/') # Get a machine overview page. check_code(client, '/v4/nts/machine/1') # Get a machine overview page in JSON format. check_code(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_code(client, '/v4/nts/all_orders') # Get an order page. check_code(client, '/v4/nts/order/3') # Get a run result page (and associated views). check_code(client, '/v4/nts/1') check_code(client, '/v4/nts/1?json=true') check_code(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Get a graph page. This has been changed to redirect. check_code(client, '/v4/nts/1/graph?test.87=2', expected_code=302) # Get the new graph page. check_code(client, '/v4/nts/graph?plot.0=1.87.2') # Get the mean graph page. check_code(client, '/v4/nts/graph?mean=1.2') # Check some variations of the daily report work. check_code(client, '/v4/nts/daily_report/2012/4/12') check_code(client, '/v4/nts/daily_report/2012/4/11') check_code(client, '/v4/nts/daily_report/2012/4/13') check_code(client, '/v4/nts/daily_report/2012/4/10') check_code(client, '/v4/nts/daily_report/2012/4/14') # Now check the compile report # Get the V4 overview page. check_code(client, '/v4/compile/') # Get a machine overview page. check_code(client, '/v4/compile/machine/1') check_code(client, '/v4/compile/machine/2') # Get the order summary page. check_code(client, '/v4/compile/all_orders') # Get an order page. check_code(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_code(client, '/v4/compile/1') check_code(client, '/v4/compile/2') check_code(client, '/v4/compile/3') check_code(client, '/v4/compile/4') check_code(client, '/v4/compile/10', expected_code=404) # This page should not be there. check_code(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_code(client, '/v4/compile/graph?plot.38=2.38.9') # Get the mean graph page. check_code(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_code(client, '/v4/compile/daily_report/2014/6/4')
def action_view_comparison(name, args): """view a report comparison using a temporary server""" import lnt.server.instance import lnt.server.ui.app import lnt.server.db.migrate parser = OptionParser("%s [options] <report A> <report B>" % (name,)) parser.add_option("", "--hostname", dest="hostname", type=str, help="host interface to use [%default]", default='localhost') parser.add_option("", "--port", dest="port", type=int, metavar="N", help="local port to use [%default]", default=8000) parser.add_option("", "--dry-run", dest="dry_run", help="Do a dry run through the comparison. [%default]" " [%default]", action="store_true", default=False) (opts, args) = parser.parse_args(args) if len(args) != 2: parser.error("invalid number of arguments") report_a_path, report_b_path = args # Set up the default logger. logger = logging.getLogger("lnt") logger.setLevel(logging.ERROR) handler = logging.StreamHandler(sys.stderr) handler.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) # Create a temporary directory to hold the instance. tmpdir = tempfile.mkdtemp(suffix='lnt') try: # Create a temporary instance. url = 'http://%s:%d' % (opts.hostname, opts.port) db_path = os.path.join(tmpdir, 'data.db') db_info = lnt.server.config.DBInfo( 'sqlite:///%s' % (db_path,), '0.4', None, lnt.server.config.EmailConfig(False, '', '', []), "0") # _(self, name, zorgURL, dbDir, tempDir, # profileDir, secretKey, databases, blacklist): config = lnt.server.config.Config('LNT', url, db_path, tmpdir, None, None, {'default': db_info}, None) instance = lnt.server.instance.Instance(None, config) # Create the database. lnt.server.db.migrate.update_path(db_path) # Import the two reports. with contextlib.closing(config.get_database('default')) as db: import_and_report( config, 'default', db, report_a_path, '<auto>', commit=True) import_and_report( config, 'default', db, report_b_path, '<auto>', commit=True) # Dispatch another thread to start the webbrowser. comparison_url = '%s/v4/nts/2?compare_to=1' % (url,) note("opening comparison view: %s" % (comparison_url,)) if not opts.dry_run: thread.start_new_thread(start_browser, (comparison_url, True)) # Run the webserver. app = lnt.server.ui.app.App.create_with_instance(instance) app.debug = True if opts.dry_run: # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() response = client.get(comparison_url) assert response.status_code == 200, "Page did not return 200." else: app.run(opts.hostname, opts.port, use_reloader=False) finally: shutil.rmtree(tmpdir)
def main(): _, instance_path = sys.argv # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True app.config['WTF_CSRF_ENABLED'] = False # Create a test client. client = app.test_client() # Fetch the index page. check_code(client, '/') # Rules the index page. check_code(client, '/rules') # Get the V4 overview page. check_code(client, '/v4/nts/') # Get a machine overview page. check_code(client, '/v4/nts/machine/1') # Check invalid machine gives error. check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND) # Get a machine overview page in JSON format. check_code(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_code(client, '/v4/nts/all_orders') # Get an order page. check_code(client, '/v4/nts/order/3') # Check invalid order gives error. check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND) # Check that we can promote a baseline, then demote. form_data = dict(name="foo_baseline", description="foo_descrimport iption", prmote=True) r = client.post('/v4/nts/order/3', data=form_data) # We should redirect to the last page and flash. assert r.status_code == HTTP_REDIRECT # Try with redirect. r = client.post('/v4/nts/order/3', data=form_data, follow_redirects=True) assert r.status_code == HTTP_OK # Should see baseline displayed in page body. assert "Baseline - foo_baseline" in r.data # Now demote it. data2 = dict(name="foo_baseline", description="foo_description", update=False, promote=False, demote=True) r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True) assert r.status_code == HTTP_OK # Baseline should no longer be shown in page baseline. assert "Baseline - foo_baseline" not in r.data # Leave a baseline in place for the rest of the tests. client.post('/v4/nts/order/3', data=form_data) check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT) with app.test_client() as c: c.get('/v4/nts/set_baseline/1') session.get('baseline-default-nts') == 1 # Get a run result page (and associated views). check_code(client, '/v4/nts/1') check_code(client, '/v4/nts/1?json=true') check_code(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Check invalid run numbers give errors. check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/text_report', expected_code=HTTP_NOT_FOUND) # Get a graph page. This has been changed to redirect. check_redirect(client, '/v4/nts/1/graph?test.3=2', 'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$') # Get the new graph page. check_code(client, '/v4/nts/graph?plot.0=1.3.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?plot.9999=1.3.2') check_code(client, '/v4/nts/graph?plot.0=9999.3.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.3.9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True') # Get the mean graph page. check_code(client, '/v4/nts/graph?mean=1.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?mean=9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?mean=1.9999', expected_code=HTTP_NOT_FOUND) # Check baselines work. check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3') # Check some variations of the daily report work. check_code(client, '/v4/nts/daily_report/2012/4/12') check_code(client, '/v4/nts/daily_report/2012/4/11') check_code(client, '/v4/nts/daily_report/2012/4/13') check_code(client, '/v4/nts/daily_report/2012/4/10') check_code(client, '/v4/nts/daily_report/2012/4/14') check_redirect(client, '/v4/nts/daily_report', '/v4/nts/daily_report/\d+/\d+/\d+$') check_redirect(client, '/v4/nts/daily_report?num_days=7', '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$') # Don't crash when using a parameter that happens to have the same name as # a flask URL variable. check_redirect(client, '/v4/nts/daily_report?day=15', '/v4/nts/daily_report/\d+/\d+/\d+$') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/daily_report/1999/4/12') check_code(client, '/v4/nts/daily_report/-1/4/12', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/daily_report/2012/13/12', expected_code=HTTP_BAD_REQUEST) check_code(client, '/v4/nts/daily_report/2012/4/32', expected_code=HTTP_BAD_REQUEST) # check ?filter-machine-regex= filter check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0) # Don't crash on an invalid regular expression: # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently # ignoring the invalid regex. check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3) # check that a regression seen between 2 consecutive runs that are # more than a day apart gets reported result_table_20120504 = get_results_table( client, '/v4/nts/daily_report/2012/5/04', "execution_time") check_table_content(result_table_20120504, [["test1", ""], ["", "machine2", "1.0000", "-", "900.00%", ""], ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]]) check_body_nr_tests_table( client, '/v4/nts/daily_report/2012/5/04', [['machine2', '2', '0', '1']]) # Check that a failing result does not show up in the spark line # as a dot with value 0. result_table_20120513 = get_results_table( client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time") check_table_content(result_table_20120513, [["test6", ""], ["", "machine2", "1.0000", "FAIL", "PASS", ""], ["test_hash1", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ["test_hash2", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ["test_mhash_on_run", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ]) sparkline_test6_xml = \ get_sparkline(result_table_20120513, "test6", "machine2") nr_sample_points = len(extract_sample_points(sparkline_test6_xml)) assert 2 == nr_sample_points, \ "Expected 2 sample points, found %d" % nr_sample_points # Check that a different background color is used in the sparkline # when the hash values recorded are different. At the same time, # check that no background color is drawn on missing hash values, # using a sequence of (hash1, no hash, hash2) over 3 consecutive # days. sparkline_hash1_xml = \ get_sparkline(result_table_20120513, "test_hash1", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash1_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash1_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color3 is not None assert color1 != color3 assert color2 is None # Check that the same background color is used in the sparkline # when the hash values recorded are the same, using a # (hash1, hash2, hash1) sequence. sparkline_hash2_xml = \ get_sparkline(result_table_20120513, "test_hash2", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash2_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash2_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color1 == color3 assert color1 != color2 assert color2 is not None # Check that we don't crash if a single run produces multiple # samples with different hash values for the same run. This could # happen e.g. when the compiler under test doesn't produce # object code deterministically. sparkline_mhashonrun_xml = get_sparkline( result_table_20120513, "test_mhash_on_run", "machine2") nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml)) assert 4 == nr_sample_points, \ "Expected 4 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color2 is None assert color1 != color3 # Now check the compile report # Get the V4 overview page. check_code(client, '/v4/compile/') # Get a machine overview page. check_code(client, '/v4/compile/machine/1') check_code(client, '/v4/compile/machine/2') check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT) # Don't crash when requesting non-existing data check_code(client, '/v4/compile/machine/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND) # Check the compare machine form gives correct redirects. resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT) assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4" resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT) assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9" # Get the order summary page. check_code(client, '/v4/compile/all_orders') # Get an order page. check_code(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_code(client, '/v4/compile/1') check_code(client, '/v4/compile/2') check_code(client, '/v4/compile/3') check_code(client, '/v4/compile/4') check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_code(client, '/v4/compile/graph?plot.3=2.3.9') # Get the mean graph page. check_code(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_code(client, '/v4/compile/daily_report/2014/6/4') check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1') check_code(client, '/v4/nts/regressions/') check_code(client, '/v4/nts/regressions/?machine_filter=machine2') check_code(client, '/v4/nts/regressions/?machine_filter=machine0') check_code(client, '/v4/nts/regressions/1') check_json(client, '/v4/nts/regressions/1?json=True') # Make sure the new option does not break anything check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update') check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update') check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2') check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
def main(): _,instance_path = sys.argv # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() # Fetch the index page. check_code(client, '/') # Get the V4 overview page. check_code(client, '/v4/nts/') # Get a machine overview page. check_code(client, '/v4/nts/machine/1') # Check invalid machine gives error. check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND) # Get a machine overview page in JSON format. check_code(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_code(client, '/v4/nts/all_orders') # Get an order page. check_code(client, '/v4/nts/order/3') # Check invalid order gives error. check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND) # Get a run result page (and associated views). check_code(client, '/v4/nts/1') check_code(client, '/v4/nts/1?json=true') check_code(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Check invalid run numbers give errors. check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/text_report', expected_code=HTTP_NOT_FOUND) # Get a graph page. This has been changed to redirect. check_redirect(client, '/v4/nts/1/graph?test.87=2', 'v4/nts/graph\?plot\.0=1\.87\.2&highlight_run=1$') # Get the new graph page. check_code(client, '/v4/nts/graph?plot.0=1.87.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?plot.9999=1.87.2') check_code(client, '/v4/nts/graph?plot.0=9999.87.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.87.9999', expected_code=HTTP_NOT_FOUND) # Get the mean graph page. check_code(client, '/v4/nts/graph?mean=1.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?mean=9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?mean=1.9999', expected_code=HTTP_NOT_FOUND) # Check some variations of the daily report work. check_code(client, '/v4/nts/daily_report/2012/4/12') check_code(client, '/v4/nts/daily_report/2012/4/11') check_code(client, '/v4/nts/daily_report/2012/4/13') check_code(client, '/v4/nts/daily_report/2012/4/10') check_code(client, '/v4/nts/daily_report/2012/4/14') check_redirect(client, '/v4/nts/daily_report', '/v4/nts/daily_report/\d+/\d+/\d+$') check_redirect(client, '/v4/nts/daily_report?num_days=7', '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$') # Don't crash when using a parameter that happens to have the same name as # a flask URL variable. check_redirect(client, '/v4/nts/daily_report?day=15', '/v4/nts/daily_report/\d+/\d+/\d+$') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/daily_report/1999/4/12') check_code(client, '/v4/nts/daily_report/-1/4/12', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/daily_report/2012/13/12', expected_code=HTTP_BAD_REQUEST) check_code(client, '/v4/nts/daily_report/2012/4/32', expected_code=HTTP_BAD_REQUEST) # check ?filter-machine-regex= filter check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0) # Don't crash on an invalid regular expression: # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently # ignoring the invalid regex. check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3) # check that a regression seen between 2 consecutive runs that are # more than a day apart gets reported check_body_result_table(client, '/v4/nts/daily_report/2012/5/04', "execution_time", [["test1", ""], ["", "machine2", "1.0000", "-", "900.00%", ""], ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]]) # Now check the compile report # Get the V4 overview page. check_code(client, '/v4/compile/') # Get a machine overview page. check_code(client, '/v4/compile/machine/1') check_code(client, '/v4/compile/machine/2') # Don't crash when requesting non-existing data check_code(client, '/v4/compile/machine/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND) # Get the order summary page. check_code(client, '/v4/compile/all_orders') # Get an order page. check_code(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_code(client, '/v4/compile/1') check_code(client, '/v4/compile/2') check_code(client, '/v4/compile/3') check_code(client, '/v4/compile/4') check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_code(client, '/v4/compile/graph?plot.38=2.38.9') # Get the mean graph page. check_code(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_code(client, '/v4/compile/daily_report/2014/6/4')
def main(): _, instance_path = sys.argv # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() # Fetch the index page. check_code(client, '/') # Rules the index page. check_code(client, '/rules') # Get the V4 overview page. check_code(client, '/v4/nts/') # Get a machine overview page. check_code(client, '/v4/nts/machine/1') # Check invalid machine gives error. check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND) # Get a machine overview page in JSON format. check_code(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_code(client, '/v4/nts/all_orders') # Get an order page. check_code(client, '/v4/nts/order/3') # Check invalid order gives error. check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND) # Get a run result page (and associated views). check_code(client, '/v4/nts/1') check_code(client, '/v4/nts/1?json=true') check_code(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Check invalid run numbers give errors. check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/text_report', expected_code=HTTP_NOT_FOUND) # Get a graph page. This has been changed to redirect. check_redirect(client, '/v4/nts/1/graph?test.3=2', 'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$') # Get the new graph page. check_code(client, '/v4/nts/graph?plot.0=1.3.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?plot.9999=1.3.2') check_code(client, '/v4/nts/graph?plot.0=9999.3.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.3.9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True') # Get the mean graph page. check_code(client, '/v4/nts/graph?mean=1.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?mean=9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?mean=1.9999', expected_code=HTTP_NOT_FOUND) # Check baselines work. check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3') # Check some variations of the daily report work. check_code(client, '/v4/nts/daily_report/2012/4/12') check_code(client, '/v4/nts/daily_report/2012/4/11') check_code(client, '/v4/nts/daily_report/2012/4/13') check_code(client, '/v4/nts/daily_report/2012/4/10') check_code(client, '/v4/nts/daily_report/2012/4/14') check_redirect(client, '/v4/nts/daily_report', '/v4/nts/daily_report/\d+/\d+/\d+$') check_redirect(client, '/v4/nts/daily_report?num_days=7', '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$') # Don't crash when using a parameter that happens to have the same name as # a flask URL variable. check_redirect(client, '/v4/nts/daily_report?day=15', '/v4/nts/daily_report/\d+/\d+/\d+$') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/daily_report/1999/4/12') check_code(client, '/v4/nts/daily_report/-1/4/12', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/daily_report/2012/13/12', expected_code=HTTP_BAD_REQUEST) check_code(client, '/v4/nts/daily_report/2012/4/32', expected_code=HTTP_BAD_REQUEST) # check ?filter-machine-regex= filter check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0) # Don't crash on an invalid regular expression: # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently # ignoring the invalid regex. check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3) # check that a regression seen between 2 consecutive runs that are # more than a day apart gets reported result_table_20120504 = get_results_table( client, '/v4/nts/daily_report/2012/5/04', "execution_time") check_table_content(result_table_20120504, [["test1", ""], ["", "machine2", "1.0000", "-", "900.00%", ""], ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]]) check_body_nr_tests_table( client, '/v4/nts/daily_report/2012/5/04', [['machine2', '2', '0', '1']]) # Check that a failing result does not show up in the spark line # as a dot with value 0. result_table_20120513 = get_results_table( client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time") check_table_content(result_table_20120513, [["test6", ""], ["", "machine2", "1.0000", "FAIL", "PASS", ""], ["test_hash1", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ["test_hash2", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ["test_mhash_on_run", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ]) sparkline_test6_xml = \ get_sparkline(result_table_20120513, "test6", "machine2") nr_sample_points = len(extract_sample_points(sparkline_test6_xml)) assert 2 == nr_sample_points, \ "Expected 2 sample points, found %d" % nr_sample_points # Check that a different background color is used in the sparkline # when the hash values recorded are different. At the same time, # check that no background color is drawn on missing hash values, # using a sequence of (hash1, no hash, hash2) over 3 consecutive # days. sparkline_hash1_xml = \ get_sparkline(result_table_20120513, "test_hash1", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash1_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash1_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color3 is not None assert color1 != color3 assert color2 is None # Check that the same background color is used in the sparkline # when the hash values recorded are the same, using a # (hash1, hash2, hash1) sequence. sparkline_hash2_xml = \ get_sparkline(result_table_20120513, "test_hash2", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash2_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash2_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color1 == color3 assert color1 != color2 assert color2 is not None # Check that we don't crash if a single run produces multiple # samples with different hash values for the same run. This could # happen e.g. when the compiler under test doesn't produce # object code deterministically. sparkline_mhashonrun_xml = get_sparkline( result_table_20120513, "test_mhash_on_run", "machine2") nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml)) assert 4 == nr_sample_points, \ "Expected 4 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color2 is None assert color1 != color3 # Now check the compile report # Get the V4 overview page. check_code(client, '/v4/compile/') # Get a machine overview page. check_code(client, '/v4/compile/machine/1') check_code(client, '/v4/compile/machine/2') # Don't crash when requesting non-existing data check_code(client, '/v4/compile/machine/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND) # Get the order summary page. check_code(client, '/v4/compile/all_orders') # Get an order page. check_code(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_code(client, '/v4/compile/1') check_code(client, '/v4/compile/2') check_code(client, '/v4/compile/3') check_code(client, '/v4/compile/4') check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_code(client, '/v4/compile/graph?plot.3=2.3.9') # Get the mean graph page. check_code(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_code(client, '/v4/compile/daily_report/2014/6/4') check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1') check_code(client, '/v4/nts/regressions/') check_code(client, '/v4/nts/regressions/1') check_json(client, '/v4/nts/regressions/1?json=True')
def action_view_comparison(report_a, report_b, hostname, port, dry_run, testsuite): """view a report comparison using a temporary server""" from .common import init_logger from lnt.util import logger from lnt.util.ImportData import import_and_report import contextlib import lnt.server.db.migrate import lnt.server.instance import lnt.server.ui.app import logging import os import shutil import tempfile import thread init_logger(logging.ERROR) # Create a temporary directory to hold the instance. tmpdir = tempfile.mkdtemp(suffix='lnt') try: # Create a temporary instance. url = 'http://%s:%d' % (hostname, port) db_path = os.path.join(tmpdir, 'data.db') db_info = lnt.server.config.DBInfo( 'sqlite:///%s' % (db_path,), None, lnt.server.config.EmailConfig(False, '', '', []), "0") # _(self, name, zorgURL, dbDir, tempDir, # profileDir, secretKey, databases, blacklist): config = lnt.server.config.Config('LNT', url, db_path, tmpdir, None, "Not secret key.", {'default': db_info}, None, None) instance = lnt.server.instance.Instance(None, config) # Create the database. lnt.server.db.migrate.update_path(db_path) # Import the two reports. with contextlib.closing(config.get_database('default')) as db: session = db.make_session() import_and_report(config, 'default', db, session, report_a, '<auto>', testsuite, select_machine='match') import_and_report(config, 'default', db, session, report_b, '<auto>', testsuite, select_machine='match') # Dispatch another thread to start the webbrowser. comparison_url = '%s/v4/nts/2?compare_to=1' % (url,) logger.info("opening comparison view: %s" % (comparison_url,)) if not dry_run: thread.start_new_thread(_start_browser, (comparison_url, True)) # Run the webserver. app = lnt.server.ui.app.App.create_with_instance(instance) app.debug = True if dry_run: # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() response = client.get(comparison_url) assert response.status_code == 200, "Page did not return 200." else: app.run(hostname, port, use_reloader=False) finally: shutil.rmtree(tmpdir)
def main(): _, instance_path = sys.argv # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() # Fetch the index page. check_code(client, '/') # Rules the index page. check_code(client, '/rules') # Get the V4 overview page. check_code(client, '/v4/nts/') # Get a machine overview page. check_code(client, '/v4/nts/machine/1') # Check invalid machine gives error. check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND) # Get a machine overview page in JSON format. check_code(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_code(client, '/v4/nts/all_orders') # Get an order page. check_code(client, '/v4/nts/order/3') # Check invalid order gives error. check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND) # Get a run result page (and associated views). check_code(client, '/v4/nts/1') check_code(client, '/v4/nts/1?json=true') check_code(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Check invalid run numbers give errors. check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/text_report', expected_code=HTTP_NOT_FOUND) # Get a graph page. This has been changed to redirect. check_redirect(client, '/v4/nts/1/graph?test.3=2', 'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$') # Get the new graph page. check_code(client, '/v4/nts/graph?plot.0=1.3.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?plot.9999=1.3.2') check_code(client, '/v4/nts/graph?plot.0=9999.3.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.3.9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True') # Get the mean graph page. check_code(client, '/v4/nts/graph?mean=1.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?mean=9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?mean=1.9999', expected_code=HTTP_NOT_FOUND) # Check some variations of the daily report work. check_code(client, '/v4/nts/daily_report/2012/4/12') check_code(client, '/v4/nts/daily_report/2012/4/11') check_code(client, '/v4/nts/daily_report/2012/4/13') check_code(client, '/v4/nts/daily_report/2012/4/10') check_code(client, '/v4/nts/daily_report/2012/4/14') check_redirect(client, '/v4/nts/daily_report', '/v4/nts/daily_report/\d+/\d+/\d+$') check_redirect(client, '/v4/nts/daily_report?num_days=7', '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$') # Don't crash when using a parameter that happens to have the same name as # a flask URL variable. check_redirect(client, '/v4/nts/daily_report?day=15', '/v4/nts/daily_report/\d+/\d+/\d+$') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/daily_report/1999/4/12') check_code(client, '/v4/nts/daily_report/-1/4/12', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/daily_report/2012/13/12', expected_code=HTTP_BAD_REQUEST) check_code(client, '/v4/nts/daily_report/2012/4/32', expected_code=HTTP_BAD_REQUEST) # check ?filter-machine-regex= filter check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0) # Don't crash on an invalid regular expression: # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently # ignoring the invalid regex. check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3) # check that a regression seen between 2 consecutive runs that are # more than a day apart gets reported check_body_result_table(client, '/v4/nts/daily_report/2012/5/04', "execution_time", [["test1", ""], ["", "machine2", "1.0000", "-", "900.00%", ""], ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]]) # Check that a failing result does not show up in the spark line # as a dot with value 0. check_body_result_table(client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time", [["test6", ""], ["", "machine2", "1.0000", "FAIL", "PASS", ""]]) sparkline_xml = get_sparkline(client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time", "test6", "machine2") nr_sample_points = len(extract_sample_points(sparkline_xml)) assert 2 == nr_sample_points, \ "Expected 2 sample points, found %d" % nr_sample_points # Now check the compile report # Get the V4 overview page. check_code(client, '/v4/compile/') # Get a machine overview page. check_code(client, '/v4/compile/machine/1') check_code(client, '/v4/compile/machine/2') # Don't crash when requesting non-existing data check_code(client, '/v4/compile/machine/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND) # Get the order summary page. check_code(client, '/v4/compile/all_orders') # Get an order page. check_code(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_code(client, '/v4/compile/1') check_code(client, '/v4/compile/2') check_code(client, '/v4/compile/3') check_code(client, '/v4/compile/4') check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_code(client, '/v4/compile/graph?plot.3=2.3.9') # Get the mean graph page. check_code(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_code(client, '/v4/compile/daily_report/2014/6/4') check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1') check_code(client, '/v4/nts/regressions/') check_code(client, '/v4/nts/regressions/1')
def main(): _, instance_path = sys.argv # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() # Fetch the index page. check_code(client, '/') # Rules the index page. check_code(client, '/rules') # Get the V4 overview page. check_code(client, '/v4/nts/') # Get a machine overview page. check_code(client, '/v4/nts/machine/1') # Check invalid machine gives error. check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND) # Get a machine overview page in JSON format. check_code(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_code(client, '/v4/nts/all_orders') # Get an order page. check_code(client, '/v4/nts/order/3') # Check invalid order gives error. check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND) # Get a run result page (and associated views). check_code(client, '/v4/nts/1') check_code(client, '/v4/nts/1?json=true') check_code(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Check invalid run numbers give errors. check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/text_report', expected_code=HTTP_NOT_FOUND) # Get a graph page. This has been changed to redirect. check_redirect(client, '/v4/nts/1/graph?test.3=2', 'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$') # Get the new graph page. check_code(client, '/v4/nts/graph?plot.0=1.3.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?plot.9999=1.3.2') check_code(client, '/v4/nts/graph?plot.0=9999.3.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.3.9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True') # Get the mean graph page. check_code(client, '/v4/nts/graph?mean=1.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?mean=9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?mean=1.9999', expected_code=HTTP_NOT_FOUND) # Check baselines work. check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3') # Check some variations of the daily report work. check_code(client, '/v4/nts/daily_report/2012/4/12') check_code(client, '/v4/nts/daily_report/2012/4/11') check_code(client, '/v4/nts/daily_report/2012/4/13') check_code(client, '/v4/nts/daily_report/2012/4/10') check_code(client, '/v4/nts/daily_report/2012/4/14') check_redirect(client, '/v4/nts/daily_report', '/v4/nts/daily_report/\d+/\d+/\d+$') check_redirect(client, '/v4/nts/daily_report?num_days=7', '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$') # Don't crash when using a parameter that happens to have the same name as # a flask URL variable. check_redirect(client, '/v4/nts/daily_report?day=15', '/v4/nts/daily_report/\d+/\d+/\d+$') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/daily_report/1999/4/12') check_code(client, '/v4/nts/daily_report/-1/4/12', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/daily_report/2012/13/12', expected_code=HTTP_BAD_REQUEST) check_code(client, '/v4/nts/daily_report/2012/4/32', expected_code=HTTP_BAD_REQUEST) # check ?filter-machine-regex= filter check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1) check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0) # Don't crash on an invalid regular expression: # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently # ignoring the invalid regex. check_nr_machines_reported( client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3) # check that a regression seen between 2 consecutive runs that are # more than a day apart gets reported result_table_20120504 = get_results_table( client, '/v4/nts/daily_report/2012/5/04', "execution_time") check_table_content( result_table_20120504, [["test1", ""], ["", "machine2", "1.0000", "-", "900.00%", ""], ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]]) check_body_nr_tests_table(client, '/v4/nts/daily_report/2012/5/04', [['machine2', '2', '0', '1']]) # Check that a failing result does not show up in the spark line # as a dot with value 0. result_table_20120513 = get_results_table( client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time") check_table_content(result_table_20120513, [ ["test6", ""], ["", "machine2", "1.0000", "FAIL", "PASS", ""], ["test_hash1", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ["test_hash2", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ["test_mhash_on_run", ""], ["", "machine2", "1.0000", '-', '20.00%', ""], ]) sparkline_test6_xml = \ get_sparkline(result_table_20120513, "test6", "machine2") nr_sample_points = len(extract_sample_points(sparkline_test6_xml)) assert 2 == nr_sample_points, \ "Expected 2 sample points, found %d" % nr_sample_points # Check that a different background color is used in the sparkline # when the hash values recorded are different. At the same time, # check that no background color is drawn on missing hash values, # using a sequence of (hash1, no hash, hash2) over 3 consecutive # days. sparkline_hash1_xml = \ get_sparkline(result_table_20120513, "test_hash1", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash1_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash1_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color3 is not None assert color1 != color3 assert color2 is None # Check that the same background color is used in the sparkline # when the hash values recorded are the same, using a # (hash1, hash2, hash1) sequence. sparkline_hash2_xml = \ get_sparkline(result_table_20120513, "test_hash2", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash2_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash2_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color1 == color3 assert color1 != color2 assert color2 is not None # Check that we don't crash if a single run produces multiple # samples with different hash values for the same run. This could # happen e.g. when the compiler under test doesn't produce # object code deterministically. sparkline_mhashonrun_xml = get_sparkline(result_table_20120513, "test_mhash_on_run", "machine2") nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml)) assert 4 == nr_sample_points, \ "Expected 4 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color2 is None assert color1 != color3 # Now check the compile report # Get the V4 overview page. check_code(client, '/v4/compile/') # Get a machine overview page. check_code(client, '/v4/compile/machine/1') check_code(client, '/v4/compile/machine/2') # Don't crash when requesting non-existing data check_code(client, '/v4/compile/machine/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND) # Get the order summary page. check_code(client, '/v4/compile/all_orders') # Get an order page. check_code(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_code(client, '/v4/compile/1') check_code(client, '/v4/compile/2') check_code(client, '/v4/compile/3') check_code(client, '/v4/compile/4') check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_code(client, '/v4/compile/graph?plot.3=2.3.9') # Get the mean graph page. check_code(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_code(client, '/v4/compile/daily_report/2014/6/4') check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1') check_code(client, '/v4/nts/regressions/') check_code(client, '/v4/nts/regressions/1') check_json(client, '/v4/nts/regressions/1?json=True')
def action_view_comparison(report_a, report_b, hostname, port, dry_run, testsuite): """view a report comparison using a temporary server""" from .common import init_logger from lnt.util import logger from lnt.util.ImportData import import_and_report import contextlib import lnt.server.db.migrate import lnt.server.instance import lnt.server.ui.app import logging import os import shutil import sys import tempfile import thread import time import urllib import webbrowser init_logger(logging.ERROR) # Create a temporary directory to hold the instance. tmpdir = tempfile.mkdtemp(suffix='lnt') try: # Create a temporary instance. url = 'http://%s:%d' % (hostname, port) db_path = os.path.join(tmpdir, 'data.db') db_info = lnt.server.config.DBInfo( 'sqlite:///%s' % (db_path, ), None, lnt.server.config.EmailConfig(False, '', '', []), "0") # _(self, name, zorgURL, dbDir, tempDir, # profileDir, secretKey, databases, blacklist): config = lnt.server.config.Config('LNT', url, db_path, tmpdir, None, "Not secret key.", {'default': db_info}, None, None) instance = lnt.server.instance.Instance(None, config) # Create the database. lnt.server.db.migrate.update_path(db_path) # Import the two reports. with contextlib.closing(config.get_database('default')) as db: session = db.make_session() r = import_and_report(config, 'default', db, session, report_a, '<auto>', testsuite, select_machine='match') import_and_report(config, 'default', db, session, report_b, '<auto>', testsuite, select_machine='match') # Dispatch another thread to start the webbrowser. comparison_url = '%s/v4/nts/2?compare_to=1' % (url, ) logger.info("opening comparison view: %s" % (comparison_url, )) if not dry_run: thread.start_new_thread(_start_browser, (comparison_url, True)) # Run the webserver. app = lnt.server.ui.app.App.create_with_instance(instance) app.debug = True if dry_run: # Don't catch out exceptions. app.testing = True # Create a test client. client = app.test_client() response = client.get(comparison_url) assert response.status_code == 200, "Page did not return 200." else: app.run(hostname, port, use_reloader=False) finally: shutil.rmtree(tmpdir)
def main(): instance_path = sys.argv[1] # Create the application instance. app = lnt.server.ui.app.App.create_standalone(instance_path) # Don't catch out exceptions. app.testing = True app.config['WTF_CSRF_ENABLED'] = False # Create a test client. client = app.test_client() # Fetch the index page. check_html(client, '/') # Get the V4 overview page. check_html(client, '/v4/nts/') # Get a machine overview page. check_html(client, '/v4/nts/machine/1') # Check invalid machine gives error. check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND) # Get a machine overview page in JSON format. check_json(client, '/v4/nts/machine/1?json=true') # Get the order summary page. check_html(client, '/v4/nts/all_orders') # Get an order page. check_html(client, '/v4/nts/order/3') # Check invalid order gives error. check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND) # Check that we can promote a baseline, then demote. form_data = dict(name="foo_baseline", description="foo_descrimport iption", prmote=True) r = client.post('/v4/nts/order/3', data=form_data) # We should redirect to the last page and flash. assert r.status_code == HTTP_REDIRECT # Try with redirect. r = client.post('/v4/nts/order/3', data=form_data, follow_redirects=True) assert r.status_code == HTTP_OK # Should see baseline displayed in page body. assert "Baseline - foo_baseline" in r.data # Now demote it. data2 = dict(name="foo_baseline", description="foo_description", update=False, promote=False, demote=True) r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True) assert r.status_code == HTTP_OK # Baseline should no longer be shown in page baseline. assert "Baseline - foo_baseline" not in r.data # Leave a baseline in place for the rest of the tests. client.post('/v4/nts/order/3', data=form_data) check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT) with app.test_client() as c: c.get('/v4/nts/set_baseline/1') session.get('baseline-default-nts') == 1 # Get a run result page (and associated views). check_html(client, '/v4/nts/1') check_json(client, '/v4/nts/1?json=true') check_html(client, '/v4/nts/1/report') check_code(client, '/v4/nts/1/text_report') # Check invalid run numbers give errors. check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/9999/text_report', expected_code=HTTP_NOT_FOUND) # Get a graph page. This has been changed to redirect. check_redirect(client, '/v4/nts/1/graph?test.3=2', 'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$') # Get a run that contains generic producer information check_producer_label(client, '/v4/nts/7', ['Current', '152293', '2012-05-10T16:28:23', '0:00:35', 'Producer']) check_producer_label(client, '/v4/nts/8', ['Current', '152294', '2012-05-11T16:28:23', '0:00:35', 'Producer']) # Get a run that contains Buildbot producer information check_producer_label(client, '/v4/nts/7', ['Previous', '152292', '2012-05-01T16:28:23', '0:00:35', 'some-builder #987']) check_producer_label(client, '/v4/nts/9', ['Current', '152295', '2012-05-12T16:28:23', '0:00:35', 'some-builder #999']) # Get the new graph page. check_html(client, '/v4/nts/graph?plot.0=1.3.2') # Don't crash when requesting non-existing data check_html(client, '/v4/nts/graph?plot.9999=1.3.2') check_code(client, '/v4/nts/graph?plot.0=9999.3.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?plot.0=1.3.9999', expected_code=HTTP_NOT_FOUND) check_json(client, '/v4/nts/graph?plot.9999=1.3.2&json=True') # Get the mean graph page. check_html(client, '/v4/nts/graph?mean=1.2') # Don't crash when requesting non-existing data check_code(client, '/v4/nts/graph?mean=9999.2', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/graph?mean=1.9999', expected_code=HTTP_NOT_FOUND) # Check baselines work. check_html(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3') # Check some variations of the daily report work. check_html(client, '/v4/nts/daily_report/2012/4/12') check_html(client, '/v4/nts/daily_report/2012/4/11') check_html(client, '/v4/nts/daily_report/2012/4/13') check_html(client, '/v4/nts/daily_report/2012/4/10') check_html(client, '/v4/nts/daily_report/2012/4/14') check_redirect(client, '/v4/nts/daily_report', '/v4/nts/daily_report/\d+/\d+/\d+$') check_redirect(client, '/v4/nts/daily_report?num_days=7', '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$') # Don't crash when using a parameter that happens to have the same name as # a flask URL variable. check_redirect(client, '/v4/nts/daily_report?day=15', '/v4/nts/daily_report/\d+/\d+/\d+$') # Don't crash when requesting non-existing data check_html(client, '/v4/nts/daily_report/1999/4/12') check_code(client, '/v4/nts/daily_report/-1/4/12', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/nts/daily_report/2012/13/12', expected_code=HTTP_BAD_REQUEST) check_code(client, '/v4/nts/daily_report/2012/4/32', expected_code=HTTP_BAD_REQUEST) # check ?filter-machine-regex= filter check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1) check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0) # Don't crash on an invalid regular expression: # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently # ignoring the invalid regex. check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3) # check that a regression seen between 2 consecutive runs that are # more than a day apart gets reported result_table_20120504 = get_results_table( client, '/v4/nts/daily_report/2012/5/04', "Execution Time") check_table_content(result_table_20120504, [["test1", ""], ["", "machine2", "1.000", "-", "900.00%", ""], ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]]) check_table_links(result_table_20120504, [[], ["/db_default/v4/nts/graph?plot.0=2.4.2&highlight_run=6"], [], ["/db_default/v4/nts/graph?plot.0=2.5.2&highlight_run=6"]]) check_body_nr_tests_table( client, '/v4/nts/daily_report/2012/5/04', [['machine2', '2', '0', '1']]) # Check that a failing result does not show up in the spark line # as a dot with value 0. result_table_20120513 = get_results_table( client, '/v4/nts/daily_report/2012/5/13?num_days=3', "Execution Time") check_table_content(result_table_20120513, [["test6", ""], ["", "machine2", "1.000", "FAIL", "PASS", ""], ["test_hash1", ""], ["", "machine2", "1.000", '-', '20.00%', ""], ["test_hash2", ""], ["", "machine2", "1.000", '-', '20.00%', ""], ["test_mhash_on_run", ""], ["", "machine2", "1.000", '-', '20.00%', ""], ]) check_table_links(result_table_20120513, [[], ['/db_default/v4/nts/graph?plot.0=2.6.2&highlight_run=9'], [], ['/db_default/v4/nts/graph?plot.0=2.7.2&highlight_run=9'], [], ['/db_default/v4/nts/graph?plot.0=2.8.2&highlight_run=9'], [], ['/db_default/v4/nts/graph?plot.0=2.9.2&highlight_run=9']]) sparkline_test6_xml = \ get_sparkline(result_table_20120513, "test6", "machine2") nr_sample_points = len(extract_sample_points(sparkline_test6_xml)) assert 2 == nr_sample_points, \ "Expected 2 sample points, found %d" % nr_sample_points # Check that a different background color is used in the sparkline # when the hash values recorded are different. At the same time, # check that no background color is drawn on missing hash values, # using a sequence of (hash1, no hash, hash2) over 3 consecutive # days. sparkline_hash1_xml = \ get_sparkline(result_table_20120513, "test_hash1", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash1_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash1_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color3 is not None assert color1 != color3 assert color2 is None # Check that the same background color is used in the sparkline # when the hash values recorded are the same, using a # (hash1, hash2, hash1) sequence. sparkline_hash2_xml = \ get_sparkline(result_table_20120513, "test_hash2", "machine2") nr_sample_points = len(extract_sample_points(sparkline_hash2_xml)) assert 3 == nr_sample_points, \ "Expected 3 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_hash2_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color1 is not None assert color1 == color3 assert color1 != color2 assert color2 is not None # Check that we don't crash if a single run produces multiple # samples with different hash values for the same run. This could # happen e.g. when the compiler under test doesn't produce # object code deterministically. sparkline_mhashonrun_xml = get_sparkline( result_table_20120513, "test_mhash_on_run", "machine2") nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml)) assert 4 == nr_sample_points, \ "Expected 4 sample points, found %d" % nr_sample_points background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3) assert len(background_colors) == 3 color1, color2, color3 = background_colors assert color2 is None assert color1 != color3 # Check some variations of the latest runs report work. check_html(client, '/v4/nts/latest_runs_report') check_redirect(client, '/db_default/submitRun', '/db_default/v4/nts/submitRun') check_html(client, '/db_default/v4/nts/submitRun') check_html(client, '/v4/nts/global_status') check_html(client, '/v4/nts/recent_activity') # Now check the compile report # Get the V4 overview page. check_html(client, '/v4/compile/') # Get a machine overview page. check_html(client, '/v4/compile/machine/1') check_html(client, '/v4/compile/machine/2') check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT) # Don't crash when requesting non-existing data check_code(client, '/v4/compile/machine/9999', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND) check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND) # Check the compare machine form gives correct redirects. resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT) assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4" resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT) assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9" # Get the order summary page. check_html(client, '/v4/compile/all_orders') # Get an order page. check_html(client, '/v4/compile/order/3') # Get a run result page (and associated views). check_html(client, '/v4/compile/1') check_html(client, '/v4/compile/2') check_html(client, '/v4/compile/3') check_html(client, '/v4/compile/4') check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND) check_html(client, '/v4/compile/1/report') check_code(client, '/v4/compile/1/text_report') # Get the new graph page. check_html(client, '/v4/compile/graph?plot.3=2.3.9') # Get the mean graph page. check_html(client, 'v4/compile/graph?mean=2.9') # Check some variations of the daily report work. check_html(client, '/v4/compile/daily_report/2014/6/5?day_start=16') check_html(client, '/v4/compile/daily_report/2014/6/4') check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1') check_html(client, '/v4/nts/regressions/') check_html(client, '/v4/nts/regressions/?machine_filter=machine2') check_html(client, '/v4/nts/regressions/?machine_filter=machine0') check_html(client, '/v4/nts/regressions/1') check_json(client, '/v4/nts/regressions/1?json=True') # Check 404 is issues for inexistent Code check_code(client, 'v4/nts/profile/9999/9999', expected_code=HTTP_NOT_FOUND) # Profile Viewer Ajax functions # Check profiles page is responsive with expected IDs check_code(client, 'v4/nts/profile/10/10') # Check ajax call functions = check_json(client, 'v4/nts/profile/ajax/getFunctions?runid=10&testid=10') number_of_functions = len(functions) first_function_name = functions[0][0] assert 1 == number_of_functions assert "fn1" == first_function_name top_level_counters = check_json(client, 'v4/nts/profile/ajax/getTopLevelCounters?runids=10&testid=10') assert "cycles" in top_level_counters assert "branch-misses" in top_level_counters code_for_fn = check_json(client, 'v4/nts/profile/ajax/getCodeForFunction?runid=10&testid=10&f=fn1') lines_in_function = len(code_for_fn) assert 2 == lines_in_function # Make sure the new option does not break anything check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update') check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update') check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2') check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true') app.testing = False error_page = check_html(client, '/explode', expected_code=500) assert "integer division or modulo by zero" in error_page.data error_page = check_html(client, '/gone', expected_code=404) assert "test" in error_page.data check_html(client, '/sleep?timeout=0', expected_code=200) check_html(client, '/db_default/summary_report') check_html(client, '/rules') check_html(client, '/log') resp = check_code(client, '/__health') assert resp.data == "Ok" resp = check_code(client, '/ping') assert resp.data == "pong" # Check we can convert a sample into a graph page. graph_to_sample = check_code(client, '/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar', expected_code=HTTP_REDIRECT) assert graph_to_sample.headers['Location'] == "http://localhost/db_default/v4/nts/graph?foo=bar&plot.0=2.6.0" # Check that is we ask for a sample or invalid field, we explode with 400s. check_code(client, '/db_default/v4/nts/graph_for_sample/10000/compile_time?foo=bar', expected_code=HTTP_NOT_FOUND) check_code(client, '/db_default/v4/nts/graph_for_sample/10/not_a_metric?foo=bar', expected_code=HTTP_BAD_REQUEST) # check get_geomean_comparison_result with empty unchanged_tests check_html(client, '/v4/nts/11') check_html(client, '/v4/nts/12')