Exemplo n.º 1
0
 def setUp(self):
     """Bind to the LNT test instance."""
     _, instance_path, shared_inputs = sys.argv
     app = lnt.server.ui.app.App.create_standalone(instance_path)
     app.testing = True
     self.client = app.test_client()
     self.shared_inputs = shared_inputs
Exemplo n.º 2
0
def sanity_check_instance(instance_path):
    # Create an application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    index = client.get('/')

    # Visit all the test suites.
    test_suite_link_rex = re.compile("""  <a href="(.*)">(.*)</a><br>""")
    test_suite_list_start = index.data.index("<h3>Test Suites</h3>")
    test_suite_list_end = index.data.index("</div>", test_suite_list_start)
    for ln in index.data[test_suite_list_start:test_suite_list_end].split("\n"):
        # Ignore non-matching lines.
        print >>sys.stderr,ln
        m = test_suite_link_rex.match(ln)
        if not m:
            continue

        # We found a test suite link...
        link,name = m.groups()
        logging.info("visiting test suite %r", name)

        # Get the test suite overview page.
        overview = client.get(os.path.join("/", link))
        assert "LNT : %s - Recent Activity" % (name,) in overview.data
Exemplo n.º 3
0
 def setUp(self):
     """Bind to the LNT test instance."""
     _, instance_path = sys.argv
     self.instance_path = instance_path
     app = lnt.server.ui.app.App.create_standalone(instance_path)
     app.testing = True
     self.client = app.test_client()
Exemplo n.º 4
0
 def setUp(self):
     """Bind to the LNT test instance."""
     instance_path = sys.argv[1]
     app = lnt.server.ui.app.App.create_standalone(instance_path)
     app.testing = True
     app.config['WTF_CSRF_ENABLED'] = False
     self.client = app.test_client()
Exemplo n.º 5
0
def sanity_check_instance(instance_path):
    # Create an application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    index = client.get('/')

    # Visit all the test suites.
    test_suite_link_rex = re.compile("""  <a href="(.*)">(.*)</a><br>""")
    data = index.get_data(as_text=True)
    test_suite_list_start = data.index("<h3>Test Suites</h3>")
    test_suite_list_end = data.index("</div>", test_suite_list_start)
    for ln in data[test_suite_list_start:test_suite_list_end].split("\n"):
        # Ignore non-matching lines.
        print(ln, file=sys.stderr)
        m = test_suite_link_rex.match(ln)
        if not m:
            continue

        # We found a test suite link...
        link, name = m.groups()
        logging.info("visiting test suite %r", name)

        # Get the test suite overview page.
        overview = client.get(os.path.join("/", link))
        assert "LNT : %s - Recent Activity" % (name, ) in overview.data
Exemplo n.º 6
0
 def setUp(self):
     """Bind to the LNT test instance."""
     _, instance_path, shared_inputs = sys.argv
     app = lnt.server.ui.app.App.create_standalone(instance_path)
     app.testing = True
     self.client = app.test_client()
     self.shared_inputs = shared_inputs
Exemplo n.º 7
0
 def setUp(self):
     """Bind to the LNT test instance."""
     instance_path = sys.argv[1]
     app = lnt.server.ui.app.App.create_standalone(instance_path)
     app.testing = True
     app.config['WTF_CSRF_ENABLED'] = False
     self.client = app.test_client()
Exemplo n.º 8
0
def main():
    instance_path = sys.argv[1]

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True
    app.config['WTF_CSRF_ENABLED'] = False

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_html(client, '/')

    # Get the V4 overview page.
    check_html(client, '/v4/nts/')

    # Get a machine overview page.
    check_html(client, '/v4/nts/machine/1')
    # Check invalid machine gives error.
    check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
    # Get a machine overview page in JSON format.
    check_json(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_html(client, '/v4/nts/all_orders')

    # Get an order page.
    check_html(client, '/v4/nts/order/3')
    # Check invalid order gives error.
    check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)

    # Check that we can promote a baseline, then demote.
    form_data = dict(name="foo_baseline",
                     description="foo_descrimport iption",
                     prmote=True)
    r = client.post('/v4/nts/order/3', data=form_data)
    # We should redirect to the last page and flash.
    assert r.status_code == HTTP_REDIRECT

    # Try with redirect.
    r = client.post('/v4/nts/order/3', data=form_data, follow_redirects=True)
    assert r.status_code == HTTP_OK
    # Should see baseline displayed in page body.
    assert "Baseline - foo_baseline" in r.data

    # Now demote it.
    data2 = dict(name="foo_baseline",
                 description="foo_description",
                 update=False,
                 promote=False,
                 demote=True)
    r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
    assert r.status_code == HTTP_OK
    # Baseline should no longer be shown in page baseline.
    assert "Baseline - foo_baseline" not in r.data

    # Leave a baseline in place for the rest of the tests.
    client.post('/v4/nts/order/3', data=form_data)

    check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT)
    with app.test_client() as c:
        c.get('/v4/nts/set_baseline/1')
        session.get('baseline-default-nts') == 1

    # Get a run result page (and associated views).
    check_html(client, '/v4/nts/1')
    check_json(client, '/v4/nts/1?json=true')
    check_html(client, '/v4/nts/1/report')
    check_code(client, '/v4/nts/1/text_report')
    # Check invalid run numbers give errors.
    check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/9999/text_report',
               expected_code=HTTP_NOT_FOUND)

    # Get a graph page. This has been changed to redirect.
    check_redirect(client, '/v4/nts/1/graph?test.3=2',
                   'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')

    # Get a run that contains generic producer information
    check_producer_label(
        client, '/v4/nts/7',
        ['Current', '152293', '2012-05-10T16:28:23', '0:00:35', 'Producer'])
    check_producer_label(
        client, '/v4/nts/8',
        ['Current', '152294', '2012-05-11T16:28:23', '0:00:35', 'Producer'])

    # Get a run that contains Buildbot producer information
    check_producer_label(client, '/v4/nts/7', [
        'Previous', '152292', '2012-05-01T16:28:23', '0:00:35',
        'some-builder #987'
    ])
    check_producer_label(client, '/v4/nts/9', [
        'Current', '152295', '2012-05-12T16:28:23', '0:00:35',
        'some-builder #999'
    ])

    # Get the new graph page.
    check_html(client, '/v4/nts/graph?plot.0=1.3.2')
    # Don't crash when requesting non-existing data
    check_html(client, '/v4/nts/graph?plot.9999=1.3.2')
    check_code(client,
               '/v4/nts/graph?plot.0=9999.3.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/graph?plot.0=1.9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/graph?plot.0=1.3.9999',
               expected_code=HTTP_NOT_FOUND)
    check_json(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
    # Get the mean graph page.
    check_html(client, '/v4/nts/graph?mean=1.2')
    # Don't crash when requesting non-existing data
    check_code(client,
               '/v4/nts/graph?mean=9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/graph?mean=1.9999',
               expected_code=HTTP_NOT_FOUND)
    #  Check baselines work.
    check_html(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')

    # Check some variations of the daily report work.
    check_html(client, '/v4/nts/daily_report/2012/4/12')
    check_html(client, '/v4/nts/daily_report/2012/4/11')
    check_html(client, '/v4/nts/daily_report/2012/4/13')
    check_html(client, '/v4/nts/daily_report/2012/4/10')
    check_html(client, '/v4/nts/daily_report/2012/4/14')
    check_redirect(client, '/v4/nts/daily_report',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    check_redirect(client, '/v4/nts/daily_report?num_days=7',
                   '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
    # Don't crash when using a parameter that happens to have the same name as
    # a flask URL variable.
    check_redirect(client, '/v4/nts/daily_report?day=15',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    # Don't crash when requesting non-existing data
    check_html(client, '/v4/nts/daily_report/1999/4/12')
    check_code(client,
               '/v4/nts/daily_report/-1/4/12',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/daily_report/2012/13/12',
               expected_code=HTTP_BAD_REQUEST)
    check_code(client,
               '/v4/nts/daily_report/2012/4/32',
               expected_code=HTTP_BAD_REQUEST)

    # check ?filter-machine-regex= filter
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2',
        1)
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine',
        2)
    check_nr_machines_reported(
        client,
        '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
    # Don't crash on an invalid regular expression:
    # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
    # ignoring the invalid regex.
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)

    # check that a regression seen between 2 consecutive runs that are
    # more than a day apart gets reported
    result_table_20120504 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/04', "Execution Time")
    check_table_content(
        result_table_20120504,
        [["test1", ""], ["", "machine2", "1.000", "-", "900.00%", ""],
         ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]])
    check_table_links(
        result_table_20120504,
        [[], ["/db_default/v4/nts/graph?plot.0=2.4.2&highlight_run=6"], [],
         ["/db_default/v4/nts/graph?plot.0=2.5.2&highlight_run=6"]])

    check_body_nr_tests_table(client, '/v4/nts/daily_report/2012/5/04',
                              [['machine2', '2', '0', '1']])

    # Check that a failing result does not show up in the spark line
    # as a dot with value 0.
    result_table_20120513 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/13?num_days=3', "Execution Time")
    check_table_content(result_table_20120513, [
        ["test6", ""],
        ["", "machine2", "1.000", "FAIL", "PASS", ""],
        ["test_hash1", ""],
        ["", "machine2", "1.000", '-', '20.00%', ""],
        ["test_hash2", ""],
        ["", "machine2", "1.000", '-', '20.00%', ""],
        ["test_mhash_on_run", ""],
        ["", "machine2", "1.000", '-', '20.00%', ""],
    ])
    check_table_links(
        result_table_20120513,
        [[], ['/db_default/v4/nts/graph?plot.0=2.6.2&highlight_run=9'], [],
         ['/db_default/v4/nts/graph?plot.0=2.7.2&highlight_run=9'], [],
         ['/db_default/v4/nts/graph?plot.0=2.8.2&highlight_run=9'], [],
         ['/db_default/v4/nts/graph?plot.0=2.9.2&highlight_run=9']])

    sparkline_test6_xml = \
        get_sparkline(result_table_20120513, "test6", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
    assert 2 == nr_sample_points, \
        "Expected 2 sample points, found %d" % nr_sample_points

    # Check that a different background color is used in the sparkline
    # when the hash values recorded are different. At the same time,
    # check that no background color is drawn on missing hash values,
    # using a sequence of (hash1, no hash, hash2) over 3 consecutive
    # days.
    sparkline_hash1_xml = \
        get_sparkline(result_table_20120513, "test_hash1", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash1_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color3 is not None
    assert color1 != color3
    assert color2 is None

    # Check that the same background color is used in the sparkline
    # when the hash values recorded are the same, using a
    # (hash1, hash2, hash1) sequence.
    sparkline_hash2_xml = \
        get_sparkline(result_table_20120513, "test_hash2", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash2_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color1 == color3
    assert color1 != color2
    assert color2 is not None

    # Check that we don't crash if a single run produces multiple
    # samples with different hash values for the same run. This could
    # happen e.g. when the compiler under test doesn't produce
    # object code deterministically.
    sparkline_mhashonrun_xml = get_sparkline(result_table_20120513,
                                             "test_mhash_on_run", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
    assert 4 == nr_sample_points, \
        "Expected 4 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color2 is None
    assert color1 != color3

    # Check some variations of the latest runs report work.
    check_html(client, '/v4/nts/latest_runs_report')

    check_redirect(client, '/db_default/submitRun',
                   '/db_default/v4/nts/submitRun')
    check_html(client, '/db_default/v4/nts/submitRun')

    check_html(client, '/v4/nts/global_status')

    check_html(client, '/v4/nts/recent_activity')

    # Now check the compile report
    # Get the V4 overview page.
    check_html(client, '/v4/compile/')

    # Get a machine overview page.
    check_html(client, '/v4/compile/machine/1')
    check_html(client, '/v4/compile/machine/2')
    check_code(client,
               '/v4/compile/machine/2/latest',
               expected_code=HTTP_REDIRECT)
    # Don't crash when requesting non-existing data
    check_code(client,
               '/v4/compile/machine/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)

    # Check the compare machine form gives correct redirects.
    resp = check_code(client,
                      '/v4/nts/machine/2/compare?compare_to_id=3',
                      expected_code=HTTP_REDIRECT)
    assert resp.headers[
        'Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4"
    resp = check_code(client,
                      '/v4/nts/machine/3/compare?compare_to_id=2',
                      expected_code=HTTP_REDIRECT)
    assert resp.headers[
        'Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"

    # Get the order summary page.
    check_html(client, '/v4/compile/all_orders')

    # Get an order page.
    check_html(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_html(client, '/v4/compile/1')
    check_html(client, '/v4/compile/2')
    check_html(client, '/v4/compile/3')
    check_html(client, '/v4/compile/4')
    check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)

    check_html(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_html(client, '/v4/compile/graph?plot.3=2.3.9')

    # Get the mean graph page.
    check_html(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_html(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_html(client, '/v4/compile/daily_report/2014/6/4')

    check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1',
                   '/v4/nts/regressions/1')
    check_html(client, '/v4/nts/regressions/')
    check_html(client, '/v4/nts/regressions/?machine_filter=machine2')
    check_html(client, '/v4/nts/regressions/?machine_filter=machine0')

    check_html(client, '/v4/nts/regressions/1')

    check_json(client, '/v4/nts/regressions/1?json=True')

    # Check 404 is issues for inexistent Code
    check_code(client,
               'v4/nts/profile/9999/9999',
               expected_code=HTTP_NOT_FOUND)

    # Profile Viewer Ajax functions
    # Check profiles page is responsive with expected IDs
    check_code(client, 'v4/nts/profile/10/10')
    # Check ajax call
    functions = check_json(
        client, 'v4/nts/profile/ajax/getFunctions?runid=10&testid=10')
    number_of_functions = len(functions)
    first_function_name = functions[0][0]
    assert 1 == number_of_functions
    assert "fn1" == first_function_name

    top_level_counters = check_json(
        client, 'v4/nts/profile/ajax/getTopLevelCounters?runids=10&testid=10')
    assert "cycles" in top_level_counters
    assert "branch-misses" in top_level_counters

    code_for_fn = check_json(
        client,
        'v4/nts/profile/ajax/getCodeForFunction?runid=10&testid=10&f=fn1')
    lines_in_function = len(code_for_fn)
    assert 2 == lines_in_function

    # Make sure the new option does not break anything
    check_html(
        client,
        '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update'
    )
    check_json(
        client,
        '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update'
    )
    check_html(client,
               '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
    check_json(
        client,
        '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
    app.testing = False
    error_page = check_html(client, '/explode', expected_code=500)
    assert "integer division or modulo by zero" in error_page.data

    error_page = check_html(client, '/gone', expected_code=404)
    assert "test" in error_page.data

    check_html(client, '/sleep?timeout=0', expected_code=200)

    check_html(client, '/db_default/summary_report')

    check_html(client, '/rules')
    check_html(client, '/log')
    resp = check_code(client, '/__health')
    assert resp.data == "Ok"
    resp = check_code(client, '/ping')
    assert resp.data == "pong"

    # Check we can convert a sample into a graph page.
    graph_to_sample = check_code(
        client,
        '/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar',
        expected_code=HTTP_REDIRECT)
    assert graph_to_sample.headers[
        'Location'] == "http://localhost/db_default/v4/nts/graph?foo=bar&plot.0=2.6.0"

    # Check that is we ask for a sample or invalid field, we explode with 400s.
    check_code(
        client,
        '/db_default/v4/nts/graph_for_sample/10000/compile_time?foo=bar',
        expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/db_default/v4/nts/graph_for_sample/10/not_a_metric?foo=bar',
               expected_code=HTTP_BAD_REQUEST)

    # check get_geomean_comparison_result with empty unchanged_tests
    check_html(client, '/v4/nts/11')
    check_html(client, '/v4/nts/12')
Exemplo n.º 9
0
Arquivo: V4Pages.py Projeto: efcs/lnt
def main():
    _,instance_path = sys.argv

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_code(client, '/')

    # Get the V4 overview page.
    check_code(client, '/v4/nts/')

    # Get a machine overview page.
    check_code(client, '/v4/nts/machine/1')

    # Get a machine overview page in JSON format.
    check_code(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_code(client, '/v4/nts/all_orders')

    # Get an order page.
    check_code(client, '/v4/nts/order/3')

    # Get a run result page (and associated views).
    check_code(client, '/v4/nts/1')

    check_code(client, '/v4/nts/1?json=true')

    check_code(client, '/v4/nts/1/report')

    check_code(client, '/v4/nts/1/text_report')


    # Get a graph page. This has been changed to redirect.
    check_code(client, '/v4/nts/1/graph?test.87=2', expected_code=302)

    # Get the new graph page.
    check_code(client, '/v4/nts/graph?plot.0=1.87.2')

    # Get the mean graph page.
    check_code(client, '/v4/nts/graph?mean=1.2')

    # Check some variations of the daily report work.
    check_code(client, '/v4/nts/daily_report/2012/4/12')
    check_code(client, '/v4/nts/daily_report/2012/4/11')
    check_code(client, '/v4/nts/daily_report/2012/4/13')
    check_code(client, '/v4/nts/daily_report/2012/4/10')
    check_code(client, '/v4/nts/daily_report/2012/4/14')

    # Now check the compile report
    # Get the V4 overview page.
    check_code(client, '/v4/compile/')

    # Get a machine overview page.
    check_code(client, '/v4/compile/machine/1')
    check_code(client, '/v4/compile/machine/2')

    # Get the order summary page.
    check_code(client, '/v4/compile/all_orders')

    # Get an order page.
    check_code(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_code(client, '/v4/compile/1')
    check_code(client, '/v4/compile/2')
    check_code(client, '/v4/compile/3')
    check_code(client, '/v4/compile/4')
    check_code(client, '/v4/compile/10', expected_code=404) # This page should not be there.

    check_code(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_code(client, '/v4/compile/graph?plot.38=2.38.9')

    # Get the mean graph page.
    check_code(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_code(client, '/v4/compile/daily_report/2014/6/4')
Exemplo n.º 10
0
def action_view_comparison(name, args):
    """view a report comparison using a temporary server"""

    import lnt.server.instance
    import lnt.server.ui.app
    import lnt.server.db.migrate

    parser = OptionParser("%s [options] <report A> <report B>" % (name,))
    parser.add_option("", "--hostname", dest="hostname", type=str,
                      help="host interface to use [%default]",
                      default='localhost')
    parser.add_option("", "--port", dest="port", type=int, metavar="N",
                      help="local port to use [%default]", default=8000)
    parser.add_option("", "--dry-run", dest="dry_run",
                      help="Do a dry run through the comparison. [%default]"
                      " [%default]", action="store_true", default=False)
    (opts, args) = parser.parse_args(args)

    if len(args) != 2:
        parser.error("invalid number of arguments")

    report_a_path, report_b_path = args

    # Set up the default logger.
    logger = logging.getLogger("lnt")
    logger.setLevel(logging.ERROR)
    handler = logging.StreamHandler(sys.stderr)
    handler.setFormatter(logging.Formatter(
        '%(asctime)s %(levelname)s: %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S'))
    logger.addHandler(handler)

    # Create a temporary directory to hold the instance.
    tmpdir = tempfile.mkdtemp(suffix='lnt')

    try:
        # Create a temporary instance.
        url = 'http://%s:%d' % (opts.hostname, opts.port)
        db_path = os.path.join(tmpdir, 'data.db')
        db_info = lnt.server.config.DBInfo(
            'sqlite:///%s' % (db_path,), '0.4', None,
            lnt.server.config.EmailConfig(False, '', '', []), "0")
        # _(self, name, zorgURL, dbDir, tempDir,
        # profileDir, secretKey, databases, blacklist):
        config = lnt.server.config.Config('LNT', url, db_path, tmpdir,
                                          None, None, {'default': db_info},
                                          None)
        instance = lnt.server.instance.Instance(None, config)

        # Create the database.
        lnt.server.db.migrate.update_path(db_path)

        # Import the two reports.
        with contextlib.closing(config.get_database('default')) as db:
            import_and_report(
                config, 'default', db, report_a_path,
                '<auto>', commit=True)
            import_and_report(
                config, 'default', db, report_b_path,
                '<auto>', commit=True)

            # Dispatch another thread to start the webbrowser.
            comparison_url = '%s/v4/nts/2?compare_to=1' % (url,)
            note("opening comparison view: %s" % (comparison_url,))
            
            if not opts.dry_run:
                thread.start_new_thread(start_browser, (comparison_url, True))

            # Run the webserver.
            app = lnt.server.ui.app.App.create_with_instance(instance)
            app.debug = True
            
            if opts.dry_run:
                # Don't catch out exceptions.
                app.testing = True
                # Create a test client.
                client = app.test_client()
                response = client.get(comparison_url)
                assert response.status_code == 200, "Page did not return 200."
            else:
                app.run(opts.hostname, opts.port, use_reloader=False)
    finally:
        shutil.rmtree(tmpdir)
Exemplo n.º 11
0
def main():
    _, instance_path = sys.argv

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True
    app.config['WTF_CSRF_ENABLED'] = False

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_code(client, '/')

    # Rules the index page.
    check_code(client, '/rules')

    # Get the V4 overview page.
    check_code(client, '/v4/nts/')

    # Get a machine overview page.
    check_code(client, '/v4/nts/machine/1')
    # Check invalid machine gives error.
    check_code(client,  '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
    # Get a machine overview page in JSON format.
    check_code(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_code(client, '/v4/nts/all_orders')

    # Get an order page.
    check_code(client, '/v4/nts/order/3')
    # Check invalid order gives error.
    check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)

    # Check that we can promote a baseline, then demote.
    form_data = dict(name="foo_baseline",
                     description="foo_descrimport iption",
                     prmote=True)
    r = client.post('/v4/nts/order/3', data=form_data)
    # We should redirect to the last page and flash.
    assert r.status_code == HTTP_REDIRECT

    # Try with redirect.
    r = client.post('/v4/nts/order/3',
                    data=form_data,
                    follow_redirects=True)
    assert r.status_code == HTTP_OK
    # Should see baseline displayed in page body.
    assert "Baseline - foo_baseline" in r.data

    # Now demote it.
    data2 = dict(name="foo_baseline",
                 description="foo_description",
                 update=False,
                 promote=False,
                 demote=True)
    r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
    assert r.status_code == HTTP_OK
    # Baseline should no longer be shown in page baseline.
    assert "Baseline - foo_baseline" not in r.data

    # Leave a baseline in place for the rest of the tests.
    client.post('/v4/nts/order/3', data=form_data)

    check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT)
    with app.test_client() as c:
        c.get('/v4/nts/set_baseline/1')
        session.get('baseline-default-nts') == 1

    # Get a run result page (and associated views).
    check_code(client, '/v4/nts/1')
    check_code(client, '/v4/nts/1?json=true')
    check_code(client, '/v4/nts/1/report')
    check_code(client, '/v4/nts/1/text_report')
    # Check invalid run numbers give errors.
    check_code(client, '/v4/nts/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999?json=true',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/report',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/text_report',
               expected_code=HTTP_NOT_FOUND)

    # Get a graph page. This has been changed to redirect.
    check_redirect(client, '/v4/nts/1/graph?test.3=2',
                   'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')

    # Get the new graph page.
    check_code(client, '/v4/nts/graph?plot.0=1.3.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
    check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
    # Get the mean graph page.
    check_code(client, '/v4/nts/graph?mean=1.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?mean=9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?mean=1.9999',
               expected_code=HTTP_NOT_FOUND)
    #  Check baselines work.
    check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')

    # Check some variations of the daily report work.
    check_code(client, '/v4/nts/daily_report/2012/4/12')
    check_code(client, '/v4/nts/daily_report/2012/4/11')
    check_code(client, '/v4/nts/daily_report/2012/4/13')
    check_code(client, '/v4/nts/daily_report/2012/4/10')
    check_code(client, '/v4/nts/daily_report/2012/4/14')
    check_redirect(client, '/v4/nts/daily_report',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    check_redirect(client, '/v4/nts/daily_report?num_days=7',
                   '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
    # Don't crash when using a parameter that happens to have the same name as
    # a flask URL variable.
    check_redirect(client, '/v4/nts/daily_report?day=15',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/daily_report/1999/4/12')
    check_code(client, '/v4/nts/daily_report/-1/4/12',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/daily_report/2012/13/12',
               expected_code=HTTP_BAD_REQUEST)
    check_code(client, '/v4/nts/daily_report/2012/4/32',
               expected_code=HTTP_BAD_REQUEST)

    # check ?filter-machine-regex= filter
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
    # Don't crash on an invalid regular expression:
    # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
    # ignoring the invalid regex.
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)

    # check that a regression seen between 2 consecutive runs that are
    # more than a day apart gets reported
    result_table_20120504 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/04', "execution_time")
    check_table_content(result_table_20120504,
                        [["test1", ""],
                         ["", "machine2", "1.0000", "-", "900.00%", ""],
                         ["test2", ""],
                         ["", "machine2", "FAIL", "-", "PASS", ""]])

    check_body_nr_tests_table(
        client, '/v4/nts/daily_report/2012/5/04',
        [['machine2', '2', '0', '1']])

    # Check that a failing result does not show up in the spark line
    # as a dot with value 0.
    result_table_20120513 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time")
    check_table_content(result_table_20120513,
                        [["test6", ""],
                         ["", "machine2", "1.0000", "FAIL", "PASS", ""],
                         ["test_hash1", ""],
                         ["", "machine2", "1.0000", '-', '20.00%', ""],
                         ["test_hash2", ""],
                         ["", "machine2", "1.0000", '-', '20.00%', ""],
                         ["test_mhash_on_run", ""],
                         ["", "machine2", "1.0000", '-', '20.00%', ""], ])

    sparkline_test6_xml = \
        get_sparkline(result_table_20120513, "test6", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
    assert 2 == nr_sample_points, \
        "Expected 2 sample points, found %d" % nr_sample_points

    # Check that a different background color is used in the sparkline
    # when the hash values recorded are different. At the same time,
    # check that no background color is drawn on missing hash values,
    # using a sequence of (hash1, no hash, hash2) over 3 consecutive
    # days.
    sparkline_hash1_xml = \
        get_sparkline(result_table_20120513, "test_hash1", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash1_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color3 is not None
    assert color1 != color3
    assert color2 is None

    # Check that the same background color is used in the sparkline
    # when the hash values recorded are the same, using a
    # (hash1, hash2, hash1) sequence.
    sparkline_hash2_xml = \
        get_sparkline(result_table_20120513, "test_hash2", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash2_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color1 == color3
    assert color1 != color2
    assert color2 is not None

    # Check that we don't crash if a single run produces multiple
    # samples with different hash values for the same run. This could
    # happen e.g. when the compiler under test doesn't produce
    # object code deterministically.
    sparkline_mhashonrun_xml = get_sparkline(
        result_table_20120513, "test_mhash_on_run", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
    assert 4 == nr_sample_points, \
        "Expected 4 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color2 is None
    assert color1 != color3

    # Now check the compile report
    # Get the V4 overview page.
    check_code(client, '/v4/compile/')

    # Get a machine overview page.
    check_code(client, '/v4/compile/machine/1')
    check_code(client, '/v4/compile/machine/2')
    check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT)
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/compile/machine/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)

    # Check the compare machine form gives correct redirects.
    resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT)
    assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4"
    resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT)
    assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"
    
    # Get the order summary page.
    check_code(client, '/v4/compile/all_orders')

    # Get an order page.
    check_code(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_code(client, '/v4/compile/1')
    check_code(client, '/v4/compile/2')
    check_code(client, '/v4/compile/3')
    check_code(client, '/v4/compile/4')
    check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)

    check_code(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_code(client, '/v4/compile/graph?plot.3=2.3.9')

    # Get the mean graph page.
    check_code(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_code(client, '/v4/compile/daily_report/2014/6/4')

    check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
    check_code(client, '/v4/nts/regressions/')
    check_code(client, '/v4/nts/regressions/?machine_filter=machine2')
    check_code(client, '/v4/nts/regressions/?machine_filter=machine0')

    check_code(client, '/v4/nts/regressions/1')

    check_json(client, '/v4/nts/regressions/1?json=True')

    # Make sure the new option does not break anything
    check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
    check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
    check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
    check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
Exemplo n.º 12
0
def main():
    _,instance_path = sys.argv

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_code(client, '/')

    # Get the V4 overview page.
    check_code(client, '/v4/nts/')

    # Get a machine overview page.
    check_code(client, '/v4/nts/machine/1')
    # Check invalid machine gives error.
    check_code(client,  '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
    # Get a machine overview page in JSON format.
    check_code(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_code(client, '/v4/nts/all_orders')

    # Get an order page.
    check_code(client, '/v4/nts/order/3')
    # Check invalid order gives error.
    check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)

    # Get a run result page (and associated views).
    check_code(client, '/v4/nts/1')
    check_code(client, '/v4/nts/1?json=true')
    check_code(client, '/v4/nts/1/report')
    check_code(client, '/v4/nts/1/text_report')
    # Check invalid run numbers give errors.
    check_code(client, '/v4/nts/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999?json=true',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/report',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/text_report',
               expected_code=HTTP_NOT_FOUND)

    # Get a graph page. This has been changed to redirect.
    check_redirect(client, '/v4/nts/1/graph?test.87=2',
                   'v4/nts/graph\?plot\.0=1\.87\.2&highlight_run=1$')

    # Get the new graph page.
    check_code(client, '/v4/nts/graph?plot.0=1.87.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?plot.9999=1.87.2')
    check_code(client, '/v4/nts/graph?plot.0=9999.87.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.87.9999',
               expected_code=HTTP_NOT_FOUND)

    # Get the mean graph page.
    check_code(client, '/v4/nts/graph?mean=1.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?mean=9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?mean=1.9999',
               expected_code=HTTP_NOT_FOUND)

    # Check some variations of the daily report work.
    check_code(client, '/v4/nts/daily_report/2012/4/12')
    check_code(client, '/v4/nts/daily_report/2012/4/11')
    check_code(client, '/v4/nts/daily_report/2012/4/13')
    check_code(client, '/v4/nts/daily_report/2012/4/10')
    check_code(client, '/v4/nts/daily_report/2012/4/14')
    check_redirect(client, '/v4/nts/daily_report',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    check_redirect(client, '/v4/nts/daily_report?num_days=7',
                   '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
    # Don't crash when using a parameter that happens to have the same name as
    # a flask URL variable.
    check_redirect(client, '/v4/nts/daily_report?day=15',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/daily_report/1999/4/12')
    check_code(client, '/v4/nts/daily_report/-1/4/12',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/daily_report/2012/13/12',
               expected_code=HTTP_BAD_REQUEST)
    check_code(client, '/v4/nts/daily_report/2012/4/32',
               expected_code=HTTP_BAD_REQUEST)

    # check ?filter-machine-regex= filter
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
    # Don't crash on an invalid regular expression:
    # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
    # ignoring the invalid regex.
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)

    # check that a regression seen between 2 consecutive runs that are
    # more than a day apart gets reported
    check_body_result_table(client, '/v4/nts/daily_report/2012/5/04',
                            "execution_time",
                            [["test1", ""],
                             ["", "machine2", "1.0000", "-", "900.00%", ""],
                             ["test2", ""],
                             ["", "machine2", "FAIL", "-", "PASS", ""]])

    # Now check the compile report
    # Get the V4 overview page.
    check_code(client, '/v4/compile/')

    # Get a machine overview page.
    check_code(client, '/v4/compile/machine/1')
    check_code(client, '/v4/compile/machine/2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/compile/machine/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)

    # Get the order summary page.
    check_code(client, '/v4/compile/all_orders')

    # Get an order page.
    check_code(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_code(client, '/v4/compile/1')
    check_code(client, '/v4/compile/2')
    check_code(client, '/v4/compile/3')
    check_code(client, '/v4/compile/4')
    check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)

    check_code(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_code(client, '/v4/compile/graph?plot.38=2.38.9')

    # Get the mean graph page.
    check_code(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_code(client, '/v4/compile/daily_report/2014/6/4')
Exemplo n.º 13
0
def main():
    _, instance_path = sys.argv

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_code(client, '/')
    
    # Rules the index page.
    check_code(client, '/rules')

    # Get the V4 overview page.
    check_code(client, '/v4/nts/')

    # Get a machine overview page.
    check_code(client, '/v4/nts/machine/1')
    # Check invalid machine gives error.
    check_code(client,  '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
    # Get a machine overview page in JSON format.
    check_code(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_code(client, '/v4/nts/all_orders')

    # Get an order page.
    check_code(client, '/v4/nts/order/3')
    # Check invalid order gives error.
    check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)

    # Get a run result page (and associated views).
    check_code(client, '/v4/nts/1')
    check_code(client, '/v4/nts/1?json=true')
    check_code(client, '/v4/nts/1/report')
    check_code(client, '/v4/nts/1/text_report')
    # Check invalid run numbers give errors.
    check_code(client, '/v4/nts/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999?json=true',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/report',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/text_report',
               expected_code=HTTP_NOT_FOUND)

    # Get a graph page. This has been changed to redirect.
    check_redirect(client, '/v4/nts/1/graph?test.3=2',
                   'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')

    # Get the new graph page.
    check_code(client, '/v4/nts/graph?plot.0=1.3.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
    check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
    # Get the mean graph page.
    check_code(client, '/v4/nts/graph?mean=1.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?mean=9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?mean=1.9999',
               expected_code=HTTP_NOT_FOUND)
    #  Check baselines work.
    check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
    
    # Check some variations of the daily report work.
    check_code(client, '/v4/nts/daily_report/2012/4/12')
    check_code(client, '/v4/nts/daily_report/2012/4/11')
    check_code(client, '/v4/nts/daily_report/2012/4/13')
    check_code(client, '/v4/nts/daily_report/2012/4/10')
    check_code(client, '/v4/nts/daily_report/2012/4/14')
    check_redirect(client, '/v4/nts/daily_report',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    check_redirect(client, '/v4/nts/daily_report?num_days=7',
                   '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
    # Don't crash when using a parameter that happens to have the same name as
    # a flask URL variable.
    check_redirect(client, '/v4/nts/daily_report?day=15',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/daily_report/1999/4/12')
    check_code(client, '/v4/nts/daily_report/-1/4/12',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/daily_report/2012/13/12',
               expected_code=HTTP_BAD_REQUEST)
    check_code(client, '/v4/nts/daily_report/2012/4/32',
               expected_code=HTTP_BAD_REQUEST)

    # check ?filter-machine-regex= filter
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
    # Don't crash on an invalid regular expression:
    # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
    # ignoring the invalid regex.
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)

    # check that a regression seen between 2 consecutive runs that are
    # more than a day apart gets reported
    result_table_20120504 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/04', "execution_time")
    check_table_content(result_table_20120504,
                        [["test1", ""],
                         ["", "machine2", "1.0000", "-", "900.00%", ""],
                         ["test2", ""],
                         ["", "machine2", "FAIL", "-", "PASS", ""]])

    check_body_nr_tests_table(
        client, '/v4/nts/daily_report/2012/5/04',
        [['machine2', '2', '0', '1']])

    # Check that a failing result does not show up in the spark line
    # as a dot with value 0.
    result_table_20120513 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time")
    check_table_content(result_table_20120513,
                        [["test6", ""],
                         ["", "machine2", "1.0000", "FAIL", "PASS", ""],
                         ["test_hash1", ""],
                         ["", "machine2", "1.0000", '-', '20.00%', ""],
                         ["test_hash2", ""],
                         ["", "machine2", "1.0000", '-', '20.00%', ""],
                         ["test_mhash_on_run", ""],
                         ["", "machine2", "1.0000", '-', '20.00%', ""], ])

    sparkline_test6_xml = \
        get_sparkline(result_table_20120513, "test6", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
    assert 2 == nr_sample_points, \
        "Expected 2 sample points, found %d" % nr_sample_points

    # Check that a different background color is used in the sparkline
    # when the hash values recorded are different. At the same time,
    # check that no background color is drawn on missing hash values,
    # using a sequence of (hash1, no hash, hash2) over 3 consecutive
    # days.
    sparkline_hash1_xml = \
        get_sparkline(result_table_20120513, "test_hash1", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash1_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color3 is not None
    assert color1 != color3
    assert color2 is None

    # Check that the same background color is used in the sparkline
    # when the hash values recorded are the same, using a
    # (hash1, hash2, hash1) sequence.
    sparkline_hash2_xml = \
        get_sparkline(result_table_20120513, "test_hash2", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash2_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color1 == color3
    assert color1 != color2
    assert color2 is not None

    # Check that we don't crash if a single run produces multiple
    # samples with different hash values for the same run. This could
    # happen e.g. when the compiler under test doesn't produce
    # object code deterministically.
    sparkline_mhashonrun_xml = get_sparkline(
        result_table_20120513, "test_mhash_on_run", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
    assert 4 == nr_sample_points, \
        "Expected 4 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color2 is None
    assert color1 != color3

    # Now check the compile report
    # Get the V4 overview page.
    check_code(client, '/v4/compile/')

    # Get a machine overview page.
    check_code(client, '/v4/compile/machine/1')
    check_code(client, '/v4/compile/machine/2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/compile/machine/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)

    # Get the order summary page.
    check_code(client, '/v4/compile/all_orders')

    # Get an order page.
    check_code(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_code(client, '/v4/compile/1')
    check_code(client, '/v4/compile/2')
    check_code(client, '/v4/compile/3')
    check_code(client, '/v4/compile/4')
    check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)

    check_code(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_code(client, '/v4/compile/graph?plot.3=2.3.9')

    # Get the mean graph page.
    check_code(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_code(client, '/v4/compile/daily_report/2014/6/4')
    
    check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
    check_code(client, '/v4/nts/regressions/')
    
    check_code(client, '/v4/nts/regressions/1')
    
    check_json(client, '/v4/nts/regressions/1?json=True')
Exemplo n.º 14
0
def action_view_comparison(report_a, report_b, hostname, port, dry_run,
                           testsuite):
    """view a report comparison using a temporary server"""
    from .common import init_logger
    from lnt.util import logger
    from lnt.util.ImportData import import_and_report
    import contextlib
    import lnt.server.db.migrate
    import lnt.server.instance
    import lnt.server.ui.app
    import logging
    import os
    import shutil
    import tempfile
    import thread

    init_logger(logging.ERROR)

    # Create a temporary directory to hold the instance.
    tmpdir = tempfile.mkdtemp(suffix='lnt')

    try:
        # Create a temporary instance.
        url = 'http://%s:%d' % (hostname, port)
        db_path = os.path.join(tmpdir, 'data.db')
        db_info = lnt.server.config.DBInfo(
            'sqlite:///%s' % (db_path,), None,
            lnt.server.config.EmailConfig(False, '', '', []), "0")
        # _(self, name, zorgURL, dbDir, tempDir,
        # profileDir, secretKey, databases, blacklist):
        config = lnt.server.config.Config('LNT', url, db_path, tmpdir,
                                          None, "Not secret key.",
                                          {'default': db_info}, None,
                                          None)
        instance = lnt.server.instance.Instance(None, config)

        # Create the database.
        lnt.server.db.migrate.update_path(db_path)

        # Import the two reports.
        with contextlib.closing(config.get_database('default')) as db:
            session = db.make_session()
            import_and_report(config, 'default', db, session, report_a,
                                  '<auto>', testsuite, select_machine='match')
            import_and_report(config, 'default', db, session, report_b,
                              '<auto>', testsuite, select_machine='match')

            # Dispatch another thread to start the webbrowser.
            comparison_url = '%s/v4/nts/2?compare_to=1' % (url,)
            logger.info("opening comparison view: %s" % (comparison_url,))

            if not dry_run:
                thread.start_new_thread(_start_browser, (comparison_url, True))

            # Run the webserver.
            app = lnt.server.ui.app.App.create_with_instance(instance)
            app.debug = True

            if dry_run:
                # Don't catch out exceptions.
                app.testing = True
                # Create a test client.
                client = app.test_client()
                response = client.get(comparison_url)
                assert response.status_code == 200, "Page did not return 200."
            else:
                app.run(hostname, port, use_reloader=False)
    finally:
        shutil.rmtree(tmpdir)
Exemplo n.º 15
0
def main():
    _, instance_path = sys.argv

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_code(client, '/')
    
    # Rules the index page.
    check_code(client, '/rules')

    # Get the V4 overview page.
    check_code(client, '/v4/nts/')

    # Get a machine overview page.
    check_code(client, '/v4/nts/machine/1')
    # Check invalid machine gives error.
    check_code(client,  '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
    # Get a machine overview page in JSON format.
    check_code(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_code(client, '/v4/nts/all_orders')

    # Get an order page.
    check_code(client, '/v4/nts/order/3')
    # Check invalid order gives error.
    check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)

    # Get a run result page (and associated views).
    check_code(client, '/v4/nts/1')
    check_code(client, '/v4/nts/1?json=true')
    check_code(client, '/v4/nts/1/report')
    check_code(client, '/v4/nts/1/text_report')
    # Check invalid run numbers give errors.
    check_code(client, '/v4/nts/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999?json=true',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/report',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/text_report',
               expected_code=HTTP_NOT_FOUND)

    # Get a graph page. This has been changed to redirect.
    check_redirect(client, '/v4/nts/1/graph?test.3=2',
                   'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')

    # Get the new graph page.
    check_code(client, '/v4/nts/graph?plot.0=1.3.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
    check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
    # Get the mean graph page.
    check_code(client, '/v4/nts/graph?mean=1.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?mean=9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?mean=1.9999',
               expected_code=HTTP_NOT_FOUND)

    # Check some variations of the daily report work.
    check_code(client, '/v4/nts/daily_report/2012/4/12')
    check_code(client, '/v4/nts/daily_report/2012/4/11')
    check_code(client, '/v4/nts/daily_report/2012/4/13')
    check_code(client, '/v4/nts/daily_report/2012/4/10')
    check_code(client, '/v4/nts/daily_report/2012/4/14')
    check_redirect(client, '/v4/nts/daily_report',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    check_redirect(client, '/v4/nts/daily_report?num_days=7',
                   '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
    # Don't crash when using a parameter that happens to have the same name as
    # a flask URL variable.
    check_redirect(client, '/v4/nts/daily_report?day=15',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/daily_report/1999/4/12')
    check_code(client, '/v4/nts/daily_report/-1/4/12',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/daily_report/2012/13/12',
               expected_code=HTTP_BAD_REQUEST)
    check_code(client, '/v4/nts/daily_report/2012/4/32',
               expected_code=HTTP_BAD_REQUEST)

    # check ?filter-machine-regex= filter
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
    # Don't crash on an invalid regular expression:
    # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
    # ignoring the invalid regex.
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)

    # check that a regression seen between 2 consecutive runs that are
    # more than a day apart gets reported
    check_body_result_table(client, '/v4/nts/daily_report/2012/5/04',
                            "execution_time",
                            [["test1", ""],
                             ["", "machine2", "1.0000", "-", "900.00%", ""],
                             ["test2", ""],
                             ["", "machine2", "FAIL", "-", "PASS", ""]])

    # Check that a failing result does not show up in the spark line
    # as a dot with value 0.
    check_body_result_table(client,
                            '/v4/nts/daily_report/2012/5/13?num_days=3',
                            "execution_time",
                            [["test6", ""],
                             ["", "machine2", "1.0000", "FAIL", "PASS", ""]])
    sparkline_xml = get_sparkline(client,
                                  '/v4/nts/daily_report/2012/5/13?num_days=3',
                                  "execution_time", "test6", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_xml))
    assert 2 == nr_sample_points, \
        "Expected 2 sample points, found %d" % nr_sample_points

    # Now check the compile report
    # Get the V4 overview page.
    check_code(client, '/v4/compile/')

    # Get a machine overview page.
    check_code(client, '/v4/compile/machine/1')
    check_code(client, '/v4/compile/machine/2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/compile/machine/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)

    # Get the order summary page.
    check_code(client, '/v4/compile/all_orders')

    # Get an order page.
    check_code(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_code(client, '/v4/compile/1')
    check_code(client, '/v4/compile/2')
    check_code(client, '/v4/compile/3')
    check_code(client, '/v4/compile/4')
    check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)

    check_code(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_code(client, '/v4/compile/graph?plot.3=2.3.9')

    # Get the mean graph page.
    check_code(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_code(client, '/v4/compile/daily_report/2014/6/4')
    
    check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
    check_code(client, '/v4/nts/regressions/')
    
    check_code(client, '/v4/nts/regressions/1')
Exemplo n.º 16
0
def main():
    _, instance_path = sys.argv

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_code(client, '/')

    # Rules the index page.
    check_code(client, '/rules')

    # Get the V4 overview page.
    check_code(client, '/v4/nts/')

    # Get a machine overview page.
    check_code(client, '/v4/nts/machine/1')
    # Check invalid machine gives error.
    check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
    # Get a machine overview page in JSON format.
    check_code(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_code(client, '/v4/nts/all_orders')

    # Get an order page.
    check_code(client, '/v4/nts/order/3')
    # Check invalid order gives error.
    check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)

    # Get a run result page (and associated views).
    check_code(client, '/v4/nts/1')
    check_code(client, '/v4/nts/1?json=true')
    check_code(client, '/v4/nts/1/report')
    check_code(client, '/v4/nts/1/text_report')
    # Check invalid run numbers give errors.
    check_code(client, '/v4/nts/9999', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999?json=true', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/report', expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/9999/text_report',
               expected_code=HTTP_NOT_FOUND)

    # Get a graph page. This has been changed to redirect.
    check_redirect(client, '/v4/nts/1/graph?test.3=2',
                   'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')

    # Get the new graph page.
    check_code(client, '/v4/nts/graph?plot.0=1.3.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
    check_code(client,
               '/v4/nts/graph?plot.0=9999.3.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/graph?plot.0=1.9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/graph?plot.0=1.3.9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
    # Get the mean graph page.
    check_code(client, '/v4/nts/graph?mean=1.2')
    # Don't crash when requesting non-existing data
    check_code(client,
               '/v4/nts/graph?mean=9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/graph?mean=1.9999',
               expected_code=HTTP_NOT_FOUND)
    #  Check baselines work.
    check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')

    # Check some variations of the daily report work.
    check_code(client, '/v4/nts/daily_report/2012/4/12')
    check_code(client, '/v4/nts/daily_report/2012/4/11')
    check_code(client, '/v4/nts/daily_report/2012/4/13')
    check_code(client, '/v4/nts/daily_report/2012/4/10')
    check_code(client, '/v4/nts/daily_report/2012/4/14')
    check_redirect(client, '/v4/nts/daily_report',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    check_redirect(client, '/v4/nts/daily_report?num_days=7',
                   '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
    # Don't crash when using a parameter that happens to have the same name as
    # a flask URL variable.
    check_redirect(client, '/v4/nts/daily_report?day=15',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/daily_report/1999/4/12')
    check_code(client,
               '/v4/nts/daily_report/-1/4/12',
               expected_code=HTTP_NOT_FOUND)
    check_code(client,
               '/v4/nts/daily_report/2012/13/12',
               expected_code=HTTP_BAD_REQUEST)
    check_code(client,
               '/v4/nts/daily_report/2012/4/32',
               expected_code=HTTP_BAD_REQUEST)

    # check ?filter-machine-regex= filter
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2',
        1)
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine',
        2)
    check_nr_machines_reported(
        client,
        '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
    # Don't crash on an invalid regular expression:
    # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
    # ignoring the invalid regex.
    check_nr_machines_reported(
        client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)

    # check that a regression seen between 2 consecutive runs that are
    # more than a day apart gets reported
    result_table_20120504 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/04', "execution_time")
    check_table_content(
        result_table_20120504,
        [["test1", ""], ["", "machine2", "1.0000", "-", "900.00%", ""],
         ["test2", ""], ["", "machine2", "FAIL", "-", "PASS", ""]])

    check_body_nr_tests_table(client, '/v4/nts/daily_report/2012/5/04',
                              [['machine2', '2', '0', '1']])

    # Check that a failing result does not show up in the spark line
    # as a dot with value 0.
    result_table_20120513 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time")
    check_table_content(result_table_20120513, [
        ["test6", ""],
        ["", "machine2", "1.0000", "FAIL", "PASS", ""],
        ["test_hash1", ""],
        ["", "machine2", "1.0000", '-', '20.00%', ""],
        ["test_hash2", ""],
        ["", "machine2", "1.0000", '-', '20.00%', ""],
        ["test_mhash_on_run", ""],
        ["", "machine2", "1.0000", '-', '20.00%', ""],
    ])

    sparkline_test6_xml = \
        get_sparkline(result_table_20120513, "test6", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
    assert 2 == nr_sample_points, \
        "Expected 2 sample points, found %d" % nr_sample_points

    # Check that a different background color is used in the sparkline
    # when the hash values recorded are different. At the same time,
    # check that no background color is drawn on missing hash values,
    # using a sequence of (hash1, no hash, hash2) over 3 consecutive
    # days.
    sparkline_hash1_xml = \
        get_sparkline(result_table_20120513, "test_hash1", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash1_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color3 is not None
    assert color1 != color3
    assert color2 is None

    # Check that the same background color is used in the sparkline
    # when the hash values recorded are the same, using a
    # (hash1, hash2, hash1) sequence.
    sparkline_hash2_xml = \
        get_sparkline(result_table_20120513, "test_hash2", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash2_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color1 == color3
    assert color1 != color2
    assert color2 is not None

    # Check that we don't crash if a single run produces multiple
    # samples with different hash values for the same run. This could
    # happen e.g. when the compiler under test doesn't produce
    # object code deterministically.
    sparkline_mhashonrun_xml = get_sparkline(result_table_20120513,
                                             "test_mhash_on_run", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
    assert 4 == nr_sample_points, \
        "Expected 4 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color2 is None
    assert color1 != color3

    # Now check the compile report
    # Get the V4 overview page.
    check_code(client, '/v4/compile/')

    # Get a machine overview page.
    check_code(client, '/v4/compile/machine/1')
    check_code(client, '/v4/compile/machine/2')
    # Don't crash when requesting non-existing data
    check_code(client,
               '/v4/compile/machine/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)

    # Get the order summary page.
    check_code(client, '/v4/compile/all_orders')

    # Get an order page.
    check_code(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_code(client, '/v4/compile/1')
    check_code(client, '/v4/compile/2')
    check_code(client, '/v4/compile/3')
    check_code(client, '/v4/compile/4')
    check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)

    check_code(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_code(client, '/v4/compile/graph?plot.3=2.3.9')

    # Get the mean graph page.
    check_code(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_code(client, '/v4/compile/daily_report/2014/6/4')

    check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1',
                   '/v4/nts/regressions/1')
    check_code(client, '/v4/nts/regressions/')

    check_code(client, '/v4/nts/regressions/1')

    check_json(client, '/v4/nts/regressions/1?json=True')
Exemplo n.º 17
0
def action_view_comparison(report_a, report_b, hostname, port, dry_run,
                           testsuite):
    """view a report comparison using a temporary server"""
    from .common import init_logger
    from lnt.util import logger
    from lnt.util.ImportData import import_and_report
    import contextlib
    import lnt.server.db.migrate
    import lnt.server.instance
    import lnt.server.ui.app
    import logging
    import os
    import shutil
    import sys
    import tempfile
    import thread
    import time
    import urllib
    import webbrowser

    init_logger(logging.ERROR)

    # Create a temporary directory to hold the instance.
    tmpdir = tempfile.mkdtemp(suffix='lnt')

    try:
        # Create a temporary instance.
        url = 'http://%s:%d' % (hostname, port)
        db_path = os.path.join(tmpdir, 'data.db')
        db_info = lnt.server.config.DBInfo(
            'sqlite:///%s' % (db_path, ), None,
            lnt.server.config.EmailConfig(False, '', '', []), "0")
        # _(self, name, zorgURL, dbDir, tempDir,
        # profileDir, secretKey, databases, blacklist):
        config = lnt.server.config.Config('LNT', url, db_path, tmpdir, None,
                                          "Not secret key.",
                                          {'default': db_info}, None, None)
        instance = lnt.server.instance.Instance(None, config)

        # Create the database.
        lnt.server.db.migrate.update_path(db_path)

        # Import the two reports.
        with contextlib.closing(config.get_database('default')) as db:
            session = db.make_session()
            r = import_and_report(config,
                                  'default',
                                  db,
                                  session,
                                  report_a,
                                  '<auto>',
                                  testsuite,
                                  select_machine='match')
            import_and_report(config,
                              'default',
                              db,
                              session,
                              report_b,
                              '<auto>',
                              testsuite,
                              select_machine='match')

            # Dispatch another thread to start the webbrowser.
            comparison_url = '%s/v4/nts/2?compare_to=1' % (url, )
            logger.info("opening comparison view: %s" % (comparison_url, ))

            if not dry_run:
                thread.start_new_thread(_start_browser, (comparison_url, True))

            # Run the webserver.
            app = lnt.server.ui.app.App.create_with_instance(instance)
            app.debug = True

            if dry_run:
                # Don't catch out exceptions.
                app.testing = True
                # Create a test client.
                client = app.test_client()
                response = client.get(comparison_url)
                assert response.status_code == 200, "Page did not return 200."
            else:
                app.run(hostname, port, use_reloader=False)
    finally:
        shutil.rmtree(tmpdir)
Exemplo n.º 18
0
def main():
    instance_path = sys.argv[1]

    # Create the application instance.
    app = lnt.server.ui.app.App.create_standalone(instance_path)

    # Don't catch out exceptions.
    app.testing = True
    app.config['WTF_CSRF_ENABLED'] = False

    # Create a test client.
    client = app.test_client()

    # Fetch the index page.
    check_html(client, '/')

    # Get the V4 overview page.
    check_html(client, '/v4/nts/')

    # Get a machine overview page.
    check_html(client, '/v4/nts/machine/1')
    # Check invalid machine gives error.
    check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
    # Get a machine overview page in JSON format.
    check_json(client, '/v4/nts/machine/1?json=true')

    # Get the order summary page.
    check_html(client, '/v4/nts/all_orders')

    # Get an order page.
    check_html(client, '/v4/nts/order/3')
    # Check invalid order gives error.
    check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)

    # Check that we can promote a baseline, then demote.
    form_data = dict(name="foo_baseline",
                     description="foo_descrimport iption",
                     prmote=True)
    r = client.post('/v4/nts/order/3', data=form_data)
    # We should redirect to the last page and flash.
    assert r.status_code == HTTP_REDIRECT

    # Try with redirect.
    r = client.post('/v4/nts/order/3',
                    data=form_data,
                    follow_redirects=True)
    assert r.status_code == HTTP_OK
    # Should see baseline displayed in page body.
    assert "Baseline - foo_baseline" in r.data

    # Now demote it.
    data2 = dict(name="foo_baseline",
                 description="foo_description",
                 update=False,
                 promote=False,
                 demote=True)
    r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
    assert r.status_code == HTTP_OK
    # Baseline should no longer be shown in page baseline.
    assert "Baseline - foo_baseline" not in r.data

    # Leave a baseline in place for the rest of the tests.
    client.post('/v4/nts/order/3', data=form_data)

    check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT)
    with app.test_client() as c:
        c.get('/v4/nts/set_baseline/1')
        session.get('baseline-default-nts') == 1

    # Get a run result page (and associated views).
    check_html(client, '/v4/nts/1')
    check_json(client, '/v4/nts/1?json=true')
    check_html(client, '/v4/nts/1/report')
    check_code(client, '/v4/nts/1/text_report')
    # Check invalid run numbers give errors.
    check_code(client, '/v4/nts/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999?json=true',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/report',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/9999/text_report',
               expected_code=HTTP_NOT_FOUND)

    # Get a graph page. This has been changed to redirect.
    check_redirect(client, '/v4/nts/1/graph?test.3=2',
                   'v4/nts/graph\?plot\.0=1\.3\.2&highlight_run=1$')

    # Get a run that contains generic producer information
    check_producer_label(client, '/v4/nts/7',
                         ['Current', '152293', '2012-05-10T16:28:23',
                          '0:00:35', 'Producer'])
    check_producer_label(client, '/v4/nts/8',
                         ['Current', '152294', '2012-05-11T16:28:23',
                          '0:00:35', 'Producer'])

    # Get a run that contains Buildbot producer information
    check_producer_label(client, '/v4/nts/7',
                         ['Previous', '152292', '2012-05-01T16:28:23',
                          '0:00:35', 'some-builder #987'])
    check_producer_label(client, '/v4/nts/9',
                         ['Current', '152295', '2012-05-12T16:28:23',
                          '0:00:35', 'some-builder #999'])

    # Get the new graph page.
    check_html(client, '/v4/nts/graph?plot.0=1.3.2')
    # Don't crash when requesting non-existing data
    check_html(client, '/v4/nts/graph?plot.9999=1.3.2')
    check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
               expected_code=HTTP_NOT_FOUND)
    check_json(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
    # Get the mean graph page.
    check_html(client, '/v4/nts/graph?mean=1.2')
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/nts/graph?mean=9999.2',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/graph?mean=1.9999',
               expected_code=HTTP_NOT_FOUND)
    #  Check baselines work.
    check_html(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')

    # Check some variations of the daily report work.
    check_html(client, '/v4/nts/daily_report/2012/4/12')
    check_html(client, '/v4/nts/daily_report/2012/4/11')
    check_html(client, '/v4/nts/daily_report/2012/4/13')
    check_html(client, '/v4/nts/daily_report/2012/4/10')
    check_html(client, '/v4/nts/daily_report/2012/4/14')
    check_redirect(client, '/v4/nts/daily_report',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    check_redirect(client, '/v4/nts/daily_report?num_days=7',
                   '/v4/nts/daily_report/\d+/\d+/\d+\?num_days=7$')
    # Don't crash when using a parameter that happens to have the same name as
    # a flask URL variable.
    check_redirect(client, '/v4/nts/daily_report?day=15',
                   '/v4/nts/daily_report/\d+/\d+/\d+$')
    # Don't crash when requesting non-existing data
    check_html(client, '/v4/nts/daily_report/1999/4/12')
    check_code(client, '/v4/nts/daily_report/-1/4/12',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/nts/daily_report/2012/13/12',
               expected_code=HTTP_BAD_REQUEST)
    check_code(client, '/v4/nts/daily_report/2012/4/32',
               expected_code=HTTP_BAD_REQUEST)

    # check ?filter-machine-regex= filter
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12', 3)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine2', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=machine', 2)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*[34]$', 1)
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=ma.*4', 0)
    # Don't crash on an invalid regular expression:
    # FIXME - this should probably return HTTP_BAD_REQUEST instead of silently
    # ignoring the invalid regex.
    check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)

    # check that a regression seen between 2 consecutive runs that are
    # more than a day apart gets reported
    result_table_20120504 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/04', "Execution Time")
    check_table_content(result_table_20120504,
                        [["test1", ""],
                         ["", "machine2", "1.000", "-", "900.00%", ""],
                         ["test2", ""],
                         ["", "machine2", "FAIL", "-", "PASS", ""]])
    check_table_links(result_table_20120504,
                      [[],
                       ["/db_default/v4/nts/graph?plot.0=2.4.2&highlight_run=6"],
                       [],
                       ["/db_default/v4/nts/graph?plot.0=2.5.2&highlight_run=6"]])

    check_body_nr_tests_table(
        client, '/v4/nts/daily_report/2012/5/04',
        [['machine2', '2', '0', '1']])

    # Check that a failing result does not show up in the spark line
    # as a dot with value 0.
    result_table_20120513 = get_results_table(
        client, '/v4/nts/daily_report/2012/5/13?num_days=3', "Execution Time")
    check_table_content(result_table_20120513,
                        [["test6", ""],
                         ["", "machine2", "1.000", "FAIL", "PASS", ""],
                         ["test_hash1", ""],
                         ["", "machine2", "1.000", '-', '20.00%', ""],
                         ["test_hash2", ""],
                         ["", "machine2", "1.000", '-', '20.00%', ""],
                         ["test_mhash_on_run", ""],
                         ["", "machine2", "1.000", '-', '20.00%', ""], ])
    check_table_links(result_table_20120513,
                      [[],
                       ['/db_default/v4/nts/graph?plot.0=2.6.2&highlight_run=9'],
                       [],
                       ['/db_default/v4/nts/graph?plot.0=2.7.2&highlight_run=9'],
                       [],
                       ['/db_default/v4/nts/graph?plot.0=2.8.2&highlight_run=9'],
                       [],
                       ['/db_default/v4/nts/graph?plot.0=2.9.2&highlight_run=9']])

    sparkline_test6_xml = \
        get_sparkline(result_table_20120513, "test6", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
    assert 2 == nr_sample_points, \
        "Expected 2 sample points, found %d" % nr_sample_points

    # Check that a different background color is used in the sparkline
    # when the hash values recorded are different. At the same time,
    # check that no background color is drawn on missing hash values,
    # using a sequence of (hash1, no hash, hash2) over 3 consecutive
    # days.
    sparkline_hash1_xml = \
        get_sparkline(result_table_20120513, "test_hash1", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash1_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color3 is not None
    assert color1 != color3
    assert color2 is None

    # Check that the same background color is used in the sparkline
    # when the hash values recorded are the same, using a
    # (hash1, hash2, hash1) sequence.
    sparkline_hash2_xml = \
        get_sparkline(result_table_20120513, "test_hash2", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
    assert 3 == nr_sample_points, \
        "Expected 3 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_hash2_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color1 is not None
    assert color1 == color3
    assert color1 != color2
    assert color2 is not None

    # Check that we don't crash if a single run produces multiple
    # samples with different hash values for the same run. This could
    # happen e.g. when the compiler under test doesn't produce
    # object code deterministically.
    sparkline_mhashonrun_xml = get_sparkline(
        result_table_20120513, "test_mhash_on_run", "machine2")
    nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
    assert 4 == nr_sample_points, \
        "Expected 4 sample points, found %d" % nr_sample_points
    background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
    assert len(background_colors) == 3
    color1, color2, color3 = background_colors
    assert color2 is None
    assert color1 != color3

    # Check some variations of the latest runs report work.
    check_html(client, '/v4/nts/latest_runs_report')

    check_redirect(client, '/db_default/submitRun',
                   '/db_default/v4/nts/submitRun')
    check_html(client, '/db_default/v4/nts/submitRun')

    check_html(client, '/v4/nts/global_status')

    check_html(client, '/v4/nts/recent_activity')

    # Now check the compile report
    # Get the V4 overview page.
    check_html(client, '/v4/compile/')

    # Get a machine overview page.
    check_html(client, '/v4/compile/machine/1')
    check_html(client, '/v4/compile/machine/2')
    check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT)
    # Don't crash when requesting non-existing data
    check_code(client, '/v4/compile/machine/9999',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
    check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)

    # Check the compare machine form gives correct redirects.
    resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT)
    assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4"
    resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT)
    assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"

    # Get the order summary page.
    check_html(client, '/v4/compile/all_orders')

    # Get an order page.
    check_html(client, '/v4/compile/order/3')

    # Get a run result page (and associated views).
    check_html(client, '/v4/compile/1')
    check_html(client, '/v4/compile/2')
    check_html(client, '/v4/compile/3')
    check_html(client, '/v4/compile/4')
    check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)

    check_html(client, '/v4/compile/1/report')

    check_code(client, '/v4/compile/1/text_report')

    # Get the new graph page.
    check_html(client, '/v4/compile/graph?plot.3=2.3.9')

    # Get the mean graph page.
    check_html(client, 'v4/compile/graph?mean=2.9')

    # Check some variations of the daily report work.
    check_html(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
    check_html(client, '/v4/compile/daily_report/2014/6/4')

    check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
    check_html(client, '/v4/nts/regressions/')
    check_html(client, '/v4/nts/regressions/?machine_filter=machine2')
    check_html(client, '/v4/nts/regressions/?machine_filter=machine0')

    check_html(client, '/v4/nts/regressions/1')

    check_json(client, '/v4/nts/regressions/1?json=True')

    # Check 404 is issues for inexistent Code
    check_code(client, 'v4/nts/profile/9999/9999', expected_code=HTTP_NOT_FOUND)

    # Profile Viewer Ajax functions
    # Check profiles page is responsive with expected IDs
    check_code(client, 'v4/nts/profile/10/10')
    # Check ajax call
    functions = check_json(client, 'v4/nts/profile/ajax/getFunctions?runid=10&testid=10')
    number_of_functions = len(functions)
    first_function_name = functions[0][0]
    assert 1 == number_of_functions
    assert "fn1" == first_function_name

    top_level_counters = check_json(client, 'v4/nts/profile/ajax/getTopLevelCounters?runids=10&testid=10')
    assert "cycles" in top_level_counters
    assert "branch-misses" in top_level_counters

    code_for_fn = check_json(client, 'v4/nts/profile/ajax/getCodeForFunction?runid=10&testid=10&f=fn1')
    lines_in_function = len(code_for_fn)
    assert 2 == lines_in_function

    # Make sure the new option does not break anything
    check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
    check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
    check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
    check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
    app.testing = False
    error_page = check_html(client, '/explode', expected_code=500)
    assert "integer division or modulo by zero" in error_page.data

    error_page = check_html(client, '/gone', expected_code=404)
    assert "test" in error_page.data

    check_html(client, '/sleep?timeout=0', expected_code=200)

    check_html(client, '/db_default/summary_report')

    check_html(client, '/rules')
    check_html(client, '/log')
    resp = check_code(client, '/__health')
    assert resp.data == "Ok"
    resp = check_code(client, '/ping')
    assert resp.data == "pong"

    # Check we can convert a sample into a graph page.
    graph_to_sample = check_code(client, '/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar',
                                 expected_code=HTTP_REDIRECT)
    assert graph_to_sample.headers['Location'] == "http://localhost/db_default/v4/nts/graph?foo=bar&plot.0=2.6.0"

    # Check that is we ask for a sample or invalid field, we explode with 400s.
    check_code(client, '/db_default/v4/nts/graph_for_sample/10000/compile_time?foo=bar',
               expected_code=HTTP_NOT_FOUND)
    check_code(client, '/db_default/v4/nts/graph_for_sample/10/not_a_metric?foo=bar',
               expected_code=HTTP_BAD_REQUEST)

    # check get_geomean_comparison_result with empty unchanged_tests
    check_html(client, '/v4/nts/11')
    check_html(client, '/v4/nts/12')