def test_history(): sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) form = pandokia.pcgi.form test_name = form["test_name"].value context = form["context"].value host = form["host"].value test_run = form["test_run"].value project = form["project"].value tb = text_table.text_table() tb.set_html_table_attributes("border=1") row = 0 tb.set_value(row, 0, "project") tb.set_value(row, 1, project) row += 1 tb.set_value(row, 0, "host") tb.set_value(row, 1, host) row += 1 tb.set_value(row, 0, "context") tb.set_value(row, 1, context) row += 1 tb.set_value(row, 0, "test_name") tb.set_value(row, 1, test_name) row += 1 print(tb.get_html()) print("<br>") tb = text_table.text_table() tb.set_html_table_attributes("border=1") c = pdk_db.execute( "SELECT test_run, status, key_id FROM result_scalar WHERE " "test_name = :1 AND context = :2 AND host = :3 AND project = :4 ORDER BY test_run DESC", (test_name, context, host, project)) row = 0 for x in c: r_test_run, status, key_id = x tb.set_value(row, 1, r_test_run, link=common.selflink({'key_id': key_id}, linkmode='detail')) tb.set_value(row, 2, status) if test_run == r_test_run: tb.set_value(row, 0, '->') row = row + 1 print(tb.get_html())
def test_history(): sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) form = pandokia.pcgi.form test_name = form.getvalue("test_name") context = form.getvalue("context") host = form.getvalue("host") test_run = form.getvalue("test_run") project = form.getvalue("project") tb = text_table.text_table() tb.set_html_table_attributes("border=1") row = 0 tb.set_value(row, 0, "project") tb.set_value(row, 1, project) row += 1 tb.set_value(row, 0, "host") tb.set_value(row, 1, host) row += 1 tb.set_value(row, 0, "context") tb.set_value(row, 1, context) row += 1 tb.set_value(row, 0, "test_name") tb.set_value(row, 1, test_name) row += 1 print(tb.get_html()) print("<br>") tb = text_table.text_table() tb.set_html_table_attributes("border=1") c = pdk_db.execute( "SELECT test_run, status, key_id FROM result_scalar WHERE " "test_name = :1 AND context = :2 AND host = :3 AND project = :4 ORDER BY test_run DESC", (test_name, context, host, project)) row = 0 for x in c: r_test_run, status, key_id = x tb.set_value(row, 1, r_test_run, link=common.selflink( {'key_id': key_id}, linkmode='detail')) tb.set_value(row, 2, status) if test_run == r_test_run: tb.set_value(row, 0, '->') row = row + 1 print(tb.get_html())
def expected(): import pandokia.text_table as text_table pdk_db = pandokia.cfg.pdk_db input_query = pandokia.pcgi.form_to_dict(pandokia.pcgi.form) if 'format' in input_query: format = input_query['format'][0] else: format = 'html' tbl = text_table.text_table() q = {} for x in ('test_run_type', 'project', 'host', 'context'): if x in input_query: q[x] = input_query[x] where_str, where_dict = pdk_db.where_dict(q) c_t = pdk_db.execute( "SELECT DISTINCT test_run_type FROM expected %s ORDER BY test_run_type " % where_str, where_dict) row = 0 for test_run_type, in c_t: prev_project = None tbl.set_value(row, 'test_run_type', test_run_type) row = row + 1 q['test_run_type'] = test_run_type where_str, where_dict = pdk_db.where_dict(q) c = pdk_db.execute( "SELECT DISTINCT project, host, context, count(*) FROM " "expected %s GROUP BY project, host, context ORDER BY project, host, context " % where_str, where_dict) for x in c: if x[0] != prev_project: tbl.set_value(row, 'project', x[0]) prev_project = x[0] for number, name in enumerate(('host', 'context', 'count')): tbl.set_value(row, name, x[number + 1]) row = row + 1 tbl.pad() if format == 'html': print("content-type: text/html\n") print("<h2>Expected test summary</h2>") tbl.set_html_table_attributes(' border=1 ') print(tbl.get_html(headings=1)) else: print("content-type: text/plain\n") print(tbl.get(format=format, headings=1))
def suppress_attr_all_same(result_table, column_select_values=set({})): # try to suppress attribute columns where all the data values are the # same global any_attr any_attr = sorted(any_attr) same_table = text_table.text_table() same_table.set_html_table_attributes("border=1") same_row = 0 rowcount = result_table.get_row_count() for x in any_attr: if x not in column_select_values: continue all_same = 1 txt = result_table._row_col_cell(0, x).text for y in range(1, rowcount): ntxt = result_table._row_col_cell(y, x).text if txt != ntxt: all_same = 0 break if all_same: same_table.set_value(same_row, 0, x) same_table.set_value(same_row, 1, txt) same_row = same_row + 1 result_table.suppress(x) return same_table
def suppress_attr_all_same(result_table, column_select_values=set({})): # try to suppress attribute columns where all the data values are the # same global any_attr any_attr = sorted(any_attr) same_table = text_table.text_table() same_table.set_html_table_attributes("border=1") same_row = 0 rowcount = result_table.get_row_count() for x in any_attr: if x not in column_select_values: continue all_same = 1 txt = result_table._row_col_cell(0, x).text for y in range(1, rowcount): ntxt = result_table._row_col_cell(y, x).text if txt != ntxt: all_same = 0 break if all_same: same_table.set_value(same_row, 0, x) same_table.set_value(same_row, 1, txt) same_row = same_row + 1 result_table.suppress(x) return same_table
def report_html(db, run_name, info_callback=info_callback_status, hlevel=1): raw = raw_report(db, run_name, info_callback) s = StringIO() s.write('<h%d>%s</h%d>\n' % (hlevel, run_name, hlevel)) hlevel = hlevel + 1 if len(raw.host_failures) > 0: s.write('<h%d>hosts with failures</h%d>\n' % (hlevel, hlevel)) t = text_table.text_table() t.set_html_table_attributes('border=1') for x in sorted(raw.host_failures): h = '%d<br>\n' % len(raw.host_failures[x]) for table, cmd in raw.host_failures[x]: name = '%s/%s' % (table, cmd) h = h + '<a href="#row_%s">%s</a><br>\n' % (name, name) t.set_value(0, x, h, html=h) s.write(t.get_html()) for header, body in zip(raw.table_header, raw.table_body): s.write('<h%d>%s</h%d>\n' % (hlevel, header, hlevel)) s.write(body.get_html()) return s.getvalue()
def report_html( db, run_name, info_callback = info_callback_status, hlevel=1 ) : raw = raw_report( db, run_name, info_callback ) s = StringIO() s.write('<h%d>%s</h%d>\n'%(hlevel,run_name,hlevel)) hlevel = hlevel + 1 if len(raw.host_failures) > 0 : s.write('<h%d>hosts with failures</h%d>\n'%(hlevel,hlevel)) t = text_table.text_table() t.set_html_table_attributes('border=1') for x in sorted(raw.host_failures) : h = '%d<br>\n'%len(raw.host_failures[x]) for table, cmd in raw.host_failures[x] : name = '%s/%s'%(table,cmd) h = h + '<a href="#row_%s">%s</a><br>\n'%(name,name) t.set_value(0,x,h, html=h) s.write(t.get_html()) for header, body in zip( raw.table_header, raw.table_body ) : s.write('<h%d>%s</h%d>\n'%(hlevel,header,hlevel)) s.write(body.get_html()) return s.getvalue()
def expected(): import pandokia.text_table as text_table pdk_db = pandokia.cfg.pdk_db input_query = pandokia.pcgi.form_to_dict(pandokia.pcgi.form) if 'format' in input_query: format = input_query['format'][0] else: format = 'html' tbl = text_table.text_table() q = {} for x in ('test_run_type', 'project', 'host', 'context', 'custom'): if x in input_query: q[x] = input_query[x] where_str, where_dict = pdk_db.where_dict(q) c_t = pdk_db.execute( "SELECT DISTINCT test_run_type FROM expected %s ORDER BY test_run_type " % where_str, where_dict) row = 0 for test_run_type, in c_t: prev_project = None tbl.set_value(row, 'test_run_type', test_run_type) row = row + 1 q['test_run_type'] = test_run_type where_str, where_dict = pdk_db.where_dict(q) c = pdk_db.execute( "SELECT DISTINCT project, host, context, custom, count(*) FROM " "expected %s GROUP BY project, host, context, custom ORDER BY project, host, context " % where_str, where_dict) for x in c: if x[0] != prev_project: tbl.set_value(row, 'project', x[0]) prev_project = x[0] for number, name in enumerate(('host', 'context', 'custom', 'count')): tbl.set_value(row, name, x[number + 1]) row = row + 1 tbl.pad() if format == 'html': print("content-type: text/html\n") print("<h2>Expected test summary</h2>") tbl.set_html_table_attributes(' border=1 ') print(tbl.get_html(headings=1)) else: print("content-type: text/plain\n") print(tbl.get(format=format, headings=1))
def get_table(db, run_name, tablename, info_callback, showdepth=0): t = text_table.text_table() t.set_html_table_attributes('border=1') t.define_column('-', html=' ', showname='-') # the command name in column 0 if showdepth: t.define_column('depth') c = db.cursor() c.execute( "select distinct host from sm_status where tablename = ? and run = ? order by host asc", (tablename, run_name)) for host, in c: t.define_column(host) c.execute( "select cmd, max(depth) as d from sm_status where tablename = ? and run = ? group by cmd order by d asc", (tablename, run_name)) row = -1 cmd_to_row = {} for cmd, depth in c: row = row + 1 cmd_to_row[cmd] = row cmd_html = '<a name="row_%s/%s"></a> %s' % (tablename, cmd, cmd) t.set_value(row, 0, cmd, html=cmd_html) if showdepth: t.set_value(row, 'depth', depth) c.execute( """select cmd, host, status, start_time, end_time, notes from sm_status where tablename = ? and run = ? order by cmd asc """, (tablename, run_name)) t.sm_host_failures = {} # list of failures on this host row = 0 for x in c: cmd, host, status, start_time, end_time, notes = x try: if (status in ('E', 'P')) or (int(status) != 0): t.sm_host_failures[host] = t.sm_host_failures.get( host, []) + [(tablename, cmd)] except ValueError: pass row = cmd_to_row[cmd] info = info_callback(db, run_name, tablename, host, cmd) if 'html' in info: t.set_value(row, host, text=info['text'], html=info['html']) else: t.set_value(row, host, text=info['text']) t.pad() return t
def list_users(): # Make sure that every user in he user_email_pref table also has a # a user name in user_prefs table. We can probably find a better place # to put this. c = cfg.pdk_db.execute( " SELECT DISTINCT username FROM user_email_pref WHERE " " username NOT IN ( SELECT username FROM user_prefs ) ") for x, in c: print("user %s not in user_prefs table - adding<br>" % cgi.escape(x)) cfg.pdk_db.execute( "INSERT INTO user_prefs ( username ) VALUES ( :1 )", (x,)) cfg.pdk_db.commit() # Make a table showing all the user prefs. tb = text_table.text_table() tb.define_column('username') tb.define_column('email') row = 0 # find all the project names that anybody knows about. This list is # only chosen from those projects that somebody has asked to get email # about, so it can look in the user preferences instead of searching all # the results for project names. project = [] c = cfg.pdk_db.execute( "SELECT DISTINCT project FROM user_email_pref ORDER BY project") for (x,) in c: project.append(x) tb.define_column('p.' + x, showname=x) # for each user, add a row to the table c = cfg.pdk_db.execute( "SELECT username, email FROM user_prefs ORDER BY username") for user, email in c: # stuff the fixed material into the table tb.set_value(row, 0, user) tb.set_value(row, 1, email) # find for each project that this user has a preference about: c1 = cfg.pdk_db.execute( "SELECT project, format, maxlines FROM user_email_pref WHERE username = :1", (user, )) for p, f, m in c1: # stuff that preference into the table. if m is not None: f = '%s %s' % (f, m) tb.set_value(row, 'p.' + p, cgi.escape(f)) row = row + 1 tb.set_html_table_attributes('border=1') output.write(tb.get_html())
def list_users(): # Make sure that every user in he user_email_pref table also has a # a user name in user_prefs table. We can probably find a better place # to put this. c = cfg.pdk_db.execute( " SELECT DISTINCT username FROM user_email_pref WHERE " " username NOT IN ( SELECT username FROM user_prefs ) ") for x, in c: print("user %s not in user_prefs table - adding<br>" % html_escape(x)) cfg.pdk_db.execute( "INSERT INTO user_prefs ( username ) VALUES ( :1 )", (x,)) cfg.pdk_db.commit() # Make a table showing all the user prefs. tb = text_table.text_table() tb.define_column('username') tb.define_column('email') row = 0 # find all the project names that anybody knows about. This list is # only chosen from those projects that somebody has asked to get email # about, so it can look in the user preferences instead of searching all # the results for project names. project = [] c = cfg.pdk_db.execute( "SELECT DISTINCT project FROM user_email_pref ORDER BY project") for (x,) in c: project.append(x) tb.define_column('p.' + x, showname=x) # for each user, add a row to the table c = cfg.pdk_db.execute( "SELECT username, email FROM user_prefs ORDER BY username") for user, email in c: # stuff the fixed material into the table tb.set_value(row, 0, user) tb.set_value(row, 1, email) # find for each project that this user has a preference about: c1 = cfg.pdk_db.execute( "SELECT project, format, maxlines FROM user_email_pref WHERE username = :1", (user, )) for p, f, m in c1: # stuff that preference into the table. if m is not None: f = '%s %s' % (f, m) tb.set_value(row, 'p.' + p, html_escape(f)) row = row + 1 tb.set_html_table_attributes('border=1') output.write(tb.get_html())
def delete_are_you_sure(): form = pandokia.pcgi.form test_run = form.getfirst("test_run") project = form.getfirst('project', '*') context = form.getfirst('context', '*') custom = form.getfirst('custom', '*') host = form.getfirst('host', '*') sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) if pandokia.cleaner.check_valuable(test_run): print("valuable test run - There should not be a link that comes here") return print("Delete data for:<br>") tt = text_table.text_table() tt.set_html_table_attributes("border=1") tt.set_value(0, 0, 'test_run') tt.set_value(0, 1, test_run) tt.set_value(1, 0, 'project') tt.set_value(1, 1, project) tt.set_value(2, 0, 'host') tt.set_value(2, 1, host) tt.set_value(3, 0, 'context') tt.set_value(3, 1, context) tt.set_value(4, 0, 'custom') tt.set_value(4, 1, custom) print(tt.get_html()) print("<br>") where_str, where_dict = pdk_db.where_dict([('test_run', test_run), ('project', project), ('context', context), ('custom', custom), ('host', host)]) print("%s<br>" % where_str) print("%s<br>" % where_dict) c = pdk_db.execute('SELECT count(*) FROM result_scalar %s' % where_str, where_dict) (x, ) = c.fetchone() print("%d records<br>" % x) print('<a href="%s">Confirm delete</a>' % common.selflink( { 'test_run': test_run, 'project': project, 'context': context, 'custom': custom, 'host': host }, 'delete_run.conf'))
def delete_are_you_sure(): form = pandokia.pcgi.form test_run = form.getfirst("test_run") project = form.getfirst('project', '*') context = form.getfirst('context', '*') custom = form.getfirst('custom', '*') host = form.getfirst('host', '*') sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) if pandokia.cleaner.check_valuable(test_run): print("valuable test run - There should not be a link that comes here") return print("Delete data for:<br>") tt = text_table.text_table() tt.set_html_table_attributes("border=1") tt.set_value(0, 0, 'test_run') tt.set_value(0, 1, test_run) tt.set_value(1, 0, 'project') tt.set_value(1, 1, project) tt.set_value(2, 0, 'host') tt.set_value(2, 1, host) tt.set_value(3, 0, 'context') tt.set_value(3, 1, context) tt.set_value(4, 0, 'custom') tt.set_value(4, 1, custom) print(tt.get_html()) print("<br>") where_str, where_dict = pdk_db.where_dict( [('test_run', test_run), ('project', project), ('context', context), ('custom', custom), ('host', host)]) print("%s<br>" % where_str) print("%s<br>" % where_dict) c = pdk_db.execute( 'SELECT count(*) FROM result_scalar %s' % where_str, where_dict) (x,) = c.fetchone() print("%d records<br>" % x) print( '<a href="%s">Confirm delete</a>' % common.selflink( { 'test_run': test_run, 'project': project, 'context': context, 'custom': custom, 'host': host}, 'delete_run.conf'))
def get_table( db, run_name, tablename, info_callback, showdepth=0 ) : t = text_table.text_table() t.set_html_table_attributes('border=1') t.define_column('-',html=' ',showname='-') # the command name in column 0 if showdepth : t.define_column('depth') c = db.cursor() c.execute("select distinct host from sm_status where tablename = ? and run = ? order by host asc",(tablename, run_name)) for host, in c : t.define_column(host) c.execute("select cmd, max(depth) as d from sm_status where tablename = ? and run = ? group by cmd order by d asc",(tablename, run_name)) row = -1 cmd_to_row = { } for cmd, depth in c : row = row + 1 cmd_to_row[cmd] = row cmd_html = '<a name="row_%s/%s"></a> %s'%(tablename, cmd, cmd) t.set_value(row, 0, cmd, html=cmd_html) if showdepth : t.set_value(row, 'depth', depth) c.execute("""select cmd, host, status, start_time, end_time, notes from sm_status where tablename = ? and run = ? order by cmd asc """, ( tablename, run_name ) ) t.sm_host_failures = { } # list of failures on this host row = 0 for x in c : cmd, host, status, start_time, end_time, notes = x try : if ( status in ( 'E', 'P' ) ) or ( int(status) != 0 ) : t.sm_host_failures[host] = t.sm_host_failures.get(host,[]) + [ ( tablename, cmd ) ] except ValueError : pass row = cmd_to_row[cmd] info = info_callback( db, run_name, tablename, host, cmd ) if 'html' in info : t.set_value( row, host, text=info['text'], html=info['html'] ) else : t.set_value( row, host, text=info['text']) t.pad() return t
def print_conditions(): boring = {} exec('pass', boring) l = sorted([x for x in nodes.saved_conditions]) row = 0 tt = text_table.text_table() for x in l: if x in boring: continue if callable(x): v = x() else: v = nodes.saved_conditions[x] tt.set_value(row, 0, x) tt.set_value(row, 1, str(v)) row = row + 1 print(tt.get_rst())
def print_conditions() : boring = { } exec('pass', boring) l = sorted( [ x for x in nodes.saved_conditions ] ) row = 0 tt = text_table.text_table() for x in l : if x in boring : continue if callable(x) : v = x() else : v = nodes.saved_conditions[x] tt.set_value(row,0,x) tt.set_value(row,1,str(v)) row = row + 1 print(tt.get_rst())
def report_text(db, run_name, info_callback=info_callback_status): raw = raw_report(db, run_name, info_callback) s = StringIO() table_list = get_table_list(db, run_name) if len(raw.host_failures) > 0: s.write('hosts with failures:\n') t = text_table.text_table() for x in sorted(raw.host_failures): t.set_value(0, x, raw.host_failures[x]) s.write(t.get_track_wiki(headings=True)) for header, body in zip(raw.table_header, raw.table_body): s.write("------\n") s.write(header) s.write('\n') s.write(body.get_trac_wiki(headings=True)) return s.getvalue()
def report_text( db, run_name, info_callback = info_callback_status ) : raw = raw_report( db, run_name, info_callback ) s = StringIO() table_list = get_table_list(db, run_name) if len(raw.host_failures) > 0 : s.write('hosts with failures:\n') t = text_table.text_table() for x in sorted(raw.host_failures) : t.set_value(0,x,raw.host_failures[x]) s.write(t.get_track_wiki(headings=True)) for header, body in zip( raw.table_header, raw.table_body ) : s.write("------\n") s.write(header) s.write('\n') s.write( body.get_trac_wiki(headings=True) ) return s.getvalue()
def qid_list(): import datetime print "content-type: text/html\n" print common.page_header() print "<h1>Interesting QID</h1>" input_query = pandokia.pcgi.form_to_dict(pandokia.pcgi.form) order_by = 'qid' order_dir = 'asc' if 'order_by' in input_query: x = input_query['order_by'][0] if x in ('qid', 'expires', 'username', 'notes'): order_by = x if 'order_dir' in input_query: x = input_query['order_dir'][0] if x in ('asc', 'desc'): order_dir = x def sl(s): d = 'asc' if order_dir == 'asc': d = 'desc' return common.selflink({ 'order_by': s, 'order_dir': d }, linkmode='qid_list') t = text_table.text_table() t.define_column("QID", html='<a href=%s>QID</a>' % sl('qid')) t.define_column("Expires", html='<a href=%s>expires</a>' % sl('expires')) t.define_column("User", html='<a href=%s>username</a>' % sl('username')) t.define_column("Notes", html='<a href=%s>notes</a>' % sl('notes')) t.set_html_table_attributes("border=1") qdict = {} c = pdk_db.execute( "SELECT qid, expires, username, notes FROM query_id WHERE expires = :1 OR username <> '' OR notes <> '' ORDER BY %s %s" % (order_by, order_dir), (pandokia.never_expires, )) row = 0 for x in c: v = int(x[0]) qdict['qid'] = v t.set_value(row, 0, v, html='<a href="%s">%s</a>' % (common.selflink(qdict, linkmode="summary"), v)) try: v = int(x[1]) if v == pandokia.never_expires: v = 'never' else: v = str(datetime.date.fromtimestamp(v)) except: v = '?' t.set_value(row, 1, v) v = str(x[2]) t.set_value(row, 2, v, html=cgi.escape(v)) if x[3] is None: v = '' else: v = str(x[3]) t.set_value(row, 3, v, html='<pre>' + cgi.escape(v) + '</pre>') row = row + 1 print t.get_html()
def rpt1(): form = pandokia.pcgi.form if "test_run" in form: test_run = form.getvalue("test_run") else: test_run = '*' if test_run == '-me': test_run = 'user_' + common.current_user() + '_*' my_run_prefix = 'user_' + common.current_user() admin = common.current_user() in common.cfg.admin_user_list # c = db.execute("SELECT DISTINCT test_run FROM result_scalar WHERE test_run GLOB ? ORDER BY test_run DESC ",( test_run,)) where_str, where_dict = pdk_db.where_dict([('test_run', test_run)]) sql = "SELECT test_run, valuable, record_count, note, min_time, max_time FROM distinct_test_run %s ORDER BY test_run DESC " % where_str c = pdk_db.execute(sql, where_dict) table = text_table.text_table() table.set_html_table_attributes("border=1") table.define_column('addval', showname='') table.define_column('run', showname='test_run') table.define_column('tree', showname='') table.define_column('del', showname='') table.define_column('min', showname='start') table.define_column('max', showname='end') table.define_column('tdiff', showname='duration') table.define_column('count', showname='records') table.define_column('note', showname='note') # query parameters for various links # link to day_report # link to tree walk of test run tquery = {'project': '*', 'host': '*'} # link to declare a run as valuable vquery = {'valuable_run': 1} # link to update the count in a test run cquery = {} row = 0 for x, val, record_count, note, min_time, max_time in c: if x is None: continue tquery["test_run"] = x vquery["test_run"] = x cquery["count_run"] = x # mark as valuable: # https://ssb.stsci.edu/pandokia/c41.cgi?query=action&test_run=daily_2011-08-24&valuable_run=1 table.set_value( row, 'addval', html='<a href="%s">!</a> ' % common.selflink( vquery, "action")) table.set_value( row, 'run', text=x, link=common.selflink( tquery, "day_report.2")) table.set_value( row, 'tree', text='(tree display)', link=common.selflink( tquery, "treewalk")) if val == '0': if x.startswith(my_run_prefix): table.set_value( row, 'del', text='(delete)', link=common.selflink( tquery, "delete_run.ays")) else: table.set_value( row, 'del', text='(delete)', html='<font color=gray>(delete)</font>', link=common.selflink( tquery, "delete_run.ays")) else: table.set_value(row, 'del', text='(valuable)') if note is None: table.set_value(row, 'note', text='') else: table.set_value(row, 'note', text=note) if min_time is not None and max_time is not None: table.set_value( row, 'tdiff', str( lib.time_diff( max_time, min_time))) min_time = str(min_time).split('.')[0] max_time = str(max_time).split('.')[0] t1 = min_time.split() t2 = max_time.split() if len(t2) > 1 and len(t1) > 1 and t1[0] == t2[0] and len(t2) > 1: max_time = t2[1] table.set_value(row, 'min', text=min_time) table.set_value(row, 'max', text=max_time) else: if min_time is not None: min_time = str(min_time).split('.')[0] table.set_value(row, 'min', text=min_time) if max_time is not None: max_time = str(max_time).split('.')[0] table.set_value(row, 'max', text=max_time) table.set_value(row, 'tdiff', '') # update the count field # https://ssb.stsci.edu/pandokia/c41.cgi?query=action&count_run=daily_2011-08-24 update_count = common.selflink(cquery, 'action') if record_count is None or record_count <= 0: record_count = ' ' table.set_value( row, 'count', html=str(record_count), link=update_count) row = row + 1 if pandokia.pcgi.output_format == 'html': sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) sys.stdout.write('<h2>%s</h2>' % cgi.escape(test_run)) sys.stdout.write(table.get_html(headings=1)) sys.stdout.write( "<br>Click on the ! to mark a test run as too valuable to delete\n") sys.stdout.write( "<br>Click on record count to check the count and update it\n") sys.stdout.write("""<style> table { border-collapse: collapse; } table, th, td { border: 2px solid black; padding: 3px; } </style> """) elif pandokia.pcgi.output_format == 'csv': sys.stdout.write(common.cgi_header_csv) sys.stdout.write(table.get_csv()) sys.stdout.flush() return
def dictionary_comp( data_dict, # the actual data file_base, # implies output file name # implies reference file name # implies ok file name # implies refs/file.fields for compare/ignore fields # assumes PDK_REFS environment tda=None, tra=None, # if present, fill tra with output, relative difference interesting_fields=None, # if interesting, compare only those uninteresting_fields=[], # if uninteresting, exclude those from the default list fp_tolerance=1e-7, # tolerance_dict={}, # you can give a specific tolerance for specific fields # default is no special preference print_info=False, # okfh=None, # fail_only=False, # if true, only report fields that are involved in a failure ): # Start off saying the whole comparison has not failed. This # will be the state until the end, unless some element fails. fail = False # The list of fields that had an error in the comparison. errors = [] # so we don't have to keep saying "if tda : tda[xx] = yy" # these dicts will go out of scope at the end of the function if tda is None: tda = {} if tra is None: tra = {} # a reference dict, generated by hand or from a previous result. # or nothing, if it is not there -- gets us through the test # BUG: support PDK_REFS ref_fn = "ref/" + file_base # Read the reference dictionary. no_reference_file will be the # exception if there is no reference file; it is re-raised at # the end. We don't just raise it now because we want all # the reporting that comes below. try: ref_dict = read_reference(ref_fn) no_reference_file = None except NoReferenceFile as e: ref_dict = {} no_reference_file = e # flatten the data dict (the ref dict is already flattened) data_dict = flatten(data_dict) # identify interesting fields. if explicitly specified, use # just that. Otherwise it defaults to all the fields except # those identified as uninteresting. if interesting_fields is None: interesting_fields = set([x for x in data_dict] + [x for x in ref_dict]) - set(uninteresting_fields) # save the interesting fields from the flattened dict out_fn = write_output(file_base, data_dict, interesting_fields) # add a line to the okfile if okfh: append_okfile(okfh, out_fn, ref_fn) # Which fields were in the expected answers, but missing from the result. missing = set() # Which fields did not compare closely enough. failed = [] # Report the default tolerance. tda['tolerance'] = fp_tolerance # Set up to report the max discrep if not fail_only: tra['max_discrep'] = 0 tra['max_discrep_var'] = None # Loop over the interesting keys, comparing each one. for k in interesting_fields: # flag of whether this iteration of the loop fails the compare this_fail = False # value of the discrepancy for this iteration of the loop discrep = None # see if there are custom tolerances for this field try: tol = tolerance_dict[k] tda['tol_%s' % k] = tol except KeyError: tol = fp_tolerance # pick out the data value try: data = data_dict[k] except AttributeError: tra['discrep_%s' % k] = 'missing data' this_fail = True missing.add(k) continue # pick out the reference value try: ref = ref_dict[k] except KeyError: tra['discrep_%s' % k] = 'missing ref' this_fail = True missing.add(k) continue # comparing numbers if isinstance( data, numbers.Number) and isinstance( ref, numbers.Number): # If both are zero, everything is good. if data == 0 and ref == 0: discrep = 0 else: # Compare the discrepancy as a proportion of the reference # value. try: discrep = (data - ref) / ref if (abs(discrep) > tol): failed.append(k) this_fail = True # If the reference is zero, use absolute difference instead. except ZeroDivisionError as e: discrep = (data - ref) if (abs(discrep) > tol): failed.append(k) this_fail = True # comparing strings elif isinstance(data, string_type) and isinstance(ref, string_type): if ref.strip() != data.strip(): failed.append(k) discrep = True this_fail = True # comparing non-number, non-string, non-dict, non-list # If you use json for your data, this just means booleans # or two different types else: # neither string nor number - all we have for comparing is equality if data != ref: failed.append(k) discrep = True this_fail = True # If this element of the data failed, the whole test failed. if this_fail: fail = True # Report this element in the tra if it failed, or if we are # reporting all elements. if this_fail or not fail_only: # Report the computed value and the reference value as tra. tra[k] = str(data) tra['ref_%s' % k] = str(ref) tra['discrep_%s' % k] = discrep # end: for k in interesting_fields # If any of the fields failed, report what they were. if len(failed) > 0: failed.sort() tra['failed'] = failed # For failed numerical values, report the max discrepancy and variable discrep, vname = 0.0, None for k in failed: val = abs(tra['discrep_%s' % k]) if (isinstance(val, float) and val > discrep): discrep = val vname = k if vname is not None: tra['max_discrep'] = discrep tra['max_discrep_var'] = vname # anything that wasn't there is also a problem if missing: m = sorted(missing) tra['missing'] = str(m) fail = True # This is a pretty display for the interactive user. if print_info and (fail or (not fail_only)): from pandokia.text_table import text_table tt = text_table() failed_set = set(failed) tt.define_column('name') tt.define_column('expect') tt.define_column('actual') tt.define_column('F') tt.define_column('discrep') for row, k in enumerate(sorted(interesting_fields)): tt.set_value(row, 'name', k) if k in ref_dict: tt.set_value(row, 'expect', ref_dict[k]) if k in data_dict: tt.set_value(row, 'actual', data_dict[k]) s = 'discrep_%s' % k if s in tra: tt.set_value(row, 'discrep', tra[s]) if k in failed_set: tt.set_value(row, 'F', '*') print(tt.get_rst(headings=1)) if missing: m = sorted(missing) print("Missing keys: %s" % m) if fail: print("FAILED") if no_reference_file: raise no_reference_file if len(errors): raise Exception(str(errors)) if fail: raise AssertionError('%s failed' % file_base)
def get_table(qid, sort_link, cmp_run, cmptype, show_attr): # # this query finds all the test results that are an interesting part of this request # result_table = text_table.text_table() result_table.set_html_table_attributes("border=1") tda_table = text_table.text_table() tda_table.set_html_table_attributes("border=1") tra_table = text_table.text_table() tra_table.set_html_table_attributes("border=1") # note when we last touched this qid now = time.time() pdk_db.execute("UPDATE query_id SET time = :1 WHERE qid = :2", (now, qid)) # bump the expiration maybe expires = now + (pandokia.cfg.default_qid_expire_days * 86400) pdk_db.execute( "UPDATE query_id SET expires = :1 WHERE qid = :2 AND :1 > query_id.expires", (expires, qid)) pdk_db.commit() # c = pdk_db.execute("SELECT key_id FROM query WHERE qid = :1", (qid,)) result_table.define_column("line #", showname=' ') result_table.define_column("runner") result_table.define_column("checkbox", showname=' ') result_table.define_column("attn", link=sort_link + "Uattn") result_table.define_column("test_run", link=sort_link + "Utest_run") result_table.define_column("project", link=sort_link + "Uproject") result_table.define_column("host", link=sort_link + "Uhost") result_table.define_column("context", link=sort_link + "Ucontext") result_table.define_column("custom", link=sort_link + "Ucustom") result_table.define_column("test_name", link=sort_link + "Utest_name") result_table.define_column("contact", link=sort_link + "Ucontact") #result_table.define_column("start", link=sort_link+"Ustart") result_table.define_column("duration", link=sort_link + "Uduration") if cmp_run != "": result_table.define_column("diff", link=sort_link + "Udiff") result_table.define_column("other", link=sort_link + "Uother") result_table.define_column("stat", link=sort_link + "Ustat") # these are used to suppress a column when all the results are the same all_test_run = {} all_project = {} all_host = {} all_context = {} all_custom = {} different = 0 rowcount = 0 for x in c: (key_id, ) = x # # find the result of this test # c1 = pdk_db.execute( "SELECT test_run, project, host, context, custom, test_name, status, attn, test_runner, start_time, end_time FROM result_scalar WHERE key_id = :1 ", (key_id, )) y = c1.fetchone() # unique index if y is None: # this can only happen if somebody deletes tests from the database # after we populate the qid continue (test_run, project, host, context, custom, test_name, status, attn, runner, start_time, end_time) = y # if we are comparing to another run, find the other one; # suppress lines that are different - should be optional if cmp_run != "": c2 = pdk_db.execute( "SELECT status, key_id FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND test_name = :4 AND context = :5 AND custom = :6", (cmp_run, project, host, test_name, context, custom)) other_status = c2.fetchone() # unique index if other_status is None: pass else: (other_status, other_key_id) = other_status # if the other one is the same, go to next row if other_status == status: if cmptype == 'd': continue else: if cmptype == 's': continue result_table.set_value(rowcount, "diff", text=">") other_link = common.selflink( {'key_id': other_key_id}, linkmode="detail") if other_status == "P": result_table.set_value( rowcount, "other", other_status, link=other_link) else: result_table.set_value( rowcount, "other", other_status, html="<font color=red>" + str(other_status) + "</font>", link=other_link) result_table.set_html_cell_attributes( rowcount, "other", "bgcolor=lightgray") if other_status != status: different = different + 1 all_test_run[test_run] = 1 all_project[project] = 1 all_host[host] = 1 all_context[context] = 1 all_custom[custom] = 1 detail_query = {"key_id": key_id} result_table.set_value(rowcount, "runner", runner) result_table.set_value( rowcount, "checkbox", '', html='<input type=checkbox name=%s>' % key_id) result_table.set_value(rowcount, "attn", attn) result_table.set_value(rowcount, "test_run", test_run) result_table.set_value(rowcount, "project", project) result_table.set_value(rowcount, "host", host) result_table.set_value(rowcount, "context", context) result_table.set_value(rowcount, "custom", custom) this_link = common.selflink(detail_query, linkmode="detail") result_table.set_value( rowcount, "test_name", text=test_name, link=this_link) result_table.set_value( rowcount, "contact", common.get_contact( project, test_name, 'str')) start_time = lib.decode_time_float(start_time) end_time = lib.decode_time_float(end_time) #result_table.set_value(rowcount,"start", lib.decode_time_str(start_time)) if start_time is not None and end_time is not None: result_table.set_value( rowcount, "duration", "%.3f" % (end_time - start_time)) if status == "P": result_table.set_value(rowcount, "stat", status, link=this_link) else: result_table.set_value( rowcount, "stat", status, html="<font color=red>" + str(status) + "</font>", link=this_link) if show_attr: c3 = pdk_db.execute( "SELECT name, value FROM result_tda WHERE key_id = :1 ORDER BY name ASC", (key_id, )) load_in_table(tda_table, rowcount, c3, "tda_", sort_link) del c3 c3 = pdk_db.execute( "SELECT name, value FROM result_tra WHERE key_id = :1 ORDER BY name ASC", (key_id, )) load_in_table(tra_table, rowcount, c3, "tra_", sort_link) del c3 rowcount += 1 del c1 if show_attr: result_table.join(tda_table) result_table.join(tra_table) return result_table, all_test_run, all_project, all_host, all_context, all_custom, rowcount, different
def do_result(key_id): c = pdk_db.execute( "SELECT key_id, test_run, project, host, context, test_name, status, attn, start_time, end_time, location, test_runner FROM result_scalar WHERE key_id = :1 ", (key_id, )) result_count = 0 for x in c: result_count = result_count + 1 (key_id, test_run, project, host, context, test_name, status, attn, start_time, end_time, location, test_runner) = (x) linkback_dict = { 'project': project, 'host': host, 'context': context, 'test_run': test_run, 'test_name': test_name} prevday_dict = { 'project': project, 'host': host, 'context': context, 'test_run': test_run, 'test_name': test_name} row = 0 tb = text_table.text_table() tb.set_html_table_attributes("border=1") tb.set_value(row, 0, "test_run") tmp = next_prev(prevday_dict, test_run) if tmp != '': tb.set_value(row, 1, test_run, html=test_run + tmp) else: tb.set_value(row, 1, test_run) row += 1 tb.set_value(row, 0, "key_id") tb.set_value(row, 1, key_id) row += 1 tb.set_value(row, 0, "project") tb.set_value(row, 1, project) row += 1 tb.set_value(row, 0, "host") tb.set_value(row, 1, host) row += 1 tb.set_value(row, 0, "context") tb.set_value(row, 1, context) row += 1 tb.set_value(row, 0, "test_name") tb.set_value(row, 1, test_name) row += 1 tb.set_value(row, 0, "contact") tb.set_value(row, 1, common.get_contact(project, test_name)) row += 1 tb.set_value(row, 0, "status") if status != "P": tb.set_value( row, 1, text=status, html="<font color=red>" + status + "</font>") else: tb.set_value(row, 1, status) row += 1 tb.set_value(row, 0, "attn") tb.set_value(row, 1, attn) row += 1 tb.set_value(row, 0, "test_runner") tb.set_value(row, 1, test_runner) row += 1 try: # ignoring exceptions here because we don't know how good the values in # the database are. some preliminary software does not fill them in # correctly, for example. tb.set_value(row, 1, float(end_time) - float(start_time)) tb.set_value(row, 0, "duration") row += 1 except: pass # merge all of this into a generalized "get the times" function that # can be used anywhere if start_time == '0': st = 'unknown' else: try: # you might think that you could use .%f in strftime, but you would be wrong # the documentation describes it, but it isn't implemented st = datetime.datetime.fromtimestamp(float(start_time)) st = st.strftime("%Y-%m-%d %H:%M:%S") + "." + \ "%03d" % (st.microsecond / 1000) except: # it is some non-time_t format - just display it st = start_time if end_time == '0': et = 'unknown' else: try: # you might think that you could use .%f in strftime, but you would be wrong # the documentation describes it, but it isn't implemented et = datetime.datetime.fromtimestamp(float(end_time)) et = et.strftime("%Y-%m-%d %H:%M:%S") + "." + \ "%03d" % (et.microsecond / 1000) except: # it is some non-time_t format - just display it et = end_time tb.set_value(row, 0, "start_time") tb.set_value(row, 1, st) row += 1 tb.set_value(row, 0, "end_time") tb.set_value(row, 1, et) row += 1 tb.set_value(row, 0, "location") tb.set_value(row, 1, location) row += 1 c1 = pdk_db.execute( "SELECT name, value FROM result_tda WHERE key_id = :1 ORDER BY name ASC", (key_id, )) for y in c1: (name, value) = y tb.set_value(row, 0, "tda_" + name) tb.set_value(row, 1, value) row += 1 c1 = pdk_db.execute( "SELECT name, value FROM result_tra WHERE key_id = :1 ORDER BY name ASC", (key_id, )) for y in c1: (name, value) = y tb.set_value(row, 0, "tra_" + name) if 'xception' in name: tb.set_value(row, 1, value, code=True) else: tb.set_value(row, 1, value) row += 1 sys.stdout.write(tb.get_html()) sys.stdout.write( "<a href=%s>back to treewalk</a><br>\n" % common.selflink( linkback_dict, linkmode='treewalk')) sys.stdout.write( "<a href=%s>status history</a><br>\n" % common.selflink( linkback_dict, linkmode='test_history')) c1 = pdk_db.execute( "SELECT log FROM result_log WHERE key_id = :1 ", (key_id, )) for y in c1: if not PY3: if isinstance(y, tuple): y = tuple(x.decode() for x in y) (y, ) = y if y != "": if getattr(cfg, 'enable_magic_html_log'): if '<!DOCTYPE' in y or '<html' in y: sys.stdout.write( "<a href=%s>HTML block in a single page</a><br>" % common.selflink( { 'magic_html_log': key_id, }, linkmode='magic_html_log')) sys.stdout.write("Log:<br><pre>") sys.stdout.write(cgi.escape(y)) sys.stdout.write("</pre>\n") sys.stdout.write("<br>\n") sys.stdout.write("<br><hr>\n") return result_count
def get_table(qid, sort_link, cmp_run, cmptype, show_attr): # # this query finds all the test results that are an interesting part of this request # result_table = text_table.text_table() result_table.set_html_table_attributes("border=1") tda_table = text_table.text_table() tda_table.set_html_table_attributes("border=1") tra_table = text_table.text_table() tra_table.set_html_table_attributes("border=1") # note when we last touched this qid now = time.time() pdk_db.execute("UPDATE query_id SET time = :1 WHERE qid = :2", (now, qid)) # bump the expiration maybe expires = now + (pandokia.cfg.default_qid_expire_days * 86400) pdk_db.execute( "UPDATE query_id SET expires = :1 WHERE qid = :2 AND :1 > query_id.expires", (expires, qid)) pdk_db.commit() # c = pdk_db.execute("SELECT key_id FROM query WHERE qid = :1", (qid,)) result_table.define_column("line #", showname=' ') result_table.define_column("runner") result_table.define_column("checkbox", showname=' ') result_table.define_column("attn", link=sort_link + "Uattn") result_table.define_column("test_run", link=sort_link + "Utest_run") result_table.define_column("project", link=sort_link + "Uproject") result_table.define_column("host", link=sort_link + "Uhost") result_table.define_column("context", link=sort_link + "Ucontext") result_table.define_column("test_name", link=sort_link + "Utest_name") result_table.define_column("contact", link=sort_link + "Ucontact") #result_table.define_column("start", link=sort_link+"Ustart") result_table.define_column("duration", link=sort_link + "Uduration") if cmp_run != "": result_table.define_column("diff", link=sort_link + "Udiff") result_table.define_column("other", link=sort_link + "Uother") result_table.define_column("stat", link=sort_link + "Ustat") # these are used to suppress a column when all the results are the same all_test_run = {} all_project = {} all_host = {} all_context = {} different = 0 rowcount = 0 for x in c: (key_id, ) = x # # find the result of this test # c1 = pdk_db.execute( "SELECT test_run, project, host, context, test_name, status, attn, test_runner, start_time, end_time FROM result_scalar WHERE key_id = :1 ", (key_id, )) y = c1.fetchone() # unique index if y is None: # this can only happen if somebody deletes tests from the database # after we populate the qid continue (test_run, project, host, context, test_name, status, attn, runner, start_time, end_time) = y # if we are comparing to another run, find the other one; # suppress lines that are different - should be optional if cmp_run != "": c2 = pdk_db.execute( "SELECT status, key_id FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND test_name = :4 AND context = :5", (cmp_run, project, host, test_name, context)) other_status = c2.fetchone() # unique index if other_status is None: pass else: (other_status, other_key_id) = other_status # if the other one is the same, go to next row if other_status == status: if cmptype == 'd': continue else: if cmptype == 's': continue result_table.set_value(rowcount, "diff", text=">") other_link = common.selflink( {'key_id': other_key_id}, linkmode="detail") if other_status == "P": result_table.set_value( rowcount, "other", other_status, link=other_link) else: result_table.set_value( rowcount, "other", other_status, html="<font color=red>" + str(other_status) + "</font>", link=other_link) result_table.set_html_cell_attributes( rowcount, "other", "bgcolor=lightgray") if other_status != status: different = different + 1 all_test_run[test_run] = 1 all_project[project] = 1 all_host[host] = 1 all_context[context] = 1 detail_query = {"key_id": key_id} result_table.set_value(rowcount, "runner", runner) result_table.set_value( rowcount, "checkbox", '', html='<input type=checkbox name=%s>' % key_id) result_table.set_value(rowcount, "attn", attn) result_table.set_value(rowcount, "test_run", test_run) result_table.set_value(rowcount, "project", project) result_table.set_value(rowcount, "host", host) result_table.set_value(rowcount, "context", context) this_link = common.selflink(detail_query, linkmode="detail") result_table.set_value( rowcount, "test_name", text=test_name, link=this_link) result_table.set_value( rowcount, "contact", common.get_contact( project, test_name, 'str')) start_time = lib.decode_time_float(start_time) end_time = lib.decode_time_float(end_time) #result_table.set_value(rowcount,"start", lib.decode_time_str(start_time)) if start_time is not None and end_time is not None: result_table.set_value( rowcount, "duration", "%.3f" % (end_time - start_time)) if status == "P": result_table.set_value(rowcount, "stat", status, link=this_link) else: result_table.set_value( rowcount, "stat", status, html="<font color=red>" + str(status) + "</font>", link=this_link) if show_attr: c3 = pdk_db.execute( "SELECT name, value FROM result_tda WHERE key_id = :1 ORDER BY name ASC", (key_id, )) load_in_table(tda_table, rowcount, c3, "tda_", sort_link) del c3 c3 = pdk_db.execute( "SELECT name, value FROM result_tra WHERE key_id = :1 ORDER BY name ASC", (key_id, )) load_in_table(tra_table, rowcount, c3, "tra_", sort_link) del c3 rowcount += 1 del c1 if show_attr: result_table.join(tda_table) result_table.join(tra_table) return result_table, all_test_run, all_project, all_host, all_context, rowcount, different
def sql_commands(self, s, format='rst'): # s is a single string, maybe with newlines. Split it into a # list of lines. s = s.split('\n') # tear comments out of all lines. hmmm... don't use -- in a # string constant... import re comment = re.compile('--.*$') s = [comment.sub('', x).strip() for x in s] # turns on and off for conditional processing active = True c = '' line = 0 for x in s: line = line + 1 if x == '': continue # This implements the ++driver convention. A line that # begins with ++name begins a section that is only performed # when using that database driver. A line that contains just ++ # beings a section that is performed for all drivers. if x.startswith("++"): x = x[2:].split() if len(x) == 0: active = True continue if self.pandokia_driver_name in x: active = True continue active = False continue if not active: continue # gather the line c = c + x + '\n' # end of a command. if c.endswith(';\n'): # perform the command cursor = self.execute(c) # display the result tbl = text_table.text_table() # define columns, if we know them if cursor.description: for name in cursor.description: name = name[0] tbl.define_column(name) # Display the results, if any try: # fill the table cells for rownum, rowval in enumerate(cursor): for colnum, colval in enumerate(rowval): tbl.set_value(rownum, colnum, colval) if len(tbl.rows) > 0: # show the table in the format the user asked print(tbl.get(format=format, headings=1)) except self.ProgrammingError as e: if 'no results to fetch' in str(e): pass else: print("Programming Error for %s" % c) print(e) except self.IntegrityError as e: print("Integrity Error for %s" % c) print(e) cursor.close() c = '' self.commit()
def gen_daily_table(test_run, projects, query_context, query_host, valuable=0, chronic=False): # chronic is true if we should do the chronic column, false otherwise # convert special names, e.g. daily_latest to the name of the latest daily_* test_run = common.find_test_run(test_run) # show_delete = not valuable if show_delete: my_run_prefix = 'user_' + common.current_user() if test_run.startswith(my_run_prefix): del_text = '(del)' else: if common.current_user() in common.cfg.admin_user_list: del_text = '<font color=gray>(del)</font>' else: show_delete = False # this is the skeleton of the cgi queries for various links query = {"test_run": test_run} # This is a single table for all projects, because we want the # pass/fail/error columns to be aligned from one to the next # table = text_table.text_table() # The texttable object doesn't understand colspans, but we hack a # colspan into it anyway. Thee result is ugly if you have borders. table.set_html_table_attributes("cellpadding=2 ") status_types = common.cfg.statuses row = 0 table.define_column("host") table.define_column("context") table.define_column("os") table.define_column("total") for x in status_types: table.define_column(x) if chronic: table.define_column("chronic") table.define_column("del") table.define_column("note") n_cols = 3 + len(status_types) + 2 # # # # # # # # # # # loop across hosts prev_project = None all_sum = {'total': 0, 'chronic': 0} for status in status_types: all_sum[status] = 0 hc_where, hc_where_dict = pdk_db.where_dict([('test_run', test_run), ('project', projects), ('context', query_context), ('host', query_host)]) c = pdk_db.execute( "SELECT DISTINCT project, host, context FROM result_scalar %s ORDER BY project, host, context " % hc_where, hc_where_dict) if chronic: chronic_str = "AND ( chronic = '0' or chronic IS NULL )" else: chronic_str = "" # now we forget the list of projects that came in and construct a # list of projects that we actually saw. projects = [] for project, host, context in c: ## make a new project section of the table if project != prev_project: projects.append(project) table.set_value(row, 0, "") prev_project = project # values common to all the links we will write in this pass through the loop query["project"] = project query["host"] = "%" # this link text is common to all the links for this project link = common.selflink(query_dict=query, linkmode="treewalk") # the heading for a project subsection of the table project_text = cgi.escape(project) project_text = '<hr><big><strong><b><a name="%s" href="%s">%s</a></b></strong></big>' % ( project_text, link, project_text) table.set_value(row, 0, text=project, html=project_text) table.set_html_cell_attributes(row, 0, "colspan=%d" % n_cols) row += 1 # the column headings for this project's part of the table insert_col_headings(table, row, link, chronic) row += 1 # This will be the sum of all the tests in a particular project. # It comes just under the headers for the project, but we don't # know the values until the end. project_sum = {'total': 0, 'chronic': 0} for status in status_types: project_sum[status] = 0 project_sum_row = row # delete entire project from the test run if show_delete: table.set_value(row, 'del', html=del_text, link=delete_link( { 'test_run': test_run, 'project': project }, show_delete)) row += 1 prev_host = None # a new host/contest line query["host"] = host link = common.selflink(query_dict=query, linkmode="treewalk") if host != prev_host: table.set_value(row, 0, text=host, link=link) prev_host = host query['context'] = context link = common.selflink(query_dict=query, linkmode="treewalk") del query['context'] table.set_value(row, 1, text=context, link=link) # delete entire project from the test run if show_delete: table.set_value(row, 'del', html=del_text, link=delete_link( { 'test_run': test_run, 'project': project, 'host': host, 'context': context }, show_delete)) # show the host info hinfo = pandokia.common.hostinfo(host) table.set_value(row, 2, text=hinfo[1], link=common.selflink(query_dict={'host': host}, linkmode='hostinfo')) total_results = 0 missing_count = 0 for status in status_types: # use "or chronic is null" until we update the database for the new schema. c1 = pdk_db.execute( "SELECT COUNT(*) FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND status = :4 AND context = :5 %s" % (chronic_str, ), (test_run, project, host, status, context)) (x, ) = c1.fetchone() total_results += x project_sum[status] += x all_sum[status] += x if x == 0: table.set_value(row, status, text='0') else: table.set_value(row, status, text=str(x), link=link + "&status=" + status) table.set_html_cell_attributes(row, status, 'align="right"') if status == 'M': missing_count = x if chronic: c1 = pdk_db.execute( "SELECT COUNT(*) FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND context = :5 AND chronic = '1'", (test_run, project, host, status, context)) (x, ) = c1.fetchone() total_results += x project_sum['chronic'] += x all_sum['chronic'] += x table.set_value(row, 'chronic', text=str(x), link=link + "&chronic=1") table.set_html_cell_attributes(row, 'chronic', 'align="right"') project_sum['total'] += total_results all_sum['total'] += total_results if 0 and 'M' in status_types: # not sure how to deal with this in the presence of chronic problem tests if missing_count != 0: if missing_count == total_results: # if it equals the total, then everything is missing; we make a note of that table.set_value(row, 'note', 'all missing') else: # only some of them are missing table.set_value(row, 'note', 'some missing') table.set_value(row, 'total', text=str(total_results), link=link) table.set_html_cell_attributes(row, 'total', 'align="right"') row = row + 1 for status in status_types: table.set_value(project_sum_row, status, project_sum[status]) table.set_html_cell_attributes( project_sum_row, status, 'align="right" style="font-weight:bold"') table.set_value(project_sum_row, 'total', project_sum['total']) table.set_html_cell_attributes( project_sum_row, 'total', 'align="right" style="font-weight:bold"') # insert this blank line between projects - keeps the headings away from the previous row table.set_value(row, 0, "") # insert a total for everything table.set_value(row, 0, html='<hr>') table.set_html_cell_attributes(row, 0, "colspan=%d" % n_cols) row = row + 1 insert_col_headings(table, row, None, chronic) row = row + 1 total_row = row query['host'] = '*' query['project'] = projects query['host'] = query_host query['context'] = query_context table.set_value(total_row, 'total', text=str(all_sum['total']), link=common.selflink(query_dict=query, linkmode="treewalk")) table.set_html_cell_attributes(row, 'total', 'align="right"') for status in status_types: query['status'] = status table.set_value(total_row, status, all_sum[status], link=common.selflink(query_dict=query, linkmode="treewalk")) table.set_html_cell_attributes(total_row, status, 'align="right"') return (table, projects)
def rpt1(): form = pandokia.pcgi.form if form.has_key("test_run"): test_run = form["test_run"].value else: test_run = '*' if test_run == '-me': test_run = 'user_' + common.current_user() + '_*' my_run_prefix = 'user_' + common.current_user() admin = common.current_user() in common.cfg.admin_user_list # c = db.execute("SELECT DISTINCT test_run FROM result_scalar WHERE test_run GLOB ? ORDER BY test_run DESC ",( test_run,)) where_str, where_dict = pdk_db.where_dict([('test_run', test_run)]) sql = "SELECT test_run, valuable, record_count, note, min_time, max_time FROM distinct_test_run %s ORDER BY test_run DESC " % where_str c = pdk_db.execute(sql, where_dict) table = text_table.text_table() table.set_html_table_attributes("border=1") table.define_column('addval', showname='') table.define_column('run', showname='test_run') table.define_column('tree', showname='') table.define_column('del', showname='') table.define_column('min', showname='start') table.define_column('max', showname='end') table.define_column('tdiff', showname='duration') table.define_column('count', showname='records') table.define_column('note', showname='note') # query parameters for various links # link to day_report # link to tree walk of test run tquery = {'project': '*', 'host': '*'} # link to declare a run as valuable vquery = {'valuable_run': 1} # link to update the count in a test run cquery = {} row = 0 for x, val, record_count, note, min_time, max_time in c: if x is None: continue tquery["test_run"] = x vquery["test_run"] = x cquery["count_run"] = x # mark as valuable: # https://ssb.stsci.edu/pandokia/c41.cgi?query=action&test_run=daily_2011-08-24&valuable_run=1 table.set_value(row, 'addval', html='<a href="%s">!</a> ' % common.selflink(vquery, "action")) table.set_value(row, 'run', text=x, link=common.selflink(tquery, "day_report.2")) table.set_value(row, 'tree', text='(tree display)', link=common.selflink(tquery, "treewalk")) if val == '0': if x.startswith(my_run_prefix): table.set_value(row, 'del', text='(delete)', link=common.selflink(tquery, "delete_run.ays")) else: table.set_value(row, 'del', text='(delete)', html='<font color=gray>(delete)</font>', link=common.selflink(tquery, "delete_run.ays")) else: table.set_value(row, 'del', text='(valuable)') if note is None: table.set_value(row, 'note', text='') else: table.set_value(row, 'note', text=note) if min_time is not None and max_time is not None: table.set_value(row, 'tdiff', str(lib.time_diff(max_time, min_time))) min_time = str(min_time).split('.')[0] max_time = str(max_time).split('.')[0] t1 = min_time.split() t2 = max_time.split() if len(t2) > 1 and len(t1) > 1 and t1[0] == t2[0] and len(t2) > 1: max_time = t2[1] table.set_value(row, 'min', text=min_time) table.set_value(row, 'max', text=max_time) else: if min_time is not None: min_time = str(min_time).split('.')[0] table.set_value(row, 'min', text=min_time) if max_time is not None: max_time = str(max_time).split('.')[0] table.set_value(row, 'max', text=max_time) table.set_value(row, 'tdiff', '') # update the count field # https://ssb.stsci.edu/pandokia/c41.cgi?query=action&count_run=daily_2011-08-24 update_count = common.selflink(cquery, 'action') if record_count is None or record_count <= 0: record_count = ' ' table.set_value(row, 'count', html=str(record_count), link=update_count) row = row + 1 if pandokia.pcgi.output_format == 'html': sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) sys.stdout.write('<h2>%s</h2>' % cgi.escape(test_run)) sys.stdout.write(table.get_html(headings=1)) sys.stdout.write( "<br>Click on the ! to mark a test run as too valuable to delete\n" ) sys.stdout.write( "<br>Click on record count to check the count and update it\n") sys.stdout.write("""<style> table { border-collapse: collapse; } table, th, td { border: 2px solid black; padding: 3px; } </style> """) elif pandokia.pcgi.output_format == 'csv': sys.stdout.write(common.cgi_header_csv) sys.stdout.write(table.get_csv()) sys.stdout.flush() return
def do_result(key_id): c = pdk_db.execute( "SELECT key_id, test_run, project, host, context, custom, test_name, status, attn, start_time, end_time, location, test_runner FROM result_scalar WHERE key_id = :1 ", (key_id, )) result_count = 0 for x in c: result_count = result_count + 1 (key_id, test_run, project, host, context, custom, test_name, status, attn, start_time, end_time, location, test_runner) = (x) linkback_dict = { 'project': project, 'host': host, 'context': context, 'custom': custom, 'test_run': test_run, 'test_name': test_name } prevday_dict = { 'project': project, 'host': host, 'context': context, 'custom': custom, 'test_run': test_run, 'test_name': test_name } row = 0 tb = text_table.text_table() tb.set_html_table_attributes("border=1") tb.set_value(row, 0, "test_run") tmp = next_prev(prevday_dict, test_run) if tmp != '': tb.set_value(row, 1, test_run, html=test_run + tmp) else: tb.set_value(row, 1, test_run) row += 1 tb.set_value(row, 0, "key_id") tb.set_value(row, 1, key_id) row += 1 tb.set_value(row, 0, "project") tb.set_value(row, 1, project) row += 1 tb.set_value(row, 0, "host") tb.set_value(row, 1, host) row += 1 tb.set_value(row, 0, "context") tb.set_value(row, 1, context) row += 1 tb.set_value(row, 0, "custom") tb.set_value(row, 1, custom) row += 1 tb.set_value(row, 0, "test_name") tb.set_value(row, 1, test_name) row += 1 tb.set_value(row, 0, "contact") tb.set_value(row, 1, common.get_contact(project, test_name)) row += 1 tb.set_value(row, 0, "status") if status != "P": tb.set_value(row, 1, text=status, html="<font color=red>" + status + "</font>") else: tb.set_value(row, 1, status) row += 1 tb.set_value(row, 0, "attn") tb.set_value(row, 1, attn) row += 1 tb.set_value(row, 0, "test_runner") tb.set_value(row, 1, test_runner) row += 1 try: # ignoring exceptions here because we don't know how good the values in # the database are. some preliminary software does not fill them in # correctly, for example. tb.set_value(row, 1, float(end_time) - float(start_time)) tb.set_value(row, 0, "duration") row += 1 except: pass # merge all of this into a generalized "get the times" function that # can be used anywhere if start_time == '0': st = 'unknown' else: try: # you might think that you could use .%f in strftime, but you would be wrong # the documentation describes it, but it isn't implemented st = datetime.datetime.fromtimestamp(float(start_time)) st = st.strftime("%Y-%m-%d %H:%M:%S") + "." + \ "%03d" % (st.microsecond / 1000) except: # it is some non-time_t format - just display it st = start_time if end_time == '0': et = 'unknown' else: try: # you might think that you could use .%f in strftime, but you would be wrong # the documentation describes it, but it isn't implemented et = datetime.datetime.fromtimestamp(float(end_time)) et = et.strftime("%Y-%m-%d %H:%M:%S") + "." + \ "%03d" % (et.microsecond / 1000) except: # it is some non-time_t format - just display it et = end_time tb.set_value(row, 0, "start_time") tb.set_value(row, 1, st) row += 1 tb.set_value(row, 0, "end_time") tb.set_value(row, 1, et) row += 1 tb.set_value(row, 0, "location") tb.set_value(row, 1, location) row += 1 c1 = pdk_db.execute( "SELECT name, value FROM result_tda WHERE key_id = :1 ORDER BY name ASC", (key_id, )) for y in c1: (name, value) = y tb.set_value(row, 0, "tda_" + name) tb.set_value(row, 1, value) row += 1 c1 = pdk_db.execute( "SELECT name, value FROM result_tra WHERE key_id = :1 ORDER BY name ASC", (key_id, )) for y in c1: (name, value) = y tb.set_value(row, 0, "tra_" + name) if 'xception' in name: tb.set_value(row, 1, value, code=True) else: tb.set_value(row, 1, value) row += 1 sys.stdout.write(tb.get_html()) sys.stdout.write("<a href=%s>back to treewalk</a><br>\n" % common.selflink(linkback_dict, linkmode='treewalk')) sys.stdout.write( "<a href=%s>status history</a><br>\n" % common.selflink(linkback_dict, linkmode='test_history')) c1 = pdk_db.execute("SELECT log FROM result_log WHERE key_id = :1 ", (key_id, )) for y in c1: if not PY3: if isinstance(y, tuple): y = tuple(x.decode() for x in y) (y, ) = y if y != "": if getattr(cfg, 'enable_magic_html_log'): if '<!DOCTYPE' in y or '<html' in y: sys.stdout.write( "<a href=%s>HTML block in a single page</a><br>" % common.selflink({ 'magic_html_log': key_id, }, linkmode='magic_html_log')) sys.stdout.write("Log:<br><pre>") sys.stdout.write(cgi.escape(y)) sys.stdout.write("</pre>\n") sys.stdout.write("<br>\n") sys.stdout.write("<br><hr>\n") return result_count
def dictionary_comp( data_dict, # the actual data file_base, # implies output file name # implies reference file name # implies ok file name # implies refs/file.fields for compare/ignore fields # assumes PDK_REFS environment tda=None, tra=None, # if present, fill tra with output, relative difference interesting_fields=None, # if interesting, compare only those uninteresting_fields=[], # if uninteresting, exclude those from the default list fp_tolerance=1e-7, # tolerance_dict={}, # you can give a specific tolerance for specific fields # default is no special preference print_info=False, # okfh=None, # fail_only=False, # if true, only report fields that are involved in a failure ): # Start off saying the whole comparison has not failed. This # will be the state until the end, unless some element fails. fail = False # The list of fields that had an error in the comparison. errors = [] # so we don't have to keep saying "if tda : tda[xx] = yy" # these dicts will go out of scope at the end of the function if tda is None: tda = {} if tra is None: tra = {} # a reference dict, generated by hand or from a previous result. # or nothing, if it is not there -- gets us through the test # BUG: support PDK_REFS ref_fn = "ref/" + file_base # Read the reference dictionary. no_reference_file will be the # exception if there is no reference file; it is re-raised at # the end. We don't just raise it now because we want all # the reporting that comes below. try: ref_dict = read_reference(ref_fn) no_reference_file = None except NoReferenceFile as e: ref_dict = {} no_reference_file = e # flatten the data dict (the ref dict is already flattened) data_dict = flatten(data_dict) # identify interesting fields. if explicitly specified, use # just that. Otherwise it defaults to all the fields except # those identified as uninteresting. if interesting_fields is None: interesting_fields = set([x for x in data_dict] + [x for x in ref_dict] ) - set(uninteresting_fields) # save the interesting fields from the flattened dict out_fn = write_output(file_base, data_dict, interesting_fields) # add a line to the okfile if okfh: append_okfile(okfh, out_fn, ref_fn) # Which fields were in the expected answers, but missing from the result. missing = set() # Which fields did not compare closely enough. failed = [] # Report the default tolerance. tda['tolerance'] = fp_tolerance # Set up to report the max discrep if not fail_only: tra['max_discrep'] = 0 tra['max_discrep_var'] = None # Loop over the interesting keys, comparing each one. for k in interesting_fields: # flag of whether this iteration of the loop fails the compare this_fail = False # value of the discrepancy for this iteration of the loop discrep = None # see if there are custom tolerances for this field try: tol = tolerance_dict[k] tda['tol_%s' % k] = tol except KeyError: tol = fp_tolerance # pick out the data value try: data = data_dict[k] except AttributeError: tra['discrep_%s' % k] = 'missing data' this_fail = True missing.add(k) continue # pick out the reference value try: ref = ref_dict[k] except KeyError: tra['discrep_%s' % k] = 'missing ref' this_fail = True missing.add(k) continue # comparing numbers if isinstance(data, numbers.Number) and isinstance( ref, numbers.Number): # If both are zero, everything is good. if data == 0 and ref == 0: discrep = 0 else: # Compare the discrepancy as a proportion of the reference # value. try: discrep = (data - ref) / ref if (abs(discrep) > tol): failed.append(k) this_fail = True # If the reference is zero, use absolute difference instead. except ZeroDivisionError as e: discrep = (data - ref) if (abs(discrep) > tol): failed.append(k) this_fail = True # comparing strings elif isinstance(data, string_type) and isinstance(ref, string_type): if ref.strip() != data.strip(): failed.append(k) discrep = True this_fail = True # comparing non-number, non-string, non-dict, non-list # If you use json for your data, this just means booleans # or two different types else: # neither string nor number - all we have for comparing is equality if data != ref: failed.append(k) discrep = True this_fail = True # If this element of the data failed, the whole test failed. if this_fail: fail = True # Report this element in the tra if it failed, or if we are # reporting all elements. if this_fail or not fail_only: # Report the computed value and the reference value as tra. tra[k] = str(data) tra['ref_%s' % k] = str(ref) tra['discrep_%s' % k] = discrep # end: for k in interesting_fields # If any of the fields failed, report what they were. if len(failed) > 0: failed.sort() tra['failed'] = failed # For failed numerical values, report the max discrepancy and variable discrep, vname = 0.0, None for k in failed: val = abs(tra['discrep_%s' % k]) if (isinstance(val, float) and val > discrep): discrep = val vname = k if vname is not None: tra['max_discrep'] = discrep tra['max_discrep_var'] = vname # anything that wasn't there is also a problem if missing: m = sorted(missing) tra['missing'] = str(m) fail = True # This is a pretty display for the interactive user. if print_info and (fail or (not fail_only)): from pandokia.text_table import text_table tt = text_table() failed_set = set(failed) tt.define_column('name') tt.define_column('expect') tt.define_column('actual') tt.define_column('F') tt.define_column('discrep') for row, k in enumerate(sorted(interesting_fields)): tt.set_value(row, 'name', k) if k in ref_dict: tt.set_value(row, 'expect', ref_dict[k]) if k in data_dict: tt.set_value(row, 'actual', data_dict[k]) s = 'discrep_%s' % k if s in tra: tt.set_value(row, 'discrep', tra[s]) if k in failed_set: tt.set_value(row, 'F', '*') print(tt.get_rst(headings=1)) if missing: m = sorted(missing) print("Missing keys: %s" % m) if fail: print("FAILED") if no_reference_file: raise no_reference_file if len(errors): raise Exception(str(errors)) if fail: raise AssertionError('%s failed' % file_base)
if vname is not None: tra['max_discrep'] = discrep tra['max_discrep_var'] = vname # anything that wasn't there is also a problem if missing: m = list(missing) m.sort() tra['missing'] = str(m) fail = True # This is a pretty display for the interactive user. if print_info and (fail or (not fail_only)): from pandokia.text_table import text_table tt = text_table() failed_set = set(failed) tt.define_column('name') tt.define_column('expect') tt.define_column('actual') tt.define_column('F') tt.define_column('discrep') for row, k in enumerate(sorted(interesting_fields)): tt.set_value(row, 'name', k) if k in ref_dict: tt.set_value(row, 'expect', ref_dict[k]) if k in data_dict: tt.set_value(row, 'actual', data_dict[k])
def treewalk(): form = pandokia.pcgi.form output = sys.stdout output.write(common.cgi_header_html) output.write(common.page_header()) # # gather up all the expected parameters # if "test_name" in form: test_name = form.getvalue("test_name") if test_name == '': test_name = '*' else: test_name = "*" context = form.getvalue('context', '*') host = form.getvalue('host', '*') test_run = form.getvalue('test_run', '*') project = form.getvalue('project', '*') status = form.getvalue('status', '*') attn = form.getvalue('attn', '*') qid = form.getvalue('qid', None) debug_cmp = form.getvalue('debug_cmp', 0) # look for input from the compare form cmp_test_run = form.getvalue('cmp_test_run', None) cmp_context = form.getvalue('cmp_context', None) cmp_host = form.getvalue('cmp_host', None) test_run = common.find_test_run(test_run) if cmp_test_run: cmp_test_run = common.find_test_run(cmp_test_run) # look for whether the query asks for a comparison; we assume not comparing = 0 if 'compare' in form: # ok, it explicitly says one of the 3 comparison values comparing = 1 x = form.getvalue('compare', '0') if x == '' or x == '0' or x.startswith('Turn Off'): # if it is a special value that ends the compare, comparing = 0 if x.startswith('Reverse'): t = cmp_test_run cmp_test_run = test_run test_run = t t = cmp_host cmp_host = host host = t t = cmp_context cmp_context = context context = t comparing = 1 # # query values we will always pass back to the next instantion of ourself # query = { 'test_name': test_name, 'test_run': test_run, 'project': project, 'host': host, 'status': status, 'attn': attn, 'context': context, 'compare': comparing, } if qid is not None: qid = int(qid) query['qid'] = qid if cmp_test_run is not None: query['cmp_test_run'] = cmp_test_run if cmp_context is not None: query['cmp_context'] = cmp_context if cmp_host is not None: query['cmp_host'] = cmp_host # # show the narrowing parameters # header_table = text_table.text_table() header_table.set_html_table_attributes( ' style="font-size: large; font-weight: bold" ') row = 0 # # if a test_run is selected, show the test_run here # include a link to clear the test_run selection # if test_run is None: output.write('Tree not generated.<br/>No tests available.') return if test_run != "*": lquery = copy.copy(query) lquery["test_run"] = "*" test_run_line = "<h2>%s = %s %s %s %s</h2>\n" header_table.set_value(row, 0, 'test_run') header_table.set_value(row, 1, '=') header_table.set_value(row, 2, escape(test_run)) header_table.set_value( row, 3, html=common.self_href( lquery, "treewalk", remove_arrow)) tmp2 = common.run_previous(None, test_run) tmp3 = common.run_next(None, test_run) if tmp2 is not None: lquery["test_run"] = tmp2 tmp2 = common.self_href(lquery, "treewalk", " (%s)" % tmp2) header_table.set_value(row, 4, html=tmp2) else: tmp2 = "" if tmp3 is not None: lquery["test_run"] = tmp3 tmp3 = common.self_href(lquery, "treewalk", " (%s)" % tmp3) header_table.set_value(row, 5, html=tmp3) else: tmp3 = "" row = row + 1 for var, label in [ ('project', 'project'), ('host', 'host'), ('context', 'context'), ('status', 'status'), ]: if query[var] != '*': lquery = copy.copy(query) lquery[var] = '*' header_table.set_value(row, 0, label) header_table.set_value(row, 1, '=') header_table.set_value(row, 2, escape(lquery[var])) header_table.set_value( row, 3, html=common.self_href( lquery, "treewalk", remove_arrow)) row = row + 1 if qid is not None: header_table.set_value(row, 0, 'QID') header_table.set_value(row, 2, str(qid)) row = row + 1 print(header_table.get_html()) # start of "Test Prefix: line" # # Here is the actual tree browser for test names. we are at some # prefix (specified by test_name), walking down the tree # output.write("<h2>Test Prefix: ") # # show the prefix. each component of the prefix is a link back # up to that level # lquery = copy.copy(query) # get a list of the levels t = test_name lst = [] while True: y = re.search("[/.]", t) if not y: break lst.append(t[0:y.start() + 1]) t = t[y.start() + 1:] # display each level with the link t = "" for x in lst: t = t + x lquery["test_name"] = t + "*" line = common.self_href(lquery, 'treewalk', escape(x)) output.write(line) # if we are not at the very top, also include a link that gets us back # to "" (i.e. no prefix at all) if test_name != "*": lquery["test_name"] = "" output.write(" ") output.write(common.self_href(lquery, "treewalk", remove_arrow)) output.write(" ") output.write("</h2>\n") # end of "Test Prefix: line" # offer the form to compare with other runs print(cmp_form(query, comparing)) print("<p>") # show the table # "show all" line before the table lquery = copy.copy(query) if comparing: t = 'show all (not just different)' else: t = 'show all' show_all_line = common.self_href(lquery, 'treewalk.linkout', t) + ' - ' lquery['add_attributes'] = 1 show_all_line += common.self_href(lquery, 'treewalk.linkout', 'with attributes') lquery['add_attributes'] = 2 show_all_line += ' - ' + \ common.self_href(lquery, 'treewalk.linkout', 'Column Selector') output.write(show_all_line) output.write("<br>") # begin table of test results # Gather the names of the rows prefixes = collect_prefixes(query) # create the table to display; if comparing, we want a link for # every table cell; this is because we don't know later which cells # we will need to have a link on table = collect_table(prefixes, query, comparing) if comparing: query_2 = query.copy() query_2['test_run'] = cmp_test_run query_2['host'] = cmp_host query_2['context'] = cmp_context t2 = collect_table(prefixes, query_2, 1) # This part is very delicate. We know that both tables # have the same dimensions because the parameters that we # used to deterimine the number of rows/cols are the same; # from that, we also know that the meaning of each cell is # the same. # # So, for each cell, if it appears to contain an int in # both cells, subtract them. # # We know that column 1 contains the test name, so we don't # even try to handle those. # for row in range(0, len(table.rows)): for col in range(1, len(table.rows[row].lst)): # pick the table cell out of each table c1 = table.get_cell(row, col) c2 = t2 .get_cell(row, col) # see if we can convert both of them to integers. # if not, go on to the next cell try: c1v = int(c1.text) c2v = int(c2.text) except ValueError: continue # suppress the link in this cell, but only if both # parts of the comparison are zero. If they are equal, # you still might follow the link and see something. if c1v == 0 and c2v == 0: c1.link = None # compute the difference and poke the value directly back # into the table cell; this preserves the html attributes # and such diff = c1v - c2v if debug_cmp: c1.text = "%d - %d = %+d" % (c1v, c2v, diff) else: if diff == 0: c1.text = '0' else: c1.text = "%+d" % diff ### # If we are comparing to another test run, show which it is. if comparing: output.write("<p>Net difference in counts, this - other</p>") output.write(table.get_html()) output.write("<br>") output.write(show_all_line) output.write("<br>") output.flush() # # if no test_run specified, give an option to choose one (but only from the # test_runs that are in the tests specified) # if 'qid' in query: return # bug: temporary work-around. the narrow-to query is really slow, but doesn't really help the tree walker in qid mode more_where = ' qid = %d AND result_scalar.key_id = query.key_id ' % int(query[ 'qid']) else: more_where = None # bug: This whole thing could happen early. Then, when you find # exactly 1 item in the choices to narrow, you could automatically # include that in all the links. for field in ('test_run', 'project', 'context', 'host'): if '*' not in query[field]: continue lquery = {} for x in query: if query[x] is not None: lquery[x] = query[x] output.write("<h3>Narrow to %s</h3>" % field) tn = test_name if not tn.endswith("*"): tn = tn + "*" where_text, where_dict = pdk_db.where_dict([ ('test_name', tn), ('test_run', test_run), ('project', project), ('host', host), ('context', context), ('status', status), ('attn', attn), ], more_where) if more_where is None: c = pdk_db.execute( "SELECT DISTINCT %s FROM result_scalar %s GROUP BY %s ORDER BY %s" % (field, where_text, field, field), where_dict) else: c = pdk_db.execute( "SELECT DISTINCT %s FROM result_scalar, query %s GROUP BY %s ORDER BY %s" % (field, where_text, field, field), where_dict) for x, in c: if x is None: continue lquery[field] = x output.write( "<a href='" + pandokia.pcgi.cginame + "?query=treewalk&" + urlencode(lquery) + "'>" + x + "</a><br>") output.write("")
def qid_list(): import datetime print("content-type: text/html\n") print(common.page_header()) print("<h1>Interesting QID</h1>") input_query = pandokia.pcgi.form_to_dict(pandokia.pcgi.form) order_by = 'qid' order_dir = 'asc' if 'order_by' in input_query: x = input_query['order_by'][0] if x in ('qid', 'expires', 'username', 'notes'): order_by = x if 'order_dir' in input_query: x = input_query['order_dir'][0] if x in ('asc', 'desc'): order_dir = x def sl(s): d = 'asc' if order_dir == 'asc': d = 'desc' return common.selflink( {'order_by': s, 'order_dir': d}, linkmode='qid_list') t = text_table.text_table() t.define_column("QID", html='<a href=%s>QID</a>' % sl('qid')) t.define_column("Expires", html='<a href=%s>expires</a>' % sl('expires')) t.define_column("User", html='<a href=%s>username</a>' % sl('username')) t.define_column("Notes", html='<a href=%s>notes</a>' % sl('notes')) t.set_html_table_attributes("border=1") qdict = {} c = pdk_db.execute( "SELECT qid, expires, username, notes FROM query_id WHERE expires = :1 OR username <> '' OR notes <> '' ORDER BY %s %s" % (order_by, order_dir), (pandokia.never_expires,)) row = 0 for x in c: v = int(x[0]) qdict['qid'] = v t.set_value(row, 0, v, html='<a href="%s">%s</a>' % (common.selflink(qdict, linkmode="summary"), v)) try: v = int(x[1]) if v == pandokia.never_expires: v = 'never' else: v = str(datetime.date.fromtimestamp(v)) except: v = '?' t.set_value(row, 1, v) v = str(x[2]) t.set_value(row, 2, v, html=cgi.escape(v)) if x[3] is None: v = '' else: v = str(x[3]) t.set_value(row, 3, v, html='<pre>' + cgi.escape(v) + '</pre>') row = row + 1 print(t.get_html())
def collect_table(prefixes, query, always_link): # # make the actual table to display # # always_link=true means to always create an html link in the table # cell that contains a record count # always_link=false means to suppress the link if the record count is 0 status = query['status'] rownum = 0 table = text_table.text_table() table.set_html_table_attributes("border=1") table.define_column("test_name") table.define_column("count") total_col = {} count_col = {} lquery = copy.copy(query) for x in common.cfg.statuses: if (status == '*') or (x in status): lquery['status'] = x table.define_column( x, showname=common.cfg.status_names[x], link=common.selflink( lquery, 'treewalk')) total_col[x] = 0 count_col[x] = 0 total_count = 0 total_row = rownum rownum = rownum + 1 have_qid = 'qid' in query if have_qid: qid = int(query['qid']) more_where = ' qid = %d AND result_scalar.key_id = query.key_id ' % qid else: more_where = None for this_test_name in prefixes: lquery['test_name'] = this_test_name if "*" in this_test_name: linkmode = 'treewalk' else: linkmode = 'treewalk.linkout' lquery['status'] = status table.set_value( rownum, "test_name", text=this_test_name, link=common.selflink( lquery, linkmode)) table.set_html_cell_attributes(rownum, 'test_name', 'align="left"') count = 0 for x in common.cfg.statuses: if (status == '*') or (x in status): lquery['status'] = x where_text, where_dict = query_to_where_tuple( lquery, ('test_name', 'test_run', 'project', 'host', 'context', 'status', 'attn'), more_where) if not have_qid: ss = '' else: ss = ', query' c = pdk_db.execute( "SELECT count(*) FROM result_scalar%s %s" % (ss, where_text), where_dict) datum = c.fetchone() if datum is None: count_col[x] = 0 else: (count_col[x],) = datum lquery['status'] = x if not always_link and count_col[x] == 0: table.set_value(rownum, x, text='0') else: table.set_value( rownum, x, text=count_col[x], link=common.selflink( lquery, linkmode)) table.set_html_cell_attributes(rownum, x, 'align="right"') count = count + count_col[x] total_count = total_count + count_col[x] table.set_value(rownum, "count", text=count) table.set_html_cell_attributes(rownum, "count", 'align="right"') for x in total_col: total_col[x] += count_col[x] rownum = rownum + 1 table.set_value(total_row, "count", text=total_count) table.set_html_cell_attributes( total_row, 'count', 'align="right" style="font-weight:bold"') for x in common.cfg.statuses: if (status == '*') or (x in status): table.set_value(total_row, x, text=total_col[x]) table.set_html_cell_attributes( total_row, x, 'align="right" style="font-weight:bold"') return table
def show(user): form = pandokia.pcgi.form # output.write('<h1>User Preferences: %s</h1>' % cgi.escape(user)) # write the start of form, including hidden fields needed to dispatch # to the save() function after we submit the form. output.write('<form action=%s method=GET>' % common.get_cgi_name()) output.write('<input type=hidden name=query value=prefs>') output.write('<input type=hidden name=subtype value=save>') # show the user's email address c = cfg.pdk_db.execute( 'SELECT email FROM user_prefs WHERE username = :1', (user,)) x = c.fetchone() if x is None: email = user else: email, = x output.write('<h3>Email address</h3>') output.write( 'Email address: <input type=text name=email value="%s">' % email) output.write('<br>\n') # make a table of what the user has selected for each project output.write('<h3>Email Preferences</h3>') tb = text_table.text_table() tb.define_column("project", showname='Project') tb.define_column("none", showname='None') tb.define_column("contact", showname='Contact') tb.define_column("summ", showname='Summary') tb.define_column("full", showname='Full') tb.define_column("all", showname='Always') tb.define_column("line", showname='Max') row = 0 def ckif(x): if fmt == x: return 'checked' else: return '' c = cfg.pdk_db.execute( 'SELECT username, project, format, maxlines FROM user_email_pref WHERE username = :1 ORDER BY project', (user,)) c = [x for x in c] if len(c) == 0: try: x = cfg.default_user_email_preferences except: x = [] c = [(user, x[0], x[1], x[2]) for x in x] for username, project, fmt, maxlines in c: if not project_name_ok: output.write('bad project name in table<br>') continue tb.set_value(row, 'project', project) project = quote(project) # projects will be a list of all the projects we are submitting in the # form output.write('<input type=hidden name=projects value="%s">' % project) # radio.%s will be the radio button for that project name tb.set_value( row, 'none', html='<input type=radio name="radio.%s" value="n" %s>' % (project, ckif('n'))) tb.set_value( row, 'contact', html='<input type=radio name="radio.%s" value="c" %s>' % (project, ckif('c'))) tb.set_value( row, 'summ', html='<input type=radio name="radio.%s" value="s" %s>' % (project, ckif('s'))) tb.set_value( row, 'full', html='<input type=radio name="radio.%s" value="f" %s>' % (project, ckif('f'))) tb.set_value( row, 'all', html='<input type=radio name="radio.%s" value="F" %s>' % (project, ckif('F'))) # maxlines is an integer, but 0 means no limit. display it as a blank # field if (maxlines == 0): maxlines = '' # radio.%s will be the radio button for that project name tb.set_value( row, 'line', html='<input type=text name="line.%s" value="%s" size=5>' % (project, maxlines)) row = row + 1 tb.set_html_table_attributes('border=1') output.write(tb.get_html()) # some explanatory text output.write('''<p>None=no email about that project<br> Contact=email only tests you are a contact for<br> Summary=email contains only a count<br> Full=show all tests with problems, skip projects with none<br> Always=show all tests with problems, show all projects with problems or not<br> Max=list at most max tests in the email</p>''') output.write('<input type=submit name=submit value=save>') if user in cfg.admin_user_list: output.write('<br>') output.write('<input type=text name=newuser>') output.write('<input type=submit name=submit value=newuser>') output.write('</form>') # this is a different form - it just adds a project to the list of # projects that the user has preferences for output.write('<form action=%s method=GET>' % common.get_cgi_name()) output.write('<input type=hidden name=query value=prefs>') output.write('<input type=hidden name=subtype value=add_project>') output.write('<input type=text name=project>') output.write('<input type=submit name=submit value="Add Project">') output.write('</form>')
def latest(): import pandokia.text_table as text_table pdk_db = pandokia.cfg.pdk_db if pandokia.pcgi.output_format == 'html': sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) elif pandokia.pcgi.output_format == 'csv': sys.stdout.write(common.cgi_header_csv) postfixes = ('latest', 'today', 'yesterday') for postfix in postfixes: sys.stdout.write('<h2>%s</h2>' % postfix) t = text_table.text_table() t.define_column('prefix') t.define_column(postfix) t.define_column('n') d = {} total = 0 for row, prefix in enumerate( sorted(common.cfg.recurring_prefix), start=1): t.set_value(row, 0, prefix) test_run = common.find_test_run(prefix + '_' + postfix) c = pdk_db.execute( 'SELECT test_run, record_count FROM distinct_test_run WHERE test_run = :1', (test_run, )) n = c.fetchone() if n is None: t.set_value(row, 1, '') t.set_value(row, 2, '') continue (tr, count) = n if count == 0 or count is None: import pandokia.cleaner as cleaner count = cleaner.recount_test_run(tr) if count is None: count = 0 total = total + count d['test_run'] = test_run t.set_value( row, 1, text=test_run, link=common.selflink( d, "day_report.2")) count_link = common.selflink({'count_run': test_run}, 'action') t.set_value(row, 2, text=count, link=count_link) t.set_value(row + 1, 2, total) if pandokia.pcgi.output_format == 'html': t.set_html_table_attributes(' border=1 ') sys.stdout.write(t.get_html(headings=1)) elif pandokia.pcgi.output_format == 'csv': sys.stdout.write(t.get_csv())
def gen_daily_table( test_run, projects, query_context, query_host, valuable=0, chronic=False): # chronic is true if we should do the chronic column, false otherwise # convert special names, e.g. daily_latest to the name of the latest # daily_* test_run = common.find_test_run(test_run) # show_delete = not valuable if show_delete: my_run_prefix = 'user_' + common.current_user() if test_run.startswith(my_run_prefix): del_text = '(del)' else: if common.current_user() in common.cfg.admin_user_list: del_text = '<font color=gray>(del)</font>' else: show_delete = False # this is the skeleton of the cgi queries for various links query = {"test_run": test_run} # This is a single table for all projects, because we want the # pass/fail/error columns to be aligned from one to the next # table = text_table.text_table() # The texttable object doesn't understand colspans, but we hack a # colspan into it anyway. Thee result is ugly if you have borders. table.set_html_table_attributes("cellpadding=2 ") status_types = common.cfg.statuses row = 0 table.define_column("host") table.define_column("context") table.define_column("os") table.define_column("total") for x in status_types: table.define_column(x) if chronic: table.define_column("chronic") table.define_column("del") table.define_column("note") n_cols = 3 + len(status_types) + 2 # # # # # # # # # # # loop across hosts prev_project = None all_sum = {'total': 0, 'chronic': 0} for status in status_types: all_sum[status] = 0 hc_where, hc_where_dict = pdk_db.where_dict( [('test_run', test_run), ('project', projects), ('context', query_context), ('host', query_host)]) c = pdk_db.execute( "SELECT DISTINCT project, host, context FROM result_scalar %s ORDER BY project, host, context " % hc_where, hc_where_dict) if chronic: chronic_str = "AND ( chronic = '0' or chronic IS NULL )" else: chronic_str = "" # now we forget the list of projects that came in and construct a # list of projects that we actually saw. projects = [] for project, host, context in c: # make a new project section of the table if project != prev_project: projects.append(project) table.set_value(row, 0, "") prev_project = project # values common to all the links we will write in this pass through # the loop query["project"] = project query["host"] = "%" # this link text is common to all the links for this project link = common.selflink(query_dict=query, linkmode="treewalk") # the heading for a project subsection of the table project_text = cgi.escape(project) project_text = '<hr><big><strong><b><a name="%s" href="%s">%s</a></b></strong></big>' % ( project_text, link, project_text) table.set_value(row, 0, text=project, html=project_text) table.set_html_cell_attributes(row, 0, "colspan=%d" % n_cols) row += 1 # the column headings for this project's part of the table insert_col_headings(table, row, link, chronic) row += 1 # This will be the sum of all the tests in a particular project. # It comes just under the headers for the project, but we don't # know the values until the end. project_sum = {'total': 0, 'chronic': 0} for status in status_types: project_sum[status] = 0 project_sum_row = row # delete entire project from the test run if show_delete: table.set_value(row, 'del', html=del_text, link=delete_link( {'test_run': test_run, 'project': project}, show_delete)) row += 1 prev_host = None # a new host/contest line query["host"] = host link = common.selflink(query_dict=query, linkmode="treewalk") if host != prev_host: table.set_value(row, 0, text=host, link=link) prev_host = host query['context'] = context link = common.selflink(query_dict=query, linkmode="treewalk") del query['context'] table.set_value(row, 1, text=context, link=link) # delete entire project from the test run if show_delete: table.set_value(row, 'del', html=del_text, link=delete_link({'test_run': test_run, 'project': project, 'host': host, 'context': context}, show_delete)) # show the host info hinfo = pandokia.common.hostinfo(host) table.set_value( row, 2, text=hinfo[1], link=common.selflink( query_dict={ 'host': host}, linkmode='hostinfo')) total_results = 0 missing_count = 0 for status in status_types: # use "or chronic is null" until we update the database for the new # schema. c1 = pdk_db.execute( "SELECT COUNT(*) FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND status = :4 AND context = :5 %s" % (chronic_str, ), (test_run, project, host, status, context)) (x,) = c1.fetchone() total_results += x project_sum[status] += x all_sum[status] += x if x == 0: table.set_value(row, status, text='0') else: table.set_value( row, status, text=str(x), link=link + "&status=" + status) table.set_html_cell_attributes(row, status, 'align="right"') if status == 'M': missing_count = x if chronic: c1 = pdk_db.execute( "SELECT COUNT(*) FROM result_scalar WHERE test_run = :1 AND project = :2 AND host = :3 AND context = :5 AND chronic = '1'", (test_run, project, host, status, context)) (x,) = c1.fetchone() total_results += x project_sum['chronic'] += x all_sum['chronic'] += x table.set_value( row, 'chronic', text=str(x), link=link + "&chronic=1") table.set_html_cell_attributes(row, 'chronic', 'align="right"') project_sum['total'] += total_results all_sum['total'] += total_results if 0 and 'M' in status_types: # not sure how to deal with this in the presence of chronic problem # tests if missing_count != 0: if missing_count == total_results: # if it equals the total, then everything is missing; we # make a note of that table.set_value(row, 'note', 'all missing') else: # only some of them are missing table.set_value(row, 'note', 'some missing') table.set_value(row, 'total', text=str(total_results), link=link) table.set_html_cell_attributes(row, 'total', 'align="right"') row = row + 1 for status in status_types: table.set_value(project_sum_row, status, project_sum[status]) table.set_html_cell_attributes( project_sum_row, status, 'align="right" style="font-weight:bold"') table.set_value(project_sum_row, 'total', project_sum['total']) table.set_html_cell_attributes( project_sum_row, 'total', 'align="right" style="font-weight:bold"') # insert this blank line between projects - keeps the headings away # from the previous row table.set_value(row, 0, "") # insert a total for everything table.set_value(row, 0, html='<hr>') table.set_html_cell_attributes(row, 0, "colspan=%d" % n_cols) row = row + 1 insert_col_headings(table, row, None, chronic) row = row + 1 total_row = row query['host'] = '*' query['project'] = projects query['host'] = query_host query['context'] = query_context table.set_value(total_row, 'total', text=str(all_sum['total']), link=common.selflink(query_dict=query, linkmode="treewalk") ) table.set_html_cell_attributes(row, 'total', 'align="right"') for status in status_types: query['status'] = status table.set_value( total_row, status, all_sum[status], link=common.selflink( query_dict=query, linkmode="treewalk")) table.set_html_cell_attributes(total_row, status, 'align="right"') return (table, projects)
def show(user): form = pandokia.pcgi.form # output.write('<h1>User Preferences: %s</h1>' % cgi.escape(user)) # write the start of form, including hidden fields needed to dispatch # to the save() function after we submit the form. output.write('<form action=%s method=GET>' % common.get_cgi_name()) output.write('<input type=hidden name=query value=prefs>') output.write('<input type=hidden name=subtype value=save>') # show the user's email address c = cfg.pdk_db.execute('SELECT email FROM user_prefs WHERE username = :1', (user, )) x = c.fetchone() if x is None: email = user else: email, = x output.write('<h3>Email address</h3>') output.write('Email address: <input type=text name=email value="%s">' % email) output.write('<br>\n') # make a table of what the user has selected for each project output.write('<h3>Email Preferences</h3>') tb = text_table.text_table() tb.define_column("project", showname='Project') tb.define_column("none", showname='None') tb.define_column("contact", showname='Contact') tb.define_column("summ", showname='Summary') tb.define_column("full", showname='Full') tb.define_column("all", showname='Always') tb.define_column("line", showname='Max') row = 0 def ckif(x): if format == x: return 'checked' else: return '' c = cfg.pdk_db.execute( 'SELECT username, project, format, maxlines FROM user_email_pref WHERE username = :1 ORDER BY project', (user, )) c = [x for x in c] if len(c) == 0: try: x = cfg.default_user_email_preferences except: x = [] c = [(user, x[0], x[1], x[2]) for x in x] for username, project, format, maxlines in c: if not project_name_ok: output.write('bad project name in table<br>') continue tb.set_value(row, 'project', project) project = urllib.quote(project) # projects will be a list of all the projects we are submitting in the form output.write('<input type=hidden name=projects value="%s">' % project) # radio.%s will be the radio button for that project name tb.set_value(row, 'none', html='<input type=radio name="radio.%s" value="n" %s>' % (project, ckif('n'))) tb.set_value(row, 'contact', html='<input type=radio name="radio.%s" value="c" %s>' % (project, ckif('c'))) tb.set_value(row, 'summ', html='<input type=radio name="radio.%s" value="s" %s>' % (project, ckif('s'))) tb.set_value(row, 'full', html='<input type=radio name="radio.%s" value="f" %s>' % (project, ckif('f'))) tb.set_value(row, 'all', html='<input type=radio name="radio.%s" value="F" %s>' % (project, ckif('F'))) # maxlines is an integer, but 0 means no limit. display it as a blank field if (maxlines == 0): maxlines = '' # radio.%s will be the radio button for that project name tb.set_value( row, 'line', html='<input type=text name="line.%s" value="%s" size=5>' % (project, maxlines)) row = row + 1 tb.set_html_table_attributes('border=1') output.write(tb.get_html()) # some explanatory text output.write('''<p>None=no email about that project<br> Contact=email only tests you are a contact for<br> Summary=email contains only a count<br> Full=show all tests with problems, skip projects with none<br> Always=show all tests with problems, show all projects with problems or not<br> Max=list at most max tests in the email</p>''') output.write('<input type=submit name=submit value=save>') if user in cfg.admin_user_list: output.write('<br>') output.write('<input type=text name=newuser>') output.write('<input type=submit name=submit value=newuser>') output.write('</form>') # this is a different form - it just adds a project to the list of # projects that the user has preferences for output.write('<form action=%s method=GET>' % common.get_cgi_name()) output.write('<input type=hidden name=query value=prefs>') output.write('<input type=hidden name=subtype value=add_project>') output.write('<input type=text name=project>') output.write('<input type=submit name=submit value="Add Project">') output.write('</form>')
def latest(): import pandokia.text_table as text_table pdk_db = pandokia.cfg.pdk_db if pandokia.pcgi.output_format == 'html': sys.stdout.write(common.cgi_header_html) sys.stdout.write(common.page_header()) elif pandokia.pcgi.output_format == 'csv': sys.stdout.write(common.cgi_header_csv) postfixes = ('latest', 'today', 'yesterday') for postfix in postfixes: sys.stdout.write('<h2>%s</h2>' % postfix) t = text_table.text_table() t.define_column('prefix') t.define_column(postfix) t.define_column('n') d = {} total = 0 for row, prefix in enumerate(sorted(common.cfg.recurring_prefix), start=1): t.set_value(row, 0, prefix) test_run = common.find_test_run(prefix + '_' + postfix) c = pdk_db.execute( 'SELECT test_run, record_count FROM distinct_test_run WHERE test_run = :1', (test_run, )) n = c.fetchone() if n is None: t.set_value(row, 1, '') t.set_value(row, 2, '') continue (tr, count) = n if count == 0 or count is None: import pandokia.cleaner as cleaner count = cleaner.recount_test_run(tr) if count is None: count = 0 total = total + count d['test_run'] = test_run t.set_value(row, 1, text=test_run, link=common.selflink(d, "day_report.2")) count_link = common.selflink({'count_run': test_run}, 'action') t.set_value(row, 2, text=count, link=count_link) t.set_value(row + 1, 2, total) if pandokia.pcgi.output_format == 'html': t.set_html_table_attributes(' border=1 ') sys.stdout.write(t.get_html(headings=1)) elif pandokia.pcgi.output_format == 'csv': sys.stdout.write(t.get_csv())