def show_table(headings, rows, offset, show_rows, cross_reference, complete=False,
show_nav=True):
"""Show a paginated table"""
top = space_span()[a(href='/')['up']]
backwards = [('-' + str(show_rows),
max(0, offset - show_rows))] if offset > 0 else []
forwards = [] if complete else [
('+'+str(show_rows), max(0, offset+show_rows))]
start = [] if offset <= show_rows else [('0', 0)]
navs = [top, [space_span()[a(href=cross_reference(noffset, show_rows))[label]]
for label, noffset in forwards + backwards + start]] if (show_rows and show_nav) else []
return [navs, table[headings, rows], navs]
def build_navigation(builds, build, filters={}):
"""Work out previous/next/latest links for a specific build,
given other builds on the branch"""
bnum = ( [(int(k.split('-')[2]), k) for k in builds
if k.startswith('cam-oeprod-')])
index = -1
for index_prime, (_, build_prime) in enumerate(bnum):
if build_prime == build:
index = index_prime
if index == -1:
return []
non_build_filters = dict([i for i in filters.items() if
i[0] not in ['build', 'branch']])
ref = filters_to_path(non_build_filters)
out = []
for label, offset, key in [('previous', lambda x: x-1, 'Left'),
('next', lambda x: x+1, 'Right'),
('latest', lambda x: -1, 'Ctrl+Right')]:
try:
_, altbuild = bnum[offset(index)]
except IndexError:
continue
if altbuild == build:
continue
text = [altbuild, ' (', label, ')', ' [', key, ']']
target = '/build/'+altbuild
out += [
space_span()[a(href=target)[text]],
script(type='text/javascript')[
"shortcut.add(\"%s\", function(){ window.location=\"%s\"; });" %
(key, target)]]
return out
def render(x):
in_slice = [r for r in x['results'] if (
r.get('start_time') and
r['end_time'] >= t2epoch and
r.get('end_time', time()) <= t2epoch + 60 * 60)]
passed = len([r for r in in_slice if r.get('failure') == ''])
completed = len(in_slice)
if completed == 0:
content = [0]
else:
content = [passed, '/', completed]
clss = None
if completed == 0:
clss = 'progress'
elif completed == passed:
clss = 'pass'
elif passed == 0:
clss = 'failure'
elif passed < completed:
clss = 'known'
else:
return td[content]
return td(Class=clss)[a(href='/results/dut_name=%s/earliest=%s' % (x['name'], t2epoch))[content]]
def render(rec):
"""produce HTML"""
if rec.get(key) is None:
return '(unknown)'
nquery = dict(oquery)
nquery[key] = rec[key]
return tags.a(href=base + unparse(nquery))[rec.get(altkey, rec[key])]
def render(rec):
"""produce HTML"""
if rec.get(key) is None:
return '(unknown)'
nquery = dict(oquery)
nquery[key] = rec[key]
return tags.a(href = base + unparse(nquery))[rec.get(altkey, rec[key])]
def run_time(x):
"""Render run time column"""
if x.get('control_pid') is None:
return 'no pid'
if x.get('last_launch_time') is None:
return 'no start'
ex = {}
ex['title'] = x.get('control_command_line','')+ ' PID '+str(x['control_pid'])
if x.get('result_id'):
ex['href'] = '/logs/result_id='+str(x['result_id'])
return a(**ex)['%ds' % (time() - x['last_launch_time'])]
def scheduler(request):
"""Render HTML"""
def myform(text, *members):
"""Return an HTML POST form back to this page with CSRF support"""
return form(method='post', action='/scheduler')[
stan_input(type='hidden', name='operation', value=text),
list(members),
stan_input(type='submit', value=text)]
post_message = None
if request.method == 'POST':
operation = request.POST['operation']
if operation == 'cancel':
row = request.POST['id']
query = {'_id': objectid.ObjectId(row)}
CONNECTION.jobs.update(
query,
{'$set': {
'status': 'cancelled',
'finish_time': time()
}})
post_message = ['Cancelled ', row]
elif operation == 'add to queue':
dut = request.POST.get('dut')
command = request.POST['command']
if not match(
'((testsuite)|(experiments)|(./autolaunch.py))[ 0-9a-zA-Z\-]+',
command):
post_message = 'invalid command ' + command
else:
doc = {
'user': request.POST['user'],
'status': 'queued',
'submit_time': time(),
'timeout': int(request.POST['timeout']),
'urgent': 1 if request.POST.get('urgent') else 0,
'dut': dut if dut else None,
'command': request.POST['command'].split()
}
CONNECTION.jobs.save(doc)
post_message = ['Added job to queue']
else:
assert 0
if post_message:
post_html = div(
**{'class': 'postmessage'})[post_message, ' ',
a(href='/scheduler')[em['(clear)']]]
else:
post_html = []
queue_html = []
for title, filters, sort in [('Queued tests', {
'status': 'queued'
}, [('urgent', DESCENDING), ('submit_time', DESCENDING)]),
('Running tests', {
'status': 'running'
}, [('launch_time', ASCENDING)]),
('Recent finished tests', {
'status': {
'$nin': ['queued, running']
}
}, [('finish_time', DESCENDING)])]:
cur = CONNECTION.jobs.find(filters).sort(sort)
if 'finished' in title:
cur = cur.limit(10)
data = list(cur)
columns = [
('user', lambda x: x.get('user')),
('command', lambda x: ' '.join(x.get('command', ['no command']))),
('status', lambda x: a(title=x.get('failure', ''))[x.get(
'status', '(no status!)')]),
('dut', lambda x: x.get('dut', 'ANY')),
('timeout', lambda x: str(x.get('timeout')) + ' seconds'),
('priority', lambda x: 'urgent' if x.get('urgent') else 'normal'),
('submission time (UT)', lambda x:
(asctime(gmtime(x['submit_time'])) if 'submit_time' in x else []))
]
if title != 'Recent finished tests':
columns.append(('cancel', lambda x: myform(
'cancel',
stan_input(type='hidden', name='id', value=str(x['_id'])))))
if title != 'Queued tests':
columns.append(
('log', lambda x:
(a(href='/logs/result_id=' + str(x['results_id']))['view'])
if 'results_id' in x else
(a(href='/logs/job_id=' + str(x['_id']))['view'])))
columns.append(('finish time (UT)', lambda x:
(asctime(gmtime(x['finish_time']))
if 'finish_time' in x else [])))
queue = show_table.produce_table(data,
columns,
cross_reference('/scheduler', {}),
show_nav=False,
row_fn=status_styling)
queue_html += [h2[title], (queue if data else div['(none)'])]
dut_options = [
option(value=dut['_id'])[dut['_id']] for dut in CONNECTION.duts.find()
]
submit_form = myform(
'add to queue', span['User email address: ',
stan_input(type='text',
name='user',
size=30,
value='*****@*****.**')],
span[
' Command: ',
stan_input(
type='text', name='command', size=70, value='./autolaunch.py '
)], span[' DUT: ',
select(name='dut')[option(selected='selected')['ANY'],
dut_options]],
span[' Timeout: ',
stan_input(type='text', name='timeout', size=5, value='600')],
span[' Urgent: ',
stan_input(type='checkbox', name='urgent', value=1), ' '])
node_html = [h2['Test Nodes']]
ul_list = []
for i in range(0, TEST_NODES):
dut_doc = CONNECTION.duts.find_one({'num': i})
if dut_doc['acquired'] == True:
ul_list.append(li[font(color='red')[dut_doc['name']]])
else:
ul_list.append(li[font(color='green')[dut_doc['name']]])
node_html.append(ul[ul_list])
html = [
post_html, h2['Submit test job'], submit_form, node_html, queue_html
]
return render_to_response(
'generic.html',
RequestContext(request, {'content': html_fragment(html)}))
def main():
"""Command line interface"""
parser = argparse.ArgumentParser(description="Create grid")
parser.add_argument('-o', '--output-file',
default = None,
help = 'Set the output file')
parser.add_argument('-v', '--verbose', action='store_true',
help = 'Set verbosity flag')
parser.add_argument('-m', '--max-results',
type = int, default = 128 * 256,
help = 'Maximum results to output')
parser.add_argument('-b', '--branch',
default = 'master',
help = 'Branch to produce grid for')
# parser.add_argument('match', nargs='*')
args = parser.parse_args()
if args.output_file is None:
print 'Must specify an output file'
sys.exit(1)
mongo = bvtlib.mongodb.get_autotest()
builds_query = {'branch': args.branch}
results_by_build = {}
results = []
tests = set()
build_ids = []
for build in mongo.builds.find(builds_query, limit = args.max_results).sort([('tag_time', DESCENDING)]):
build_id = build['_id']
# print 'build id is', build_id
# results_for_build = {'build': build_id}
results_for_build = {}
results_query={'build': build_id}
interesting = False
for result in mongo.results.find(results_query):
if 'infrastructure_problem' not in result:
# print ' got result', result
if 'test_case' in result:
test_case = result['test_case']
if test_case is not None:
failure = result['failure'] if 'failure' in result else None
if failure is not None:
interesting = True
result = True if failure in ['', None] else failure
if test_case in results_for_build:
results_for_build[test_case].append(result)
# results_for_build[test_case] += 1
else:
results_for_build[test_case] = [ result ]
# results_for_build[test_case] = 1
if interesting:
results.append(results_for_build)
results_by_build[build_id] = results_for_build
build_ids.append(build_id)
tests.update(results_for_build.keys())
# build_ids = results_by_build.keys()
# build_ids.sort()
test_names = [ test for test in tests ]
test_names.sort()
column_number = 1
column_numbers = {}
column_names = [th['build']]
column_keys = []
test_labels = {}
for test_name in test_names:
test_label = test_name.replace(' ', '_')
column_numbers[test_name] = column_number
column_heading = th[a(href="#"+test_label, title=test_name)[repr(column_number)]]
column_names.append(column_heading)
column_keys.append(li[a(name=test_label)[a(href="http://autotest/results?reverse=1&test_case="+test_name)[test_name]]])
test_labels[test_name] = test_label
column_number += 1
rows = [column_names]
for build_id in build_ids:
build_results = results_by_build[build_id]
cells = [th[a(href="http://autotest/build/"+build_id)[build_id]]]
for test in test_names:
if test in build_results:
fail_count = len(build_results[test])
cells.append(td(bgcolor= "#ff8080" if fail_count == 1 else "#ff0000")[a(href="#"+test_labels[test], title=test)[repr(fail_count)]])
else:
cells.append(td['-'])
row = [tr[cells]]
rows.append(row)
column_key = [ol[column_keys]]
grid = [table(border='true')[rows]]
outfile = open(args.output_file, 'wb')
html = [ h1['BVT results grid for '+args.branch],
h2['Column key'],
column_key,
h2['Results'],
grid
]
page = html_fragment(html)
outfile.write(page)
outfile.close()
return
def view_results_table(request, constraint):
"""render a table of HTML results in a clean way"""
query = constraints.parse(constraint)
oquery = dict(query)
reverse = query.pop('reverse', 1)
offset = query.pop('offset', 0)
limit = query.pop('limit', 30)
bquery = dict(query)
status = query.pop('status', None)
if status == 'unknown_failures':
query['whiteboard'] = ''
query['failure'] = {'$exists': True, '$ne':''}
query['end_time'] = {'$exists':True}
elif status == 'product_problems':
query['whiteboard'] = {'$exists': True, '$ne':''}
query['infrastructure_problem'] = False
query['end_time'] = {'$exists':True}
query['failure'] = {'$exists': True, '$ne':''}
elif status == 'infrastructure_problems':
query['whiteboard'] = {'$ne':''}
query['infrastructure_problem'] = True
query['end_time'] = {'$exists':True}
elif status == 'failures':
query['infrastructure_problem'] = False
query['failure'] = {'$exists': True, '$ne':''}
query['end_time'] = {'$exists':True}
elif status == 'passes':
query['failure'] = ''
query['end_time'] = {'$exists':True}
elif status:
return render_to_response(
'generic.html',
RequestContext(request, {'content':html_fragment(
['invalid status '+repr(status)])}))
earliest = query.pop('earliest', None)
latest = query.pop('latest', None)
if earliest:
query['start_time'] = {'$gt':float(earliest)}
elif latest:
query['start_time'] = {'$lt':float(latest)}
else:
query['start_time'] = {'$exists': True}
if query.get('whiteboard') == '':
q2=dict(query)
q2['whiteboard'] = { '$exists' :False}
query = {'$or': [query, q2]}
cursor = CONNECTION.results.find(query, limit=limit,
skip=offset).sort(
'start_time', DESCENDING if reverse else ASCENDING)
lookup = lambda term: constraints.lookup('/results', oquery, term)
result_columns = [
('mode', lambda x: 'DEVELOPMENT' if x.get('development_mode')
else 'PRODUCTION'),
('Test Suite/Step information', lambda x: a(href='/run_results/result_id='+str(x['_id']))[str(x['_id'])]),
lookup('test_case'),
('start time', lambda x: time.asctime(
time.localtime(x.get('start_time')))),
('end time', lambda x:
time.asctime(time.localtime(x.get('end_time'))) if
x.get('end_time') else 'still running'),
constraints.lookup('/results', oquery, 'dut_name'),
lookup('build'),
('result', lambda x: (x.get('failure', 'PASS') if
x.get('end_time') else 'unfinished')),
('server', lambda x: x.get('automation_server', '-')),
('pid', lambda x: x.get('control_pid', '-')),
('whiteboard', lambda x: x.get('whiteboard','') or '-')]
if earliest is None and latest is None:
result_columns += [('history', lambda x: a(
href='/results/dut='+x.get('dut_name', '-')+
'/reverse=1/latest='+str(x.get('end_time') or x['start_time']))['context'] if
x.get('dut') and x.get('start_time') else '-')]
result_columns += [('log file', lambda x: a(href='/logs/result_id='+str(x['_id']))['view'])]
table = show_table.produce_table(
cursor, result_columns,
constraints.cross_reference('/results', oquery),
offset, limit, row_fn = status_styling)
nodes = [
div['there are ', CONNECTION.results.count(),
' results and this page shows the most recent starting at offset ',
offset],
table
]
return render_to_response('generic.html',
RequestContext(request, {
'content': html_fragment(nodes)
}))
def view_results_table(request, constraint):
"""render a table of HTML results in a clean way"""
query = constraints.parse(constraint)
oquery = dict(query)
reverse = query.pop('reverse', 1)
offset = query.pop('offset', 0)
limit = query.pop('limit', 30)
bquery = dict(query)
status = query.pop('status', None)
if status == 'unknown_failures':
query['whiteboard'] = ''
query['failure'] = {'$exists': True, '$ne': ''}
query['end_time'] = {'$exists': True}
elif status == 'product_problems':
query['whiteboard'] = {'$exists': True, '$ne': ''}
query['infrastructure_problem'] = False
query['end_time'] = {'$exists': True}
query['failure'] = {'$exists': True, '$ne': ''}
elif status == 'infrastructure_problems':
query['whiteboard'] = {'$ne': ''}
query['infrastructure_problem'] = True
query['end_time'] = {'$exists': True}
elif status == 'failures':
query['infrastructure_problem'] = False
query['failure'] = {'$exists': True, '$ne': ''}
query['end_time'] = {'$exists': True}
elif status == 'passes':
query['failure'] = ''
query['end_time'] = {'$exists': True}
elif status:
return render_to_response(
'generic.html',
RequestContext(
request,
{'content': html_fragment(['invalid status ' + repr(status)])
}))
earliest = query.pop('earliest', None)
latest = query.pop('latest', None)
if earliest:
query['start_time'] = {'$gt': float(earliest)}
elif latest:
query['start_time'] = {'$lt': float(latest)}
else:
query['start_time'] = {'$exists': True}
if query.get('whiteboard') == '':
q2 = dict(query)
q2['whiteboard'] = {'$exists': False}
query = {'$or': [query, q2]}
cursor = CONNECTION.results.find(query, limit=limit, skip=offset).sort(
'start_time', DESCENDING if reverse else ASCENDING)
lookup = lambda term: constraints.lookup('/results', oquery, term)
result_columns = [
('mode', lambda x: 'DEVELOPMENT'
if x.get('development_mode') else 'PRODUCTION'),
('Test Suite/Step information', lambda x: a(
href='/run_results/result_id=' + str(x['_id']))[str(x['_id'])]),
lookup('test_case'),
('start time',
lambda x: time.asctime(time.localtime(x.get('start_time')))),
('end time', lambda x: time.asctime(time.localtime(x.get('end_time')))
if x.get('end_time') else 'still running'),
constraints.lookup('/results', oquery, 'dut_name'),
lookup('build'),
('result', lambda x: (x.get('failure', 'PASS')
if x.get('end_time') else 'unfinished')),
('server', lambda x: x.get('automation_server', '-')),
('pid', lambda x: x.get('control_pid', '-')),
('whiteboard', lambda x: x.get('whiteboard', '') or '-')
]
if earliest is None and latest is None:
result_columns += [
('history', lambda x: a(href='/results/dut=' + x.get(
'dut_name', '-') + '/reverse=1/latest=' + str(
x.get('end_time') or x['start_time']))['context']
if x.get('dut') and x.get('start_time') else '-')
]
result_columns += [
('log file',
lambda x: a(href='/logs/result_id=' + str(x['_id']))['view'])
]
table = show_table.produce_table(cursor,
result_columns,
constraints.cross_reference(
'/results', oquery),
offset,
limit,
row_fn=status_styling)
nodes = [
div['there are ',
CONNECTION.results.count(),
' results and this page shows the most recent starting at offset ',
offset], table
]
return render_to_response(
'generic.html',
RequestContext(request, {'content': html_fragment(nodes)}))
def fn(x):
url = '/results/status=' + field + '/build=' + build + '/test_case=' + x[
'test_case']['description']
if dut:
url += '/dut=' + dut
return a(href=url)[x.get(field, 0)]
'$gt': build_doc['build_time']
}
}).sort([('build_time', pymongo.ASCENDING)]).limit(1)[0]
except (IndexError, KeyError), _:
pass
else:
builddocs += [build_next]
builds = [b['_id'] for b in builddocs]
html = [
describe_build.build_navigation(builds, build),
describe_build.describe_build(branch, build)
]
columns = [('description', lambda x: a(href='/results/build=' + x[
'build'] + '/test_case=' + x['test_case']['description'] + (
'/dut=' + x['dut']
if x.get('dut_name') else ''))[x['test_case']['description']])]
for dut, stuff in records:
columns2 = list(columns)
for name in process_result.CATEGORIES:
def render(field, dut):
def fn(x):
url = '/results/status=' + field + '/build=' + build + '/test_case=' + x[
'test_case']['description']
if dut:
url += '/dut=' + dut
return a(href=url)[x.get(field, 0)]
return fn
def main():
"""Command line interface"""
parser = argparse.ArgumentParser(description="Create grid")
parser.add_argument('-o',
'--output-file',
default=None,
help='Set the output file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Set verbosity flag')
parser.add_argument('-m',
'--max-results',
type=int,
default=128 * 256,
help='Maximum results to output')
parser.add_argument('-b',
'--branch',
default='master',
help='Branch to produce grid for')
# parser.add_argument('match', nargs='*')
args = parser.parse_args()
if args.output_file is None:
print 'Must specify an output file'
sys.exit(1)
mongo = src.bvtlib.mongodb.get_autotest()
builds_query = {'branch': args.branch}
results_by_build = {}
results = []
tests = set()
build_ids = []
for build in mongo.builds.find(builds_query, limit=args.max_results).sort([
('tag_time', DESCENDING)
]):
build_id = build['_id']
# print 'build id is', build_id
# results_for_build = {'build': build_id}
results_for_build = {}
results_query = {'build': build_id}
interesting = False
for result in mongo.results.find(results_query):
if 'infrastructure_problem' not in result:
# print ' got result', result
if 'test_case' in result:
test_case = result['test_case']
if test_case is not None:
failure = result[
'failure'] if 'failure' in result else None
if failure is not None:
interesting = True
result = True if failure in ['', None] else failure
if test_case in results_for_build:
results_for_build[test_case].append(result)
# results_for_build[test_case] += 1
else:
results_for_build[test_case] = [result]
# results_for_build[test_case] = 1
if interesting:
results.append(results_for_build)
results_by_build[build_id] = results_for_build
build_ids.append(build_id)
tests.update(results_for_build.keys())
# build_ids = results_by_build.keys()
# build_ids.sort()
test_names = [test for test in tests]
test_names.sort()
column_number = 1
column_numbers = {}
column_names = [th['build']]
column_keys = []
test_labels = {}
for test_name in test_names:
test_label = test_name.replace(' ', '_')
column_numbers[test_name] = column_number
column_heading = th[a(href="#" + test_label,
title=test_name)[repr(column_number)]]
column_names.append(column_heading)
column_keys.append(li[a(name=test_label)[a(
href="http://autotest/results?reverse=1&test_case=" +
test_name)[test_name]]])
test_labels[test_name] = test_label
column_number += 1
rows = [column_names]
for build_id in build_ids:
build_results = results_by_build[build_id]
cells = [th[a(href="http://autotest/build/" + build_id)[build_id]]]
for test in test_names:
if test in build_results:
fail_count = len(build_results[test])
cells.append(
td(bgcolor="#ff8080" if fail_count == 1 else "#ff0000")[a(
href="#" + test_labels[test],
title=test)[repr(fail_count)]])
else:
cells.append(td['-'])
row = [tr[cells]]
rows.append(row)
column_key = [ol[column_keys]]
grid = [table(border='true')[rows]]
outfile = open(args.output_file, 'wb')
html = [
h1['BVT results grid for ' + args.branch], h2['Column key'],
column_key, h2['Results'], grid
]
page = html_fragment(html)
outfile.write(page)
outfile.close()
return
def fn(x):
url = '/results/status='+field+'/build='+build+'/test_case='+x['test_case']['description']
if dut:
url += '/dut='+dut
return a(href=url)[x.get(field,0)]
try:
build_next = CONNECTION.builds.find({'branch': build_doc['branch'],
'build_time' : {'$gt': build_doc['build_time']}}).sort(
[('build_time',pymongo.ASCENDING)]).limit(1)[0]
except (IndexError, KeyError), _:
pass
else:
builddocs += [build_next]
builds = [b['_id'] for b in builddocs]
html = [describe_build.build_navigation(builds, build),
describe_build.describe_build(branch, build)]
columns = [('description', lambda x: a(
href='/results/build='+x['build']+'/test_case='+
x['test_case']['description']+
('/dut='+x['dut'] if x.get('dut_name') else ''))[
x['test_case']['description']])]
for dut, stuff in records:
columns2 = list(columns)
for name in process_result.CATEGORIES:
def render(field, dut):
def fn(x):
url = '/results/status='+field+'/build='+build+'/test_case='+x['test_case']['description']
if dut:
url += '/dut='+dut
return a(href=url)[x.get(field,0)]
return fn
columns2.append( (name.replace('_', ' '), render(name, dut)) )
def logs_table(request, constraint):
"""renders the latest entries in the logs capped collection"""
query = constraints.parse(constraint)
oquery = dict(query)
offset = query.pop('offset', 0)
limit = query.pop('limit', 1000)
show = query.pop('show', DEFAULT_LOGGING)
if query.get('result_id'):
rid = query['result_id'] = objectid.ObjectId(query['result_id'])
rdoc = CONNECTION.results.find_one({'_id': rid})
else:
rid = None
if query.get('job_id'):
query['job_id'] = objectid.ObjectId(query['job_id'])
if rid and request.method == 'POST':
wb = request.POST.get('whiteboard')
CONNECTION.results.update({'_id': rid}, {
'$set': {
'whiteboard':
wb,
'infrastructure_problem':
True if '[infrastructure]' in wb else False
}
})
build = rdoc.get('build')
if build:
recount(build)
postmessage = div(
Class='postmessage')['updated results for ',
str(rid), ' with whiteboard ', wb, ' ',
a(href=request.path)[em['clear']]]
else:
postmessage = []
showset = [x for x in show.split(',') if x]
kindset = LOGDB.logs.find(query).distinct('kind')
query['kind'] = {'$in': showset}
cursor = list(LOGDB.logs.find(query).skip(offset).limit(limit))
query.pop('kind')
for row in cursor:
for key in ['result_id', 'job_id']:
if key in row:
row[key] = str(row[key])
lookup = lambda term: constraints.lookup('/logs', oquery, term)
query['show'] = show
log_columns = [('time', lambda x: '%.3fs' %
(x['time'] - cursor[0]['time'])),
('kind', lambda x: x['kind']),
('message', lambda x: div(Class='log')[x['message']])]
if 'result_id' not in query:
log_columns += [('server', lambda x: x.get('automation_server', '?')),
lookup('dut'),
lookup('dut_name'),
lookup('result_id'),
lookup('job_id'),
lookup('pid')]
if cursor == []:
rel = []
else:
rel = div['Times are relative to ',
time.asctime(time.localtime(cursor[0]['time'])),
' counting from ', offset, ' of ',
LOGDB.logs.count()]
table = show_table.produce_table(cursor,
log_columns,
constraints.cross_reference(
'/logs', query),
offset=offset,
show_rows=limit,
row_fn=row_styling)
showfilter = []
q1 = dict(query)
if 'show' in q1:
del q1['show']
print 'kindset', kindset, 'showset', showset
for term in sorted(kindset):
if term in showset:
continue
nlist = showset + [term]
q2 = dict(q1, show=','.join(nlist))
rel = '/logs' + constraints.unparse(q2)
showfilter.append([span[a(href=rel)['show ', term]], ' '])
if rid:
ex = {}
rdoc = CONNECTION.results.find_one({'_id': rid})
if 'whiteboard' in rdoc:
ex['value'] = rdoc['whiteboard']
setwb = form(method='post', action='/logs/result_id=' + str(rid))[
'whiteboard=',
stan_input(type='text', size=150, name='whiteboard', **ex),
stan_input(type='submit', value='update')]
download = div[a(href="/logdownload/" +
str(rid))['Download complete log file']]
else:
setwb = []
download = []
return render_to_response(
'generic.html',
RequestContext(
request, {
'content':
html_fragment(
[postmessage, download, setwb, showfilter, rel, table])
}))
def scheduler(request):
"""Render HTML"""
def myform(text, *members):
"""Return an HTML POST form back to this page with CSRF support"""
return form(method='post', action='/scheduler')[
stan_input(type='hidden', name='operation', value=text),
list(members), stan_input(type='submit', value=text)]
post_message = None
if request.method == 'POST':
operation = request.POST['operation']
if operation == 'cancel':
row = request.POST['id']
query = {'_id':objectid.ObjectId(row)}
CONNECTION.jobs.update(query, {
'$set': {'status': 'cancelled',
'finish_time' : time()}})
post_message = ['Cancelled ', row]
elif operation == 'add to queue':
dut = request.POST.get('dut')
command = request.POST['command']
if not match('((testsuite)|(experiments)|(./autolaunch.py))[ 0-9a-zA-Z\-]+', command):
post_message = 'invalid command '+command
else:
doc = {'user': request.POST['user'], 'status':'queued',
'submit_time': time(),
'timeout': int(request.POST['timeout']),
'urgent' : 1 if request.POST.get('urgent') else 0,
'dut' : dut if dut else None,
'command': request.POST['command'].split()}
CONNECTION.jobs.save(doc)
post_message = ['Added job to queue']
else:
assert 0
if post_message:
post_html = div(**{'class':'postmessage'})[
post_message, ' ', a(href='/scheduler')[em['(clear)']]]
else:
post_html = []
queue_html = []
for title, filters, sort in [
('Queued tests', {'status':'queued'},
[ ('urgent', DESCENDING), ('submit_time', DESCENDING)]),
('Running tests', {'status':'running'},
[('launch_time', ASCENDING)]),
('Recent finished tests', {'status': {'$nin': ['queued, running']}},
[('finish_time', DESCENDING)])]:
cur = CONNECTION.jobs.find(filters).sort(sort)
if 'finished' in title:
cur = cur.limit(10)
data= list(cur)
columns = [
('user', lambda x: x.get('user')),
('command', lambda x: ' ' .join(x.get('command', ['no command']))),
('status', lambda x: a(title=x.get('failure', ''))[
x.get('status', '(no status!)')]),
('dut', lambda x: x.get('dut', 'ANY')),
('timeout', lambda x: str(x.get('timeout'))+' seconds'),
('priority', lambda x: 'urgent' if x.get('urgent') else 'normal'),
('submission time (UT)', lambda x:
(asctime(gmtime(x['submit_time'])) if
'submit_time' in x else []))]
if title != 'Recent finished tests':
columns.append(
('cancel', lambda x: myform(
'cancel',
stan_input(type='hidden', name='id',
value=str(x['_id'])))))
if title != 'Queued tests':
columns.append( ('log', lambda x: (a(href='/logs/result_id='+str(x['results_id']))['view']) if 'results_id' in x else (a(href='/logs/job_id='+str(x['_id']))['view'])))
columns.append( ('finish time (UT)', lambda x:
(asctime(gmtime(x['finish_time'])) if
'finish_time' in x else [])))
queue = show_table.produce_table(
data, columns, cross_reference('/scheduler', {}), show_nav=False, row_fn = status_styling)
queue_html += [h2[title], (queue if data else div['(none)'])]
dut_options = [option(value=dut['_id'])[dut['_id']] for dut
in CONNECTION.duts.find()]
submit_form = myform(
'add to queue',
span['User email address: ',
stan_input(type='text', name='user', size=30,
value='*****@*****.**')],
span[' Command: ',
stan_input(type='text', name='command', size=70,
value='./autolaunch.py ')],
span[' DUT: ',select(name='dut')[
option(selected='selected')['ANY'], dut_options]],
span[' Timeout: ',
stan_input(type='text', name='timeout', size=5, value='600')],
span[' Urgent: ', stan_input(type='checkbox', name='urgent',value=1), ' '])
node_html = [h2['Test Nodes']]
ul_list = []
for i in range(0, TEST_NODES):
dut_doc = CONNECTION.duts.find_one({'num':i})
if dut_doc['acquired'] == True:
ul_list.append(li[font(color='red')[dut_doc['name']]])
else:
ul_list.append(li[font(color='green')[dut_doc['name']]])
node_html.append(ul[ul_list])
html = [post_html, h2['Submit test job'], submit_form, node_html, queue_html]
return render_to_response('generic.html',
RequestContext(request, {
'content': html_fragment(html)}))
def dut_link(x):
kw = {'href': '/results/dut_name='+x['name']}
if 'mac_address' in x and 'asset_id' in x:
kw['title'] = x['mac_address']+' asset ID '+x['asset_id']
return a(**kw)[x['name']]
def logs_table(request, constraint):
"""renders the latest entries in the logs capped collection"""
query = constraints.parse(constraint)
oquery = dict(query)
offset = query.pop('offset', 0)
limit = query.pop('limit', 1000)
show = query.pop('show', DEFAULT_LOGGING)
if query.get('result_id'):
rid = query['result_id'] = objectid.ObjectId(query['result_id'])
rdoc = CONNECTION.results.find_one({'_id':rid})
else:
rid = None
if query.get('job_id'):
query['job_id'] = objectid.ObjectId(query['job_id'])
if rid and request.method == 'POST':
wb = request.POST.get('whiteboard')
CONNECTION.results.update({'_id':rid}, {
'$set':
{'whiteboard':wb, 'infrastructure_problem': True
if '[infrastructure]' in wb else False}})
build = rdoc.get('build')
if build:
recount(build)
postmessage = div(Class='postmessage')[
'updated results for ',str(rid), ' with whiteboard ', wb, ' ',
a(href=request.path)[em['clear']]]
else:
postmessage = []
showset = [x for x in show.split(',') if x]
kindset = LOGDB.logs.find(query).distinct('kind')
query['kind'] = {'$in': showset}
cursor = list(LOGDB.logs.find(query).skip(offset).limit(limit))
query.pop('kind')
for row in cursor:
for key in ['result_id', 'job_id']:
if key in row:
row[key] = str(row[key])
lookup = lambda term: constraints.lookup('/logs', oquery, term)
query['show'] = show
log_columns = [
('time', lambda x: '%.3fs' % (x['time'] - cursor[0]['time'])),
('kind', lambda x: x['kind']),
('message', lambda x: div(Class='log')[x['message']])]
if 'result_id' not in query:
log_columns += [
('server', lambda x: x.get('automation_server','?')),
lookup('dut'), lookup('dut_name'),
lookup('result_id'), lookup('job_id'),
lookup('pid')]
if cursor == []:
rel = []
else:
rel = div['Times are relative to ',
time.asctime(time.localtime(cursor[0]['time'])),
' counting from ',offset, ' of ', LOGDB.logs.count()]
table = show_table.produce_table(
cursor, log_columns,
constraints.cross_reference('/logs', query),
offset=offset, show_rows=limit, row_fn = row_styling)
showfilter = []
q1 = dict(query)
if 'show' in q1:
del q1['show']
print 'kindset', kindset, 'showset', showset
for term in sorted(kindset):
if term in showset:
continue
nlist = showset+[term]
q2 = dict(q1, show=','.join(nlist))
rel = '/logs'+constraints.unparse(q2)
showfilter.append ( [span[a(href=rel)['show ',term]], ' '])
if rid:
ex = {}
rdoc = CONNECTION.results.find_one({'_id':rid})
if 'whiteboard' in rdoc:
ex['value'] = rdoc['whiteboard']
setwb = form(method='post', action='/logs/result_id='+str(rid))[
'whiteboard=',
stan_input(type='text', size=150, name='whiteboard', **ex),
stan_input(type='submit', value='update')]
download = div[a(href="/logdownload/"+str(rid))[
'Download complete log file']]
else:
setwb = []
download = []
return render_to_response('generic.html',
RequestContext(request, {
'content': html_fragment([postmessage, download, setwb,
showfilter,rel,table])}))
def duts(request, constraint):
"""Render duts table"""
if request.method == 'POST':
operation = request.POST.get('operation')
if operation == 'update':
query = {'_id': request.POST['dut']}
olddoc = CONNECTION.duts.find_one(query)
update = {}
update['development_mode'] = \
0 if request.POST.get('publish') else 1
update['run_bvt'] = 1 if request.POST.get('run_bvt') else 0
for field in ['owner', 'notes', 'control_machine',
'power_control', 'serial_port', 'tag_regexp']:
if field in request.POST:
value = request.POST[field]
update[field] = value
CONNECTION.duts.update( query, {'$set': update})
# this fails since set_pxe_build calls src.bvtlib.run.run which
# uses signals and so can only run from main loop
# so instead we rely on people to clear up the PXE server
# if not update['run_bvt']:
# set_pxe_build(olddoc['name'], None, 'boot')
postmessage = 'updated '+request.POST['dut']+' with '+repr(update)+' as '+whoami()
else:
postmessage= 'bad form operation '+repr(operation)
post_stuff = div(**{'class':'postmessage'})[postmessage,
a(href='/duts')[em['(clear)']]]
else:
post_stuff = []
t1 = time()
data = model(time()-26*60*60)
t2 = time()
oquery = parse(constraint)
query = dict(oquery)
hours = query.pop('hours', 3)
columns = [('dut', dut_link),
('notes', lambda x:
stan_input(type='text', width=20, name='notes',
value=x.get('notes',''))),
('owner', lambda x:
stan_input(type='text', width=20, name='owner',
value=x.get('owner',''))),
('BVT', lambda x: stan_input(
type='checkbox',
name='run_bvt', **
({'checked':1} if x.get('run_bvt', 0) else {}))),
edit_row('tag_regexp'),
edit_row('control_machine', 'control_machine'),
edit_row('power_control', 'power'),
edit_row('serial_port', 'serial'),
('publish', lambda x: stan_input(
type='checkbox', name='publish', **({} if
x.get('development_mode', 0) else {'checked':'on'}))),
('change', lambda x: [
stan_input(type='hidden', name='operation', value='update'),
stan_input(type='hidden', name='dut', value=str(x['_id'])),
stan_input(type='hidden', name='csrftoken',
value=csrf(request)),
stan_input(type='submit', value='change')]),
('platform', lambda x: x.get('platform', '')),
('make', lambda x: x.get('make', '')),
('model', lambda x: x.get('model', em['run labeller!'])),
('memory', lambda x: '%dMB' % int(x['memory'] / (1e6)) if
'memory' in x else '')]
tt = localtime()
ohour = tt.tm_hour
for i in range(hours,-1, -1):
nhour = ohour - i
if nhour < 0:
nhour += 24
daybump = True
else:
daybump = False
t2 = (tt.tm_year, tt.tm_mon, tt.tm_mday, nhour,
0, 0, tt.tm_wday, tt.tm_yday, tt.tm_isdst)
t2epoch = mktime(t2)
if daybump:
t2epoch -= 24* 60 * 60
columns.append( countcol(t2epoch, i))
columns.append( ('run', run_time))
def row_fn(doc, columns):
style = {'class':'pass'} if (doc.get('owner', '') == '' and \
doc.get('run_bvt')) else {}
return tr(**style)[form(method='post', action='/duts')[columns]]
html = [post_stuff,
produce_table(data, columns, cross_reference('/duts', oquery), show_nav=False,
row_fn = row_fn), INSTRUCTIONS]
return render_to_response('generic.html',
RequestContext(request, {
'content': html_fragment(html)}))