def test_insert(self):
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
report.items = [
{'file': 'tests/foo.c', 'status': 'failure'},
{'file': 'tests/bar.c', 'status': 'success'}
]
report.insert()
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT build,step,category,generator "
"FROM bitten_report WHERE id=%s", (report.id,))
self.assertEqual((1, 'test', 'test', 'unittest'),
cursor.fetchone())
cursor.execute("SELECT item,name,value FROM bitten_report_item "
"WHERE report=%s ORDER BY item", (report.id,))
items = []
prev_item = None
for item, name, value in cursor:
if item != prev_item:
items.append({name: value})
prev_item = item
else:
items[-1][name] = value
self.assertEquals(2, len(items))
seen_foo, seen_bar = False, False
for item in items:
if item['file'] == 'tests/foo.c':
self.assertEqual('failure', item['status'])
seen_foo = True
if item['file'] == 'tests/bar.c':
self.assertEqual('success', item['status'])
seen_bar = True
self.assertEquals((True, True), (seen_foo, seen_bar))
def test_insert_empty_items(self):
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
report.items = [{}, {}]
report.insert()
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT build,step,category,generator "
"FROM bitten_report WHERE id=%s", (report.id,))
self.assertEqual((1, 'test', 'test', 'unittest'),
cursor.fetchone())
cursor.execute("SELECT COUNT(*) FROM bitten_report_item "
"WHERE report=%s", (report.id,))
self.assertEqual(0, cursor.fetchone()[0])
def get_annotation_data(self, context):
add_stylesheet(context.req, 'bitten/bitten_coverage.css')
resource = context.resource
self.log.debug("Looking for coverage report for %s@%s..." % (
resource.id, str(resource.version)))
builds = Build.select(self.env, rev=resource.version)
reports = []
for build in builds:
config = BuildConfig.fetch(self.env, build.config)
if not resource.id.startswith(config.branch):
continue
reports = Report.select(self.env, build=build.id,
category='coverage')
branch_in_config = resource.id[len(config.branch):]
for report in reports:
for item in report.items:
if item.get('file') == branch_in_config:
coverage = item.get('line_hits', '').split()
if coverage:
# Return first result with line data
self.log.debug(
"Coverage annotate for %s@%s: %s" % \
(resource.id, resource.version, coverage))
return coverage
return []
def get_annotation_data(self, context):
"""add annotation data for lint"""
context.perm.require('BUILD_VIEW')
add_stylesheet(context.req, 'bitten/bitten_coverage.css')
add_stylesheet(context.req, 'bitten/bitten_lintannotator.css')
resource = context.resource
# attempt to use the version passed in with the request,
# otherwise fall back to the latest version of this file.
try:
version = context.req.args['rev']
except (KeyError, TypeError):
version = resource.version
self.log.debug('no version passed to get_annotation_data')
builds = Build.select(self.env, rev=version)
self.log.debug("Looking for lint report for %s@%s [%s]..." %
(resource.id, str(resource.version), version))
self.itemid = 0
data = {}
reports = None
for build in builds:
config = BuildConfig.fetch(self.env, build.config)
if not resource.id.lstrip('/').startswith(config.path.lstrip('/')):
self.log.debug('Skip build %s' % build)
continue
path_in_config = resource.id[len(config.path) + 1:].lstrip('/')
reports = Report.select(self.env, build=build.id, category='lint')
for report in reports:
for item in report.items:
if item.get('file') == path_in_config:
line = item.get('line')
if line:
problem = {
'category': item.get('category', ''),
'tag': item.get('tag', ''),
'bid': build.id,
'rbuild': report.build,
'rstep': report.step,
'rid': report.id
}
data.setdefault(int(line), []).append(problem)
if data:
self.log.debug("Lint annotate for %s@%s: %s results" % \
(resource.id, resource.version, len(data)))
return data
if not builds:
self.log.debug("No builds found")
elif not reports:
self.log.debug("No reports found")
else:
self.log.debug("No item of any report matched (%s)" % reports)
return None
def test_delete(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(1, 'test', 'test', 'unittest'))
report_id = db.get_last_id(cursor, 'bitten_report')
cursor.executemany("INSERT INTO bitten_report_item "
"(report,item,name,value) VALUES (%s,%s,%s,%s)",
[(report_id, 0, 'file', 'tests/foo.c'),
(report_id, 0, 'result', 'failure'),
(report_id, 1, 'file', 'tests/bar.c'),
(report_id, 1, 'result', 'success')])
report = Report.fetch(self.env, report_id, db=db)
report.delete(db=db)
self.assertEqual(False, report.exists)
report = Report.fetch(self.env, report_id, db=db)
self.assertEqual(None, report)
def get_annotation_data(self, context):
"""add annotation data for lint"""
context.perm.require('BUILD_VIEW')
add_stylesheet(context.req, 'bitten/bitten_coverage.css')
add_stylesheet(context.req, 'bitten/bitten_lintannotator.css')
resource = context.resource
# attempt to use the version passed in with the request,
# otherwise fall back to the latest version of this file.
try:
version = context.req.args['rev']
except (KeyError, TypeError):
version = resource.version
self.log.debug('no version passed to get_annotation_data')
builds = Build.select(self.env, rev=version)
self.log.debug("Looking for lint report for %s@%s [%s]..." % (
resource.id, str(resource.version), version))
self.itemid = 0
data = {}
reports = None
for build in builds:
config = BuildConfig.fetch(self.env, build.config)
if not resource.id.lstrip('/').startswith(config.path.lstrip('/')):
self.log.debug('Skip build %s' % build)
continue
path_in_config = resource.id[len(config.path)+1:].lstrip('/')
reports = Report.select(self.env, build=build.id, category='lint')
for report in reports:
for item in report.items:
if item.get('file') == path_in_config:
line = item.get('line')
if line:
problem = {'category': item.get('category', ''),
'tag': item.get('tag', ''),
'bid': build.id,
'rbuild': report.build,
'rstep': report.step, 'rid': report.id}
data.setdefault(int(line), []).append(problem)
if data:
self.log.debug("Lint annotate for %s@%s: %s results" % \
(resource.id, resource.version, len(data)))
return data
if not builds:
self.log.debug("No builds found")
elif not reports:
self.log.debug("No reports found")
else:
self.log.debug("No item of any report matched (%s)" % reports)
return None
def _render_reports(self, req, config, build, summarizers, step):
reports = []
for report in Report.select(self.env, build=build.id, step=step.name):
summarizer = summarizers.get(report.category)
if summarizer:
tmpl, data = summarizer.render_summary(req, config, build,
step, report.category)
else:
tmpl = data = None
reports.append({'category': report.category,
'template': tmpl, 'data': data})
return reports
def _render_reports(self, req, config, build, summarizers, step):
reports = []
for report in Report.select(self.env, build=build.id, step=step.name):
summarizer = summarizers.get(report.category)
if summarizer:
tmpl, data = summarizer.render_summary(req, config, build,
step, report.category)
reports.append({'category': report.category,
'template': tmpl, 'data': data})
else:
tmpl = data = None
return reports
def test_insert_dupe(self):
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
report.insert()
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
self.assertRaises(AssertionError, report.insert)
def test_select(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(1, 'test', 'test', 'unittest'))
report1_id = db.get_last_id(cursor, 'bitten_report')
cursor.execute(
"INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(1, 'test', 'coverage', 'trace'))
report2_id = db.get_last_id(cursor, 'bitten_report')
cursor.executemany(
"INSERT INTO bitten_report_item "
"(report,item,name,value) VALUES (%s,%s,%s,%s)",
[(report1_id, 0, 'file', 'tests/foo.c'),
(report1_id, 0, 'result', 'failure'),
(report1_id, 1, 'file', 'tests/bar.c'),
(report1_id, 1, 'result', 'success'),
(report2_id, 0, 'file', 'tests/foo.c'),
(report2_id, 0, 'loc', '12'), (report2_id, 0, 'cov', '50'),
(report2_id, 1, 'file', 'tests/bar.c'),
(report2_id, 1, 'loc', '20'), (report2_id, 1, 'cov', '25')])
reports = Report.select(self.env, build=1, step='test')
for idx, report in enumerate(reports):
if report.id == report1_id:
self.assertEquals('test', report.step)
self.assertEquals('test', report.category)
self.assertEquals('unittest', report.generator)
self.assertEquals(2, len(report.items))
assert {'file': 'tests/foo.c', 'result': 'failure'} \
in report.items
assert {'file': 'tests/bar.c', 'result': 'success'} \
in report.items
elif report.id == report1_id:
self.assertEquals('test', report.step)
self.assertEquals('coverage', report.category)
self.assertEquals('trace', report.generator)
self.assertEquals(2, len(report.items))
assert {'file': 'tests/foo.c', 'loc': '12', 'cov': '50'} \
in report.items
assert {'file': 'tests/bar.c', 'loc': '20', 'cov': '25'} \
in report.items
self.assertEqual(1, idx)
def test_select(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(1, 'test', 'test', 'unittest'))
report1_id = db.get_last_id(cursor, 'bitten_report')
cursor.execute("INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(1, 'test', 'coverage', 'trace'))
report2_id = db.get_last_id(cursor, 'bitten_report')
cursor.executemany("INSERT INTO bitten_report_item "
"(report,item,name,value) VALUES (%s,%s,%s,%s)",
[(report1_id, 0, 'file', 'tests/foo.c'),
(report1_id, 0, 'result', 'failure'),
(report1_id, 1, 'file', 'tests/bar.c'),
(report1_id, 1, 'result', 'success'),
(report2_id, 0, 'file', 'tests/foo.c'),
(report2_id, 0, 'loc', '12'),
(report2_id, 0, 'cov', '50'),
(report2_id, 1, 'file', 'tests/bar.c'),
(report2_id, 1, 'loc', '20'),
(report2_id, 1, 'cov', '25')])
reports = Report.select(self.env, build=1, step='test')
for idx, report in enumerate(reports):
if report.id == report1_id:
self.assertEquals('test', report.step)
self.assertEquals('test', report.category)
self.assertEquals('unittest', report.generator)
self.assertEquals(2, len(report.items))
assert {'file': 'tests/foo.c', 'result': 'failure'} \
in report.items
assert {'file': 'tests/bar.c', 'result': 'success'} \
in report.items
elif report.id == report1_id:
self.assertEquals('test', report.step)
self.assertEquals('coverage', report.category)
self.assertEquals('trace', report.generator)
self.assertEquals(2, len(report.items))
assert {'file': 'tests/foo.c', 'loc': '12', 'cov': '50'} \
in report.items
assert {'file': 'tests/bar.c', 'loc': '20', 'cov': '25'} \
in report.items
self.assertEqual(1, idx)
def get_annotation_data(self, context):
add_stylesheet(context.req, 'bitten/bitten_coverage.css')
resource = context.resource
builds = Build.select(self.env, rev=resource.version)
reports = []
for build in builds:
config = BuildConfig.fetch(self.env, build.config)
if not resource.id.startswith(config.path):
continue
reports = Report.select(self.env, build=build.id,
category='coverage')
path_in_config = resource.id[len(config.path):].lstrip('/')
for report in reports:
for item in report.items:
if item.get('file') == path_in_config:
# TODO should aggregate coverage across builds
return item.get('line_hits', '').split()
return []
def test_fetch(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(1, 'test', 'test', 'unittest'))
report_id = db.get_last_id(cursor, 'bitten_report')
cursor.executemany("INSERT INTO bitten_report_item "
"(report,item,name,value) VALUES (%s,%s,%s,%s)",
[(report_id, 0, 'file', 'tests/foo.c'),
(report_id, 0, 'result', 'failure'),
(report_id, 1, 'file', 'tests/bar.c'),
(report_id, 1, 'result', 'success')])
report = Report.fetch(self.env, report_id)
self.assertEquals(report_id, report.id)
self.assertEquals('test', report.step)
self.assertEquals('test', report.category)
self.assertEquals('unittest', report.generator)
self.assertEquals(2, len(report.items))
assert {'file': 'tests/foo.c', 'result': 'failure'} in report.items
assert {'file': 'tests/bar.c', 'result': 'success'} in report.items
def test_process_build_step_success_with_report(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123';
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<report category="test"
generator="http://bitten.edgewall.org/tools/python#unittest">
<test fixture="my.Fixture" file="my/test/file.py">
<stdout>Doing my thing</stdout>
</test>
</report>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
reports = list(Report.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(reports))
self.assertEqual('test', reports[0].category)
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
reports[0].generator)
self.assertEqual(1, len(reports[0].items))
self.assertEqual({
'fixture': 'my.Fixture',
'file': 'my/test/file.py',
'stdout': 'Doing my thing',
'type': 'test',
}, reports[0].items[0])
def xmldb_to_db(env, db):
"""Migrate report data from Berkeley DB XML to SQL database.
Depending on the number of reports stored, this might take rather long.
After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
may be deleted. BDB XML is no longer used by Bitten.
"""
from bitten.model import Report
from bitten.util import xmlio
try:
from bsddb3 import db as bdb
import dbxml
except ImportError:
return
dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
if not os.path.isfile(dbfile):
return
dbenv = bdb.DBEnv()
dbenv.open(
os.path.dirname(dbfile), bdb.DB_CREATE | bdb.DB_INIT_LOCK
| bdb.DB_INIT_LOG | bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
mgr = dbxml.XmlManager(dbenv, 0)
xtn = mgr.createTransaction()
container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
def get_pylint_items(xml):
for problems_elem in xml.children('problems'):
for problem_elem in problems_elem.children('problem'):
item = {'type': 'problem'}
item.update(problem_elem.attr)
yield item
def get_trace_items(xml):
for cov_elem in xml.children('coverage'):
item = {
'type': 'coverage',
'name': cov_elem.attr['module'],
'file': cov_elem.attr['file'],
'percentage': cov_elem.attr['percentage']
}
lines = 0
line_hits = []
for line_elem in cov_elem.children('line'):
lines += 1
line_hits.append(line_elem.attr['hits'])
item['lines'] = lines
item['line_hits'] = ' '.join(line_hits)
yield item
def get_unittest_items(xml):
for test_elem in xml.children('test'):
item = {'type': 'test'}
item.update(test_elem.attr)
for child_elem in test_elem.children():
item[child_elem.name] = child_elem.gettext()
yield item
qc = mgr.createQueryContext()
for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
doc = value.asDocument()
metaval = dbxml.XmlValue()
build, step = None, None
if doc.getMetaData('', 'build', metaval):
build = metaval.asNumber()
if doc.getMetaData('', 'step', metaval):
step = metaval.asString()
report_types = {
'pylint': ('lint', get_pylint_items),
'trace': ('coverage', get_trace_items),
'unittest': ('test', get_unittest_items)
}
xml = xmlio.parse(value.asString())
report_type = xml.attr['type']
category, get_items = report_types[report_type]
sys.stderr.write('.')
sys.stderr.flush()
report = Report(env,
build,
step,
category=category,
generator=report_type)
report.items = list(get_items(xml))
try:
report.insert(db=db)
except AssertionError:
# Duplicate report, skip
pass
sys.stderr.write('\n')
sys.stderr.flush()
xtn.abort()
container.close()
dbenv.close(0)
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
rev = config.max_rev or repos.youngest_rev
try:
_has_permission(req.perm,
repos,
config.path,
rev=rev,
raise_error=True)
except NoSuchNode:
raise TracError("Permission checking against repository path %s "
"at revision %s failed." % (config.path, rev))
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(
Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))
data['config'] = {
'name': config.name,
'label': config.label,
'path': config.path,
'min_rev': config.min_rev,
'min_rev_href': req.href.changeset(config.min_rev),
'max_rev': config.max_rev,
'max_rev_href': req.href.changeset(config.max_rev),
'active': config.active,
'description': description,
'browser_href': req.href.browser(config.path),
'builds_pending': len(pending_builds),
'builds_inprogress': len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
platforms = list(
TargetPlatform.select(self.env, config=config_name, db=db))
data['config']['platforms'] = [{
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
} for platform in platforms]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
report_categories = list(
self._report_categories_for_config(config))
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
if category in report_categories:
chart_generators.append({
'href':
req.href.build(config.name, 'chart/' + category),
'category':
category,
'style':
self.config.get('bitten', 'chart_style'),
})
data['config']['charts'] = chart_generators
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
revisions = []
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
if rev not in builds:
revisions.append(rev)
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
builds[rev].setdefault('display_rev', repos.normalize_rev(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env,
build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
data['config']['revisions'] = revisions
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
if arity(prevnext_nav) == 4: # Trac 0.12 compat, see #450
prevnext_nav(req, 'Previous Page', 'Next Page')
else:
prevnext_nav(req, 'Page')
return data
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
rev = config.max_rev or repos.youngest_rev
try:
_has_permission(req.perm, repos, config.path, rev=rev,
raise_error=True)
except NoSuchNode:
raise TracError("Permission checking against repository path %s "
"at revision %s failed." % (config.path, rev))
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(Build.select(self.env,
config=config.name, status=Build.PENDING))
inprogress_builds = list(Build.select(self.env,
config=config.name, status=Build.IN_PROGRESS))
data['config'] = {
'name': config.name, 'label': config.label, 'path': config.path,
'min_rev': config.min_rev,
'min_rev_href': req.href.changeset(config.min_rev),
'max_rev': config.max_rev,
'max_rev_href': req.href.changeset(config.max_rev),
'active': config.active, 'description': description,
'browser_href': req.href.browser(config.path),
'builds_pending' : len(pending_builds),
'builds_inprogress' : len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(self.env).attachment_data(context)
platforms = list(TargetPlatform.select(self.env, config=config_name,
db=db))
data['config']['platforms'] = [
{ 'name': platform.name,
'id': platform.id,
'builds_pending': len(list(Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress': len(list(Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
}
for platform in platforms
]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
report_categories = list(self._report_categories_for_config(config))
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
if category in report_categories:
chart_generators.append({
'href': req.href.build(config.name, 'chart/' + category),
'category': category,
'style': self.config.get('bitten', 'chart_style'),
})
data['config']['charts'] = chart_generators
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
revisions = []
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
if rev not in builds:
revisions.append(rev)
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
builds[rev].setdefault('display_rev', repos.normalize_rev(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
data['config']['revisions'] = revisions
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
if arity(prevnext_nav) == 4: # Trac 0.12 compat, see #450
prevnext_nav(req, 'Previous Page', 'Next Page')
else:
prevnext_nav (req, 'Page')
return data
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(req.authname)
repos.authz.assert_permission(config.branch)
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(
Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))
rev = ''
for b in repos.git.get_branches():
if b[0] == config.branch:
rev = b[1]
break
data['config'] = {
'name': config.name,
'label': config.label,
'branch': config.branch,
'active': config.active,
'description': description,
'browser_href': req.href.browser(rev=rev),
'builds_pending': len(pending_builds),
'builds_inprogress': len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
platforms = list(
TargetPlatform.select(self.env, config=config_name, db=db))
data['config']['platforms'] = [{
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
} for platform in platforms]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
chart_generators.append({
'href':
req.href.build(config.name, 'chart/' + category)
})
data['config']['charts'] = chart_generators
charts_license = self.config.get('bitten', 'charts_license')
if charts_license:
data['config']['charts_license'] = charts_license
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(req.authname)
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env,
build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped, utc) - \
to_datetime(step.started, utc),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
prevnext_nav(req, 'Page')
return data
def xmldb_to_db(env, db):
"""Migrate report data from Berkeley DB XML to SQL database.
Depending on the number of reports stored, this might take rather long.
After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
may be deleted. BDB XML is no longer used by Bitten.
"""
from bitten.model import Report
from bitten.util import xmlio
try:
from bsddb3 import db as bdb
import dbxml
except ImportError:
return
dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
if not os.path.isfile(dbfile):
return
dbenv = bdb.DBEnv()
dbenv.open(os.path.dirname(dbfile),
bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
mgr = dbxml.XmlManager(dbenv, 0)
xtn = mgr.createTransaction()
container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
def get_pylint_items(xml):
for problems_elem in xml.children('problems'):
for problem_elem in problems_elem.children('problem'):
item = {'type': 'problem'}
item.update(problem_elem.attr)
yield item
def get_trace_items(xml):
for cov_elem in xml.children('coverage'):
item = {'type': 'coverage', 'name': cov_elem.attr['module'],
'file': cov_elem.attr['file'],
'percentage': cov_elem.attr['percentage']}
lines = 0
line_hits = []
for line_elem in cov_elem.children('line'):
lines += 1
line_hits.append(line_elem.attr['hits'])
item['lines'] = lines
item['line_hits'] = ' '.join(line_hits)
yield item
def get_unittest_items(xml):
for test_elem in xml.children('test'):
item = {'type': 'test'}
item.update(test_elem.attr)
for child_elem in test_elem.children():
item[child_elem.name] = child_elem.gettext()
yield item
qc = mgr.createQueryContext()
for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
doc = value.asDocument()
metaval = dbxml.XmlValue()
build, step = None, None
if doc.getMetaData('', 'build', metaval):
build = metaval.asNumber()
if doc.getMetaData('', 'step', metaval):
step = metaval.asString()
report_types = {'pylint': ('lint', get_pylint_items),
'trace': ('coverage', get_trace_items),
'unittest': ('test', get_unittest_items)}
xml = xmlio.parse(value.asString())
report_type = xml.attr['type']
category, get_items = report_types[report_type]
sys.stderr.write('.')
sys.stderr.flush()
report = Report(env, build, step, category=category,
generator=report_type)
report.items = list(get_items(xml))
try:
report.insert(db=db)
except AssertionError:
# Duplicate report, skip
pass
sys.stderr.write('\n')
sys.stderr.flush()
xtn.abort()
container.close()
dbenv.close(0)
step.insert(db=db)
# Collect log messages from the request body
for idx, log_elem in enumerate(elem.children('log')):
build_log = BuildLog(self.env, build=build.id, step=stepname,
generator=log_elem.attr.get('generator'),
orderno=idx)
for message_elem in log_elem.children('message'):
build_log.messages.append((message_elem.attr['level'],
message_elem.gettext()))
build_log.insert(db=db)
# Collect report data from the request body
for report_elem in elem.children('report'):
report = Report(self.env, build=build.id, step=stepname,
category=report_elem.attr.get('category'),
generator=report_elem.attr.get('generator'))
for item_elem in report_elem.children():
item = {'type': item_elem.name}
item.update(item_elem.attr)
for child_elem in item_elem.children():
item[child_elem.name] = child_elem.gettext()
report.items.append(item)
report.insert(db=db)
# If this was the last step in the recipe we mark the build as
# completed otherwise just update last_activity
if last_step:
self.log.info('Slave %s completed build %d ("%s" as of [%s])',
build.slave, build.id, build.config, build.rev)
build.stopped = step.stopped
for idx, log_elem in enumerate(elem.children('log')):
build_log = BuildLog(self.env,
build=build.id,
step=stepname,
generator=log_elem.attr.get('generator'),
orderno=idx)
for message_elem in log_elem.children('message'):
build_log.messages.append(
(message_elem.attr['level'], message_elem.gettext()))
build_log.insert(db=db)
# Collect report data from the request body
for report_elem in elem.children('report'):
report = Report(self.env,
build=build.id,
step=stepname,
category=report_elem.attr.get('category'),
generator=report_elem.attr.get('generator'))
for item_elem in report_elem.children():
item = {'type': item_elem.name}
item.update(item_elem.attr)
for child_elem in item_elem.children():
item[child_elem.name] = child_elem.gettext()
report.items.append(item)
report.insert(db=db)
# If this was the last step in the recipe we mark the build as
# completed otherwise just update last_activity
if last_step:
self.log.info('Slave %s completed build %d ("%s" as of [%s])',
build.slave, build.id, build.config, build.rev)
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
repos = self.env.get_repository(req.authname)
if hasattr(repos, 'sync'):
repos.sync()
repos.authz.assert_permission(config.path)
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS))
data['config'] = {
'name': config.name, 'label': config.label, 'path': config.path,
'min_rev': config.min_rev,
'min_rev_href': req.href.changeset(config.min_rev),
'max_rev': config.max_rev,
'max_rev_href': req.href.changeset(config.max_rev),
'active': config.active, 'description': description,
'browser_href': req.href.browser(config.path),
'builds_pending' : len(pending_builds),
'builds_inprogress' : len(inprogress_builds)
}
platforms = list(TargetPlatform.select(self.env, config=config_name,
db=db))
data['config']['platforms'] = [
{ 'name': platform.name,
'id': platform.id,
'builds_pending': len(list(Build.select(self.env, config=config.name, status=Build.PENDING, platform=platform.id))),
'builds_inprogress': len(list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS, platform=platform.id)))
}
for platform in platforms
]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
chart_generators.append({
'href': req.href.build(config.name, 'chart/' + category)
})
data['config']['charts'] = chart_generators
charts_license = self.config.get('bitten', 'charts_license')
if charts_license:
data['config']['charts_license'] = charts_license
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(req.authname)
if hasattr(repos, 'sync'):
repos.sync()
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': datetime.fromtimestamp(step.stopped) - \
datetime.fromtimestamp(step.started),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
prevnext_nav(req, 'Page')
return data
def test_process_build_step_success_with_report(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<report category="test"
generator="http://bitten.edgewall.org/tools/python#unittest">
<test fixture="my.Fixture" file="my/test/file.py">
<stdout>Doing my thing</stdout>
</test>
</report>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
reports = list(Report.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(reports))
self.assertEqual('test', reports[0].category)
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
reports[0].generator)
self.assertEqual(1, len(reports[0].items))
self.assertEqual(
{
'fixture': 'my.Fixture',
'file': 'my/test/file.py',
'stdout': 'Doing my thing',
'type': 'test',
}, reports[0].items[0])
