def test_insert(self):
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
report.items = [
{'file': 'tests/foo.c', 'status': 'failure'},
{'file': 'tests/bar.c', 'status': 'success'}
]
report.insert()
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT build,step,category,generator "
"FROM bitten_report WHERE id=%s", (report.id,))
self.assertEqual((1, 'test', 'test', 'unittest'),
cursor.fetchone())
cursor.execute("SELECT item,name,value FROM bitten_report_item "
"WHERE report=%s ORDER BY item", (report.id,))
items = []
prev_item = None
for item, name, value in cursor:
if item != prev_item:
items.append({name: value})
prev_item = item
else:
items[-1][name] = value
self.assertEquals(2, len(items))
seen_foo, seen_bar = False, False
for item in items:
if item['file'] == 'tests/foo.c':
self.assertEqual('failure', item['status'])
seen_foo = True
if item['file'] == 'tests/bar.c':
self.assertEqual('success', item['status'])
seen_bar = True
self.assertEquals((True, True), (seen_foo, seen_bar))
def test_insert_dupe(self):
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
report.insert()
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
self.assertRaises(AssertionError, report.insert)
def test_insert_empty_items(self):
report = Report(self.env, build=1, step='test', category='test',
generator='unittest')
report.items = [{}, {}]
report.insert()
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT build,step,category,generator "
"FROM bitten_report WHERE id=%s", (report.id,))
self.assertEqual((1, 'test', 'test', 'unittest'),
cursor.fetchone())
cursor.execute("SELECT COUNT(*) FROM bitten_report_item "
"WHERE report=%s", (report.id,))
self.assertEqual(0, cursor.fetchone()[0])
def xmldb_to_db(env, db):
"""Migrate report data from Berkeley DB XML to SQL database.
Depending on the number of reports stored, this might take rather long.
After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
may be deleted. BDB XML is no longer used by Bitten.
"""
from bitten.model import Report
from bitten.util import xmlio
try:
from bsddb3 import db as bdb
import dbxml
except ImportError:
return
dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
if not os.path.isfile(dbfile):
return
dbenv = bdb.DBEnv()
dbenv.open(os.path.dirname(dbfile),
bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
mgr = dbxml.XmlManager(dbenv, 0)
xtn = mgr.createTransaction()
container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
def get_pylint_items(xml):
for problems_elem in xml.children('problems'):
for problem_elem in problems_elem.children('problem'):
item = {'type': 'problem'}
item.update(problem_elem.attr)
yield item
def get_trace_items(xml):
for cov_elem in xml.children('coverage'):
item = {'type': 'coverage', 'name': cov_elem.attr['module'],
'file': cov_elem.attr['file'],
'percentage': cov_elem.attr['percentage']}
lines = 0
line_hits = []
for line_elem in cov_elem.children('line'):
lines += 1
line_hits.append(line_elem.attr['hits'])
item['lines'] = lines
item['line_hits'] = ' '.join(line_hits)
yield item
def get_unittest_items(xml):
for test_elem in xml.children('test'):
item = {'type': 'test'}
item.update(test_elem.attr)
for child_elem in test_elem.children():
item[child_elem.name] = child_elem.gettext()
yield item
qc = mgr.createQueryContext()
for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
doc = value.asDocument()
metaval = dbxml.XmlValue()
build, step = None, None
if doc.getMetaData('', 'build', metaval):
build = metaval.asNumber()
if doc.getMetaData('', 'step', metaval):
step = metaval.asString()
report_types = {'pylint': ('lint', get_pylint_items),
'trace': ('coverage', get_trace_items),
'unittest': ('test', get_unittest_items)}
xml = xmlio.parse(value.asString())
report_type = xml.attr['type']
category, get_items = report_types[report_type]
sys.stderr.write('.')
sys.stderr.flush()
report = Report(env, build, step, category=category,
generator=report_type)
report.items = list(get_items(xml))
try:
report.insert(db=db)
except AssertionError:
# Duplicate report, skip
pass
sys.stderr.write('\n')
sys.stderr.flush()
xtn.abort()
container.close()
dbenv.close(0)
build_log.messages.append((message_elem.attr['level'],
message_elem.gettext()))
build_log.insert(db=db)
# Collect report data from the request body
for report_elem in elem.children('report'):
report = Report(self.env, build=build.id, step=stepname,
category=report_elem.attr.get('category'),
generator=report_elem.attr.get('generator'))
for item_elem in report_elem.children():
item = {'type': item_elem.name}
item.update(item_elem.attr)
for child_elem in item_elem.children():
item[child_elem.name] = child_elem.gettext()
report.items.append(item)
report.insert(db=db)
# If this was the last step in the recipe we mark the build as
# completed otherwise just update last_activity
if last_step:
self.log.info('Slave %s completed build %d ("%s" as of [%s])',
build.slave, build.id, build.config, build.rev)
build.stopped = step.stopped
build.last_activity = build.stopped
# Determine overall outcome of the build by checking the outcome
# of the individual steps against the "onerror" specification of
# each step in the recipe
for num, recipe_step in enumerate(recipe):
step = BuildStep.fetch(self.env, build.id, recipe_step.id)
if step.status == BuildStep.FAILURE:
build_log.insert(db=db)
# Collect report data from the request body
for report_elem in elem.children('report'):
report = Report(self.env,
build=build.id,
step=stepname,
category=report_elem.attr.get('category'),
generator=report_elem.attr.get('generator'))
for item_elem in report_elem.children():
item = {'type': item_elem.name}
item.update(item_elem.attr)
for child_elem in item_elem.children():
item[child_elem.name] = child_elem.gettext()
report.items.append(item)
report.insert(db=db)
# If this was the last step in the recipe we mark the build as
# completed otherwise just update last_activity
if last_step:
self.log.info('Slave %s completed build %d ("%s" as of [%s])',
build.slave, build.id, build.config, build.rev)
build.stopped = step.stopped
build.last_activity = build.stopped
# Determine overall outcome of the build by checking the outcome
# of the individual steps against the "onerror" specification of
# each step in the recipe
for num, recipe_step in enumerate(recipe):
step = BuildStep.fetch(self.env, build.id, recipe_step.id)
if step.status == BuildStep.FAILURE:
def xmldb_to_db(env, db):
"""Migrate report data from Berkeley DB XML to SQL database.
Depending on the number of reports stored, this might take rather long.
After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
may be deleted. BDB XML is no longer used by Bitten.
"""
from bitten.model import Report
from bitten.util import xmlio
try:
from bsddb3 import db as bdb
import dbxml
except ImportError:
return
dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
if not os.path.isfile(dbfile):
return
dbenv = bdb.DBEnv()
dbenv.open(
os.path.dirname(dbfile), bdb.DB_CREATE | bdb.DB_INIT_LOCK
| bdb.DB_INIT_LOG | bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
mgr = dbxml.XmlManager(dbenv, 0)
xtn = mgr.createTransaction()
container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
def get_pylint_items(xml):
for problems_elem in xml.children('problems'):
for problem_elem in problems_elem.children('problem'):
item = {'type': 'problem'}
item.update(problem_elem.attr)
yield item
def get_trace_items(xml):
for cov_elem in xml.children('coverage'):
item = {
'type': 'coverage',
'name': cov_elem.attr['module'],
'file': cov_elem.attr['file'],
'percentage': cov_elem.attr['percentage']
}
lines = 0
line_hits = []
for line_elem in cov_elem.children('line'):
lines += 1
line_hits.append(line_elem.attr['hits'])
item['lines'] = lines
item['line_hits'] = ' '.join(line_hits)
yield item
def get_unittest_items(xml):
for test_elem in xml.children('test'):
item = {'type': 'test'}
item.update(test_elem.attr)
for child_elem in test_elem.children():
item[child_elem.name] = child_elem.gettext()
yield item
qc = mgr.createQueryContext()
for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
doc = value.asDocument()
metaval = dbxml.XmlValue()
build, step = None, None
if doc.getMetaData('', 'build', metaval):
build = metaval.asNumber()
if doc.getMetaData('', 'step', metaval):
step = metaval.asString()
report_types = {
'pylint': ('lint', get_pylint_items),
'trace': ('coverage', get_trace_items),
'unittest': ('test', get_unittest_items)
}
xml = xmlio.parse(value.asString())
report_type = xml.attr['type']
category, get_items = report_types[report_type]
sys.stderr.write('.')
sys.stderr.flush()
report = Report(env,
build,
step,
category=category,
generator=report_type)
report.items = list(get_items(xml))
try:
report.insert(db=db)
except AssertionError:
# Duplicate report, skip
pass
sys.stderr.write('\n')
sys.stderr.flush()
xtn.abort()
container.close()
dbenv.close(0)
