def add_config_to_reports(env, db):
"""Add the name of the build configuration as metadata to report documents
stored in the BDB XML database."""
try:
from bsddb3 import db as bdb
import dbxml
except ImportError:
return
dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
if not os.path.isfile(dbfile):
return
dbenv = bdb.DBEnv()
dbenv.open(os.path.dirname(dbfile),
bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
mgr = dbxml.XmlManager(dbenv, 0)
xtn = mgr.createTransaction()
container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
uc = mgr.createUpdateContext()
container.addIndex(xtn, '', 'config', 'node-metadata-equality-string', uc)
qc = mgr.createQueryContext()
for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc):
doc = value.asDocument()
metaval = dbxml.XmlValue()
if doc.getMetaData('', 'build', metaval):
build_id = int(metaval.asNumber())
cursor = db.cursor()
cursor.execute("SELECT config FROM bitten_build WHERE id=%s", (build_id,))
row = cursor.fetchone()
if row:
doc.setMetaData('', 'config', dbxml.XmlValue(row[0]))
container.updateDocument(xtn, doc, uc)
else:
# an orphaned report, for whatever reason... just remove it
container.deleteDocument(xtn, doc, uc)
xtn.commit()
container.close()
dbenv.close(0)
def xmldb_to_db(env, db):
"""Migrate report data from Berkeley DB XML to SQL database.
Depending on the number of reports stored, this might take rather long.
After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
may be deleted. BDB XML is no longer used by Bitten.
"""
from bitten.util import xmlio
try:
from bsddb3 import db as bdb
import dbxml
except ImportError:
return
dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
if not os.path.isfile(dbfile):
return
dbenv = bdb.DBEnv()
dbenv.open(os.path.dirname(dbfile),
bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
mgr = dbxml.XmlManager(dbenv, 0)
xtn = mgr.createTransaction()
container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
def get_pylint_items(xml):
for problems_elem in xml.children('problems'):
for problem_elem in problems_elem.children('problem'):
item = {'type': 'problem'}
item.update(problem_elem.attr)
yield item
def get_trace_items(xml):
for cov_elem in xml.children('coverage'):
item = {'type': 'coverage', 'name': cov_elem.attr['module'],
'file': cov_elem.attr['file'],
'percentage': cov_elem.attr['percentage']}
lines = 0
line_hits = []
for line_elem in cov_elem.children('line'):
lines += 1
line_hits.append(line_elem.attr['hits'])
item['lines'] = lines
item['line_hits'] = ' '.join(line_hits)
yield item
def get_unittest_items(xml):
for test_elem in xml.children('test'):
item = {'type': 'test'}
item.update(test_elem.attr)
for child_elem in test_elem.children():
item[child_elem.name] = child_elem.gettext()
yield item
qc = mgr.createQueryContext()
for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
doc = value.asDocument()
metaval = dbxml.XmlValue()
build, step = None, None
if doc.getMetaData('', 'build', metaval):
build = metaval.asNumber()
if doc.getMetaData('', 'step', metaval):
step = metaval.asString()
report_types = {'pylint': ('lint', get_pylint_items),
'trace': ('coverage', get_trace_items),
'unittest': ('test', get_unittest_items)}
xml = xmlio.parse(value.asString())
report_type = xml.attr['type']
category, get_items = report_types[report_type]
sys.stderr.write('.')
sys.stderr.flush()
items = list(get_items(xml))
cursor = db.cursor()
cursor.execute("SELECT bitten_report.id FROM bitten_report "
"WHERE build=%s AND step=%s AND category=%s",
(build, step, category))
rows = cursor.fetchall()
if rows:
# Duplicate report, skip
continue
cursor.execute("INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(build, step, category, report_type))
id = db.get_last_id(cursor, 'bitten_report')
for idx, item in enumerate(items):
cursor.executemany("INSERT INTO bitten_report_item "
"(report,item,name,value) VALUES (%s,%s,%s,%s)",
[(id, idx, key, value) for key, value
in item.items()])
sys.stderr.write('\n')
sys.stderr.flush()
xtn.abort()
container.close()
dbenv.close(0)
def __init__(self, xml_replace_dict={}):
self.xml_replace_dict = xml_replace_dict
self.mgr = dbxml.XmlManager()
self.oldcwd = ''
self.query_context = self.mgr.createQueryContext()
self.update_context = self.mgr.createUpdateContext()