def _process_build_initiation(self, req, config, build):
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
build.started = int(time.time())
build.last_activity = build.started
build.update()
for listener in BuildSystem(self.env).listeners:
listener.build_started(build)
repos_name, repos, repos_path = get_repos(self.env, config.path,
req.authname)
xml = xmlio.parse(config.recipe)
xml.attr['path'] = config.path
xml.attr['revision'] = build.rev
xml.attr['config'] = config.name
xml.attr['build'] = str(build.id)
target_platform = TargetPlatform.fetch(self.env, build.platform)
xml.attr['platform'] = target_platform.name
xml.attr['name'] = build.slave
xml.attr['form_token'] = req.form_token # For posting attachments
xml.attr['reponame'] = repos_name != '(default)' and repos_name or ''
xml.attr['repopath'] = repos_path.strip('/')
body = str(xml)
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
# create the first step, mark it as in-progress.
recipe = Recipe(xmlio.parse(config.recipe))
stepname = recipe.__iter__().next().id
step = self._start_new_step(build, stepname)
step.insert()
self._send_response(req,
200,
body,
headers={
'Content-Type':
'application/x-bitten+xml',
'Content-Length':
str(len(body)),
'Content-Disposition':
'attachment; filename=recipe_%s_r%s.xml' %
(config.name, build.rev)
})
def test_validate_successful(self):
xml = xmlio.parse('<build>'
'<step id="foo"><somecmd></somecmd></step>'
'<step id="bar"><othercmd></othercmd></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
recipe.validate()
def test_ParsedElement_encoding(self):
u = u'<root foo="øüé€"/>'
s = '<root foo="\xc3\xb8\xc3\xbc\xc3\xa9\xe2\x82\xac"/>'
self.assertEquals(u, s.decode('utf-8'))
# unicode input
x = xmlio.parse(u)
out_u = str(x)
self.assertEquals(out_u, s)
self.assertEquals(out_u.decode('utf-8'), u)
# utf-8 input
x = xmlio.parse(s)
out_s = str(x)
self.assertEquals(out_s, s)
self.assertEquals(out_s.decode('utf-8'), u)
# identical results
self.assertEquals(out_u, out_s)
def _execute_build(self, build_url, fileobj):
build_id = build_url and int(build_url.split('/')[-1]) or 0
xml = xmlio.parse(fileobj)
basedir = ''
try:
recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir),
self.config)
basedir = recipe.ctxt.basedir
log.debug('Running build in directory %s' % basedir)
if not os.path.exists(basedir):
os.mkdir(basedir)
for step in recipe:
log.info('Executing build step %r', step.id)
if not self._execute_step(build_url, recipe, step):
log.warning('Stopping build due to failure')
break
else:
log.info('Build completed')
if self.dry_run:
self._cancel_build(build_url)
finally:
if not self.keep_files and os.path.isdir(basedir):
log.debug('Removing build directory %s' % basedir)
_rmtree(basedir)
if self.single_build:
log.info('Exiting after single build completed.')
raise ExitSlave(EX_OK)
def test_validate_successful(self):
xml = xmlio.parse('<build>'
'<step id="foo"><somecmd></somecmd></step>'
'<step id="bar"><othercmd></othercmd></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
recipe.validate()
def _execute_build(self, build_url, fileobj):
build_id = build_url and int(build_url.split('/')[-1]) or 0
xml = xmlio.parse(fileobj)
try:
recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir),
self.config)
basedir = recipe.ctxt.basedir
log.debug('Running build in directory %s' % basedir)
if not os.path.exists(basedir):
os.mkdir(basedir)
for step in recipe:
log.info('Executing build step %r', step.id)
if not self._execute_step(build_url, recipe, step):
log.warning('Stopping build due to failure')
break
else:
log.info('Build completed')
if self.dry_run:
self._cancel_build(build_url)
finally:
if not self.keep_files:
log.debug('Removing build directory %s' % basedir)
_rmtree(basedir)
if self.single_build:
log.info('Exiting after single build completed.')
raise ExitSlave(EX_OK)
def test_ParsedElement_encoding(self):
u = u'<root foo="øüé€"/>'
s = '<root foo="\xc3\xb8\xc3\xbc\xc3\xa9\xe2\x82\xac"/>'
self.assertEquals(u, s.decode('utf-8'))
# unicode input
x = xmlio.parse(u)
out_u = str(x)
self.assertEquals(out_u, s)
self.assertEquals(out_u.decode('utf-8'), u)
# utf-8 input
x = xmlio.parse(s)
out_s = str(x)
self.assertEquals(out_s, s)
self.assertEquals(out_s.decode('utf-8'), u)
# identical results
self.assertEquals(out_u, out_s)
def coverage(ctxt, file_=None):
assert file_, 'Missing required attribute "file"'
try:
fileobj = file(ctxt.resolve(file_), 'r')
try:
results = xmlio.Fragment()
for child in xmlio.parse(fileobj).children():
cover = xmlio.Element('coverage')
for name, value in child.attr.items():
if name == 'file':
value = os.path.realpath(value)
if value.startswith(ctxt.basedir):
value = value[len(ctxt.basedir) + 1:]
value = value.replace(os.sep, '/')
else:
continue
cover.attr[name] = value
for grandchild in child.children():
cover.append(xmlio.Element(grandchild.name)[
grandchild.gettext()
])
results.append(cover)
ctxt.report('nosecoverage', results)
finally:
fileobj.close()
except IOError, e:
log.warning('Error opening coverage results file (%s)', e)
def _execute_build(self, build_url, fileobj):
build_id = build_url and int(build_url.split('/')[-1]) or 0
xml = xmlio.parse(fileobj)
basedir = ''
try:
if not self.local:
keepalive_thread = KeepAliveThread(self.opener, build_url,
self.single_build,
self.keepalive_interval)
keepalive_thread.start()
recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir),
self.config)
basedir = recipe.ctxt.basedir
log.debug('Running build in directory %s' % basedir)
if not os.path.exists(basedir):
os.mkdir(basedir)
for step in recipe:
try:
log.info('Executing build step %r, onerror = %s', step.id,
step.onerror)
if not self._execute_step(build_url, recipe, step):
log.warning('Stopping build due to failure')
break
except Exception, e:
log.error(
'Exception raised processing step %s. Reraising %s',
step.id, e)
raise
else:
def _process_build_step(self, req, config, build):
try:
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build step result: %s', e,
exc_info=True)
raise HTTPBadRequest('XML parser error')
def test_validate_step_with_duplicate_id(self):
xml = xmlio.parse('<build>'
'<step id="test"><somecmd></somecmd></step>'
'<step id="test"><othercmd></othercmd></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def _process_build_step(self, req, config, build):
try:
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build step result: %s', e,
exc_info=True)
self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
def _process_build_initiation(self, req, config, build):
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
build.started = int(time.time())
build.update()
for listener in BuildSystem(self.env).listeners:
listener.build_started(build)
xml = xmlio.parse(config.recipe)
xml.attr['path'] = config.path
xml.attr['revision'] = build.rev
xml.attr['config'] = config.name
xml.attr['build'] = str(build.id)
target_platform = TargetPlatform.fetch(self.env, build.platform)
xml.attr['platform'] = target_platform.name
body = str(xml)
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
req.send_response(200)
req.send_header('Content-Type', 'application/x-bitten+xml')
req.send_header('Content-Length', str(len(body)))
req.send_header('Content-Disposition',
'attachment; filename=recipe_%s_r%s.xml' %
(config.name, build.rev))
req.write(body)
raise RequestDone
def report_file(self, category=None, file_=None):
"""Read report data from a file and record it.
:param category: the name of the category of the report
:param file\_: the path to the file containing the report data, relative
to the base directory
"""
filename = self.resolve(file_)
try:
fileobj = file(filename, 'r')
try:
xml_elem = xmlio.Fragment()
for child in xmlio.parse(fileobj).children():
child_elem = xmlio.Element(child.name, **dict([
(name, value) for name, value in child.attr.items()
if value is not None
]))
xml_elem.append(child_elem[
[xmlio.Element(grandchild.name)[grandchild.gettext()]
for grandchild in child.children()]
])
self.output.append((Recipe.REPORT, category, None, xml_elem))
finally:
fileobj.close()
except xmlio.ParseError, e:
self.error('Failed to parse %s report at %s: %s'
% (category, filename, e))
def _process_build_initiation(self, req, config, build):
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
build.started = int(time.time())
build.update()
for listener in BuildSystem(self.env).listeners:
listener.build_started(build)
xml = xmlio.parse(config.recipe)
xml.attr['branch'] = config.branch
xml.attr['revision'] = build.rev
xml.attr['config'] = config.name
xml.attr['build'] = str(build.id)
target_platform = TargetPlatform.fetch(self.env, build.platform)
xml.attr['platform'] = target_platform.name
xml.attr['name'] = build.slave
body = str(xml)
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
self._send_response(req,
200,
body,
headers={
'Content-Type':
'application/x-bitten+xml',
'Content-Length':
str(len(body)),
'Content-Disposition':
'attachment; filename=recipe_%s_r%s.xml' %
(config.name, build.rev)
})
def coverage(ctxt, file_=None):
"""Extract data from a Phing code coverage report."""
assert file_, 'Missing required attribute "file"'
try:
summary_file = file(ctxt.resolve(file_), 'r')
try:
coverage = xmlio.Fragment()
for package in xmlio.parse(summary_file).children('package'):
for cls in package.children('class'):
statements = float(cls.attr['statementcount'])
covered = float(cls.attr['statementscovered'])
if statements:
percentage = covered / statements * 100
else:
percentage = 100
class_coverage = xmlio.Element('coverage',
name=cls.attr['name'],
lines=int(statements),
percentage=percentage
)
source = list(cls.children())[0]
if 'sourcefile' in source.attr:
sourcefile = os.path.realpath(source.attr['sourcefile'])
if sourcefile.startswith(ctxt.basedir):
sourcefile = sourcefile[len(ctxt.basedir) + 1:]
sourcefile = sourcefile.replace(os.sep, '/')
class_coverage.attr['file'] = sourcefile
coverage.append(class_coverage)
finally:
summary_file.close()
ctxt.report('coverage', coverage)
except IOError, e:
ctxt.log('Error opening coverage summary file (%s)' % e)
def _process_build_initiation(self, req, config, build):
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
build.started = int(time.time())
build.update()
for listener in BuildSystem(self.env).listeners:
listener.build_started(build)
xml = xmlio.parse(config.recipe)
xml.attr['branch'] = config.branch
xml.attr['revision'] = build.rev
xml.attr['config'] = config.name
xml.attr['build'] = str(build.id)
target_platform = TargetPlatform.fetch(self.env, build.platform)
xml.attr['platform'] = target_platform.name
xml.attr['name'] = build.slave
body = str(xml)
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
self._send_response(req, 200, body, headers={
'Content-Type': 'application/x-bitten+xml',
'Content-Length': str(len(body)),
'Content-Disposition':
'attachment; filename=recipe_%s_r%s.xml' %
(config.name, build.rev)})
def report_file(self, category=None, file_=None):
"""Read report data from a file and record it.
:param category: the name of the category of the report
:param file\_: the path to the file containing the report data, relative
to the base directory
"""
filename = self.resolve(file_)
try:
fileobj = file(filename, 'r')
try:
xml_elem = xmlio.Fragment()
for child in xmlio.parse(fileobj).children():
child_elem = xmlio.Element(
child.name,
**dict([(name, value)
for name, value in child.attr.items()
if value is not None]))
xml_elem.append(child_elem[[
xmlio.Element(grandchild.name)[grandchild.gettext()]
for grandchild in child.children()
]])
self.output.append((Recipe.REPORT, category, None, xml_elem))
finally:
fileobj.close()
except xmlio.ParseError, e:
self.error('Failed to parse %s report at %s: %s' %
(category, filename, e))
def test_validate_step_with_duplicate_id(self):
xml = xmlio.parse('<build>'
'<step id="test"><somecmd></somecmd></step>'
'<step id="test"><othercmd></othercmd></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def _process_build_step(self, req, config, build):
try:
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build step result: %s',
e,
exc_info=True)
self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
def junit(ctxt, file_=None, srcdir=None):
"""Extract test results from a JUnit XML report.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path to the JUnit XML test results; may contain globbing
wildcards for matching multiple results files
:param srcdir: name of the directory containing the test sources, used to
link test results to the corresponding source files
"""
assert file_, 'Missing required attribute "file"'
try:
total, failed = 0, 0
results = xmlio.Fragment()
for path in glob(ctxt.resolve(file_)):
fileobj = file(path, 'r')
try:
for testcase in xmlio.parse(fileobj).children('testcase'):
test = xmlio.Element('test')
test.attr['fixture'] = testcase.attr['classname']
test.attr['name'] = testcase.attr['name']
if 'time' in testcase.attr:
test.attr['duration'] = testcase.attr['time']
if srcdir is not None:
cls = testcase.attr['classname'].split('.')
test.attr['file'] = posixpath.join(srcdir, *cls) + \
'.java'
result = list(testcase.children())
if result:
test.attr['status'] = result[0].name
# Sometimes the traceback isn't prefixed with the
# exception type and message, so add it in if needed
tracebackprefix = "%s: %s" % (
result[0].attr['type'], result[0].attr['message'])
if result[0].gettext().startswith(tracebackprefix):
test.append(
xmlio.Element('traceback')[
result[0].gettext()])
else:
test.append(
xmlio.Element('traceback')["\n".join(
(tracebackprefix, result[0].gettext()))])
failed += 1
else:
test.attr['status'] = 'success'
results.append(test)
total += 1
finally:
fileobj.close()
if failed:
ctxt.error('%d of %d test%s failed' %
(failed, total, total != 1 and 's' or ''))
ctxt.report('test', results)
except IOError, e:
log.warning('Error opening JUnit results file (%s)', e)
def test_parse(self):
"""Tests that simple test data is parsed correctly"""
s = """<build xmlns:c="http://bitten.edgewall.org/tools/c">
<step id="build" description="Configure and build">
<c:configure />
</step>\
</build>"""
x = xmlio.parse(s)
assert x.name == "build"
def test_parse(self):
"""Tests that simple test data is parsed correctly"""
s = """<build xmlns:c="http://bitten.edgewall.org/tools/c">
<step id="build" description="Configure and build">
<c:configure />
</step>\
</build>"""
x = xmlio.parse(s)
assert x.name == "build"
def test_empty_step(self):
xml = xmlio.parse('<build>'
' <step id="foo" description="Bar"></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
steps = list(recipe)
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].id)
self.assertEqual('Bar', steps[0].description)
self.assertEqual('fail', steps[0].onerror)
def test_empty_step(self):
xml = xmlio.parse('<build>'
' <step id="foo" description="Bar"></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
steps = list(recipe)
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].id)
self.assertEqual('Bar', steps[0].description)
self.assertEqual('fail', steps[0].onerror)
def test_onerror_override(self):
xml = xmlio.parse('<build onerror="ignore">'
' <step id="foo" description="Bar" onerror="continue"></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
steps = list(recipe)
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].id)
self.assertEqual('Bar', steps[0].description)
self.assertEqual('continue', steps[0].onerror)
def junit(ctxt, file_=None, srcdir=None):
"""Extract test results from a JUnit XML report.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path to the JUnit XML test results; may contain globbing
wildcards for matching multiple results files
:param srcdir: name of the directory containing the test sources, used to
link test results to the corresponding source files
"""
assert file_, 'Missing required attribute "file"'
try:
total, failed = 0, 0
results = xmlio.Fragment()
for path in glob(ctxt.resolve(file_)):
fileobj = file(path, 'r')
try:
for testcase in xmlio.parse(fileobj).children('testcase'):
test = xmlio.Element('test')
test.attr['fixture'] = testcase.attr['classname']
test.attr['name'] = testcase.attr['name']
if 'time' in testcase.attr:
test.attr['duration'] = testcase.attr['time']
if srcdir is not None:
cls = testcase.attr['classname'].split('.')
test.attr['file'] = posixpath.join(srcdir, *cls) + \
'.java'
result = list(testcase.children())
if result:
test.attr['status'] = result[0].name
# Sometimes the traceback isn't prefixed with the
# exception type and message, so add it in if needed
tracebackprefix = "%s: %s" % (result[0].attr['type'],
result[0].attr['message'])
if result[0].gettext().startswith(tracebackprefix):
test.append(xmlio.Element('traceback')[
result[0].gettext()])
else:
test.append(xmlio.Element('traceback')[
"\n".join((tracebackprefix, result[0].gettext()))])
failed += 1
else:
test.attr['status'] = 'success'
results.append(test)
total += 1
finally:
fileobj.close()
if failed:
ctxt.error('%d of %d test%s failed' % (failed, total,
total != 1 and 's' or ''))
ctxt.report('test', results)
except IOError, e:
log.warning('Error opening JUnit results file (%s)', e)
def _process_build_creation(self, req):
queue = BuildQueue(self.env, build_all=self.build_all,
stabilize_wait=self.stabilize_wait,
timeout=self.slave_timeout)
queue.populate()
try:
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build initialization request: %s', e,
exc_info=True)
raise HTTPBadRequest('XML parser error')
def _process_build_initiation(self, req, config, build):
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
build.started = int(time.time())
build.last_activity = build.started
build.update()
for listener in BuildSystem(self.env).listeners:
listener.build_started(build)
xml = xmlio.parse(config.recipe)
xml.attr['path'] = config.path
xml.attr['revision'] = build.rev
xml.attr['config'] = config.name
xml.attr['build'] = str(build.id)
target_platform = TargetPlatform.fetch(self.env, build.platform)
xml.attr['platform'] = target_platform.name
xml.attr['name'] = build.slave
xml.attr['form_token'] = req.form_token # For posting attachments
body = str(xml)
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
# create the first step, mark it as in-progress.
recipe = Recipe(xmlio.parse(config.recipe))
stepname = recipe.__iter__().next().id
step = self._start_new_step(build, stepname)
step.insert()
self._send_response(req, 200, body, headers={
'Content-Type': 'application/x-bitten+xml',
'Content-Length': str(len(body)),
'Content-Disposition':
'attachment; filename=recipe_%s_r%s.xml' %
(config.name, build.rev)})
def phpunit(ctxt, file_=None):
"""Extract test results from a PHPUnit XML report."""
assert file_, 'Missing required attribute "file"'
def _process_testsuite(testsuite, results, parent_file=''):
for testcase in testsuite.children():
if testcase.name == 'testsuite':
_process_testsuite(testcase, results,
parent_file=testcase.attr.get('file', parent_file))
continue
test = xmlio.Element('test')
test.attr['fixture'] = testsuite.attr['name']
test.attr['name'] = testcase.attr['name']
test.attr['duration'] = testcase.attr['time']
result = list(testcase.children())
if result:
test.append(xmlio.Element('traceback')[
result[0].gettext()
])
test.attr['status'] = result[0].name
else:
test.attr['status'] = 'success'
if 'file' in testsuite.attr or parent_file:
testfile = os.path.realpath(
testsuite.attr.get('file', parent_file))
if testfile.startswith(ctxt.basedir):
testfile = testfile[len(ctxt.basedir) + 1:]
testfile = testfile.replace(os.sep, '/')
test.attr['file'] = testfile
results.append(test)
try:
total, failed = 0, 0
results = xmlio.Fragment()
fileobj = file(ctxt.resolve(file_), 'r')
try:
for testsuite in xmlio.parse(fileobj).children('testsuite'):
total += int(testsuite.attr['tests'])
failed += int(testsuite.attr['failures']) + \
int(testsuite.attr['errors'])
_process_testsuite(testsuite, results)
finally:
fileobj.close()
if failed:
ctxt.error('%d of %d test%s failed' % (failed, total,
total != 1 and 's' or ''))
ctxt.report('test', results)
except IOError, e:
ctxt.log('Error opening PHPUnit results file (%s)' % e)
def _process_build_creation(self, req, slave_token):
queue = BuildQueue(self.env,
build_all=self.build_all,
stabilize_wait=self.stabilize_wait,
timeout=self.slave_timeout)
queue.populate()
try:
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build initialization request: %s',
e,
exc_info=True)
self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
def phpunit(ctxt, file_=None):
"""Extract test results from a PHPUnit XML report."""
assert file_, 'Missing required attribute "file"'
def _process_testsuite(testsuite, results, parent_file=''):
for testcase in testsuite.children():
if testcase.name == 'testsuite':
_process_testsuite(testcase,
results,
parent_file=testcase.attr.get(
'file', parent_file))
continue
test = xmlio.Element('test')
test.attr['fixture'] = testsuite.attr['name']
test.attr['name'] = testcase.attr['name']
test.attr['duration'] = testcase.attr['time']
result = list(testcase.children())
if result:
test.append(xmlio.Element('traceback')[result[0].gettext()])
test.attr['status'] = result[0].name
else:
test.attr['status'] = 'success'
if 'file' in testsuite.attr or parent_file:
testfile = os.path.realpath(
testsuite.attr.get('file', parent_file))
if testfile.startswith(ctxt.basedir):
testfile = testfile[len(ctxt.basedir) + 1:]
testfile = testfile.replace(os.sep, '/')
test.attr['file'] = testfile
results.append(test)
try:
total, failed = 0, 0
results = xmlio.Fragment()
fileobj = file(ctxt.resolve(file_), 'r')
try:
for testsuite in xmlio.parse(fileobj).children('testsuite'):
total += int(testsuite.attr['tests'])
failed += int(testsuite.attr['failures']) + \
int(testsuite.attr['errors'])
_process_testsuite(testsuite, results)
finally:
fileobj.close()
if failed:
ctxt.error('%d of %d test%s failed' %
(failed, total, total != 1 and 's' or ''))
ctxt.report('test', results)
except IOError, e:
ctxt.log('Error opening PHPUnit results file (%s)' % e)
def cobertura(ctxt, file_=None):
"""Extract test coverage information from a Cobertura XML report.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path to the Cobertura XML output
"""
assert file_, 'Missing required attribute "file"'
coverage = xmlio.Fragment()
doc = xmlio.parse(open(ctxt.resolve(file_)))
srcdir = [
s.gettext().strip() for ss in doc.children('sources')
for s in ss.children('source')
][0]
classes = [
cls for pkgs in doc.children('packages')
for pkg in pkgs.children('package') for clss in pkg.children('classes')
for cls in clss.children('class')
]
counters = {}
class_names = {}
for cls in classes:
filename = cls.attr['filename'].replace(os.sep, '/')
name = cls.attr['name']
if not '$' in name: # ignore internal classes
class_names[filename] = name
counter = counters.get(filename)
if counter is None:
counter = counters[filename] = _LineCounter()
lines = [
l for ls in cls.children('lines') for l in ls.children('line')
]
for line in lines:
counter[line.attr['number']] = line.attr['hits']
for filename, name in class_names.iteritems():
counter = counters[filename]
module = xmlio.Element('coverage',
name=name,
file=posixpath.join(srcdir, filename),
lines=counter.num_lines,
percentage=counter.percentage)
module.append(xmlio.Element('line_hits')[counter.line_hits])
coverage.append(module)
ctxt.report('coverage', coverage)
def cobertura(ctxt, file_=None):
"""Extract test coverage information from a Cobertura XML report.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path to the Cobertura XML output
"""
assert file_, 'Missing required attribute "file"'
coverage = xmlio.Fragment()
doc = xmlio.parse(open(ctxt.resolve(file_)))
srcdir = [s.gettext().strip() for ss in doc.children('sources')
for s in ss.children('source')][0]
classes = [cls for pkgs in doc.children('packages')
for pkg in pkgs.children('package')
for clss in pkg.children('classes')
for cls in clss.children('class')]
counters = {}
class_names = {}
for cls in classes:
filename = cls.attr['filename'].replace(os.sep, '/')
name = cls.attr['name']
if not '$' in name: # ignore internal classes
class_names[filename] = name
counter = counters.get(filename)
if counter is None:
counter = counters[filename] = _LineCounter()
lines = [l for ls in cls.children('lines')
for l in ls.children('line')]
for line in lines:
counter[line.attr['number']] = line.attr['hits']
for filename, name in class_names.iteritems():
counter = counters[filename]
module = xmlio.Element('coverage', name=name,
file=posixpath.join(srcdir, filename),
lines=counter.num_lines,
percentage=counter.percentage)
module.append(xmlio.Element('line_hits')[counter.line_hits])
coverage.append(module)
ctxt.report('coverage', coverage)
def unittest(ctxt, file_=None):
"""Extract data from a unittest results file in XML format.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: name of the file containing the test results
"""
assert file_, 'Missing required attribute "file"'
try:
fileobj = file(ctxt.resolve(file_), 'r')
try:
total, failed = 0, 0
results = xmlio.Fragment()
for child in xmlio.parse(fileobj).children():
test = xmlio.Element('test')
for name, value in child.attr.items():
if name == 'file':
value = os.path.realpath(value)
if value.startswith(ctxt.basedir):
value = value[len(ctxt.basedir) + 1:]
value = value.replace(os.sep, '/')
else:
continue
test.attr[name] = value
if name == 'status' and value in ('error', 'failure'):
failed += 1
for grandchild in child.children():
test.append(xmlio.Element(grandchild.name)[
grandchild.gettext()
])
results.append(test)
total += 1
if failed:
ctxt.error('%d of %d test%s failed' % (failed, total,
total != 1 and 's' or ''))
ctxt.report('test', results)
finally:
fileobj.close()
except IOError, e:
log.warning('Error opening unittest results file (%s)', e)
def _execute_build(self, build_url, fileobj):
build_id = build_url and int(build_url.split("/")[-1]) or 0
xml = xmlio.parse(fileobj)
basedir = ""
try:
if not self.local:
keepalive_thread = KeepAliveThread(self.opener, build_url, self.single_build, self.keepalive_interval)
keepalive_thread.start()
recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir), self.config)
basedir = recipe.ctxt.basedir
log.debug("Running build in directory %s" % basedir)
if not os.path.exists(basedir):
os.mkdir(basedir)
for step in recipe:
try:
log.info("Executing build step %r, onerror = %s", step.id, step.onerror)
if not self._execute_step(build_url, recipe, step):
log.warning("Stopping build due to failure")
break
except Exception, e:
log.error("Exception raised processing step %s. Reraising %s", step.id, e)
raise
else:
def ant(ctxt, file_=None, target=None, keep_going=False, args=None):
"""Run an Ant build.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: name of the Ant build file
:param target: name of the target that should be executed (optional)
:param keep_going: whether Ant should keep going when errors are encountered
:param args: additional arguments to pass to Ant
"""
executable = 'ant'
ant_home = ctxt.config.get_dirpath('ant.home')
if ant_home:
executable = os.path.join(ant_home, 'bin', 'ant')
java_home = ctxt.config.get_dirpath('java.home')
if java_home:
os.environ['JAVA_HOME'] = java_home
logfile = tempfile.NamedTemporaryFile(prefix='ant_log', suffix='.xml')
logfile.close()
if args:
args = shlex.split(args)
else:
args = []
args += [
'-noinput', '-listener', 'org.apache.tools.ant.XmlLogger',
'-Dant.XmlLogger.stylesheet.uri', '""', '-DXmlLogger.file',
logfile.name
]
if file_:
args += ['-buildfile', ctxt.resolve(file_)]
if keep_going:
args.append('-keep-going')
if target:
args.append(target)
shell = False
if os.name == 'nt':
# Need to execute ant.bat through a shell on Windows
shell = True
cmdline = CommandLine(executable, args, cwd=ctxt.basedir, shell=shell)
for out, err in cmdline.execute():
if out is not None:
log.info(out)
if err is not None:
log.error(err)
error_logged = False
log_elem = xmlio.Fragment()
try:
xml_log = xmlio.parse(file(logfile.name, 'r'))
def collect_log_messages(node):
for child in node.children():
if child.name == 'message':
if child.attr['priority'] == 'debug':
continue
log_elem.append(
xmlio.Element('message', level=child.attr['priority'])
[child.gettext().replace(ctxt.basedir + os.sep,
'').replace(ctxt.basedir,
'')])
else:
collect_log_messages(child)
collect_log_messages(xml_log)
if 'error' in xml_log.attr:
ctxt.error(xml_log.attr['error'])
error_logged = True
except xmlio.ParseError, e:
log.warning('Error parsing Ant XML log file (%s)', e)
def cppunit(ctxt, file_=None, srcdir=None):
"""Collect CppUnit XML data.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path of the file containing the CppUnit results; may contain
globbing wildcards to match multiple files
:param srcdir: name of the directory containing the source files, used to
link the test results to the corresponding files
"""
assert file_, 'Missing required attribute "file"'
try:
fileobj = file(ctxt.resolve(file_), 'r')
try:
total, failed = 0, 0
results = xmlio.Fragment()
for group in xmlio.parse(fileobj):
if group.name not in ('FailedTests', 'SuccessfulTests'):
continue
for child in group.children():
test = xmlio.Element('test')
name = child.children('Name').next().gettext()
if '::' in name:
parts = name.split('::')
test.attr['fixture'] = '::'.join(parts[:-1])
name = parts[-1]
test.attr['name'] = name
for location in child.children('Location'):
for file_elem in location.children('File'):
filepath = file_elem.gettext()
if srcdir is not None:
filepath = posixpath.join(srcdir, filepath)
test.attr['file'] = filepath
break
for line_elem in location.children('Line'):
test.attr['line'] = line_elem.gettext()
break
break
if child.name == 'FailedTest':
for message in child.children('Message'):
test.append(xmlio.Element('traceback')[
message.gettext()
])
test.attr['status'] = 'failure'
failed += 1
else:
test.attr['status'] = 'success'
results.append(test)
total += 1
if failed:
ctxt.error('%d of %d test%s failed' % (failed, total,
total != 1 and 's' or ''))
ctxt.report('test', results)
finally:
fileobj.close()
except IOError, e:
log.warning('Error opening CppUnit results file (%s)', e)
def junit(ctxt, file_=None, srcdir=None):
"""Extract test results from a JUnit XML report.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path to the JUnit XML test results; may contain globbing
wildcards for matching multiple results files
:param srcdir: name of the directory containing the test sources, used to
link test results to the corresponding source files
"""
assert file_, 'Missing required attribute "file"'
try:
total, failed = 0, 0
results = xmlio.Fragment()
for path in glob(ctxt.resolve(file_)):
fileobj = file(path, 'r')
try:
output = xmlio.parse(fileobj)
finally:
fileobj.close()
if output.name == 'testsuites':
# top level wrapper for testsuites
testcases = []
for t_suite in output.children('testsuite'):
testcases.extend(
[t_case for t_case in t_suite.children('testcase')])
else:
testcases = [t_case for t_case in output.children('testcase')]
for testcase in testcases:
test = xmlio.Element('test')
test.attr['fixture'] = testcase.attr['classname']
test.attr['name'] = testcase.attr['name']
if 'time' in testcase.attr:
test.attr['duration'] = testcase.attr['time']
if srcdir is not None:
cls = testcase.attr['classname'].split('.')
test.attr['file'] = posixpath.join(srcdir, *cls) + \
'.java'
result = list(testcase.children())
if result:
junit_status = result[0].name
test.append(
xmlio.Element('traceback')[_fix_traceback(result)])
if junit_status == 'skipped':
test.attr['status'] = 'ignore'
elif junit_status == 'error':
test.attr['status'] = 'error'
failed += 1
else:
test.attr['status'] = 'failure'
failed += 1
else:
test.attr['status'] = 'success'
results.append(test)
total += 1
if failed:
ctxt.error('%d of %d test%s failed' %
(failed, total, total != 1 and 's' or ''))
ctxt.report('test', results)
except IOError, e:
log.warning('Error opening JUnit results file (%s)', e)
try:
node = repos.get_node(repos_path, max_rev)
assert node.isdir, '%s is not a directory' % node.path
except (AssertionError, TracError), e:
warnings.append('Invalid Repository Path "%s".' % path)
if min_rev:
try:
repos.get_node(repos_path, min_rev)
except TracError, e:
warnings.append('Invalid Oldest Revision: %s.' % unicode(e))
recipe_xml = req.args.get('recipe', '')
if recipe_xml:
try:
Recipe(xmlio.parse(recipe_xml)).validate()
except xmlio.ParseError, e:
warnings.append('Failure parsing recipe: %s.' % unicode(e))
except InvalidRecipeError, e:
warnings.append('Invalid Recipe: %s.' % unicode(e))
config.name = name
config.path = __multirepos__ and path or repos.normalize_path(path)
config.recipe = recipe_xml
config.min_rev = min_rev
config.max_rev = max_rev
config.label = req.args.get('label', config.name)
config.description = req.args.get('description', '')
if warnings: # abort
return warnings
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build step result: %s',
e,
exc_info=True)
self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
stepname = elem.attr['step']
# we should have created this step previously; if it hasn't,
# the master and slave are processing steps out of order.
step = BuildStep.fetch(self.env, build=build.id, name=stepname)
if not step:
self._send_error(req, HTTP_CONFLICT,
'Build step has not been created.')
recipe = Recipe(xmlio.parse(config.recipe))
index = None
current_step = None
for num, recipe_step in enumerate(recipe):
if recipe_step.id == stepname:
index = num
current_step = recipe_step
if index is None:
self._send_error(req, HTTP_FORBIDDEN,
'No such build step' % stepname)
last_step = index == num
self.log.debug(
'Slave %s (build %d) completed step %d (%s) with '
'status %s', build.slave, build.id, index, stepname,
elem.attr['status'])
def xmldb_to_db(env, db):
"""Migrate report data from Berkeley DB XML to SQL database.
Depending on the number of reports stored, this might take rather long.
After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
may be deleted. BDB XML is no longer used by Bitten.
"""
from bitten.util import xmlio
try:
from bsddb3 import db as bdb
import dbxml
except ImportError:
return
dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
if not os.path.isfile(dbfile):
return
dbenv = bdb.DBEnv()
dbenv.open(os.path.dirname(dbfile),
bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
mgr = dbxml.XmlManager(dbenv, 0)
xtn = mgr.createTransaction()
container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
def get_pylint_items(xml):
for problems_elem in xml.children('problems'):
for problem_elem in problems_elem.children('problem'):
item = {'type': 'problem'}
item.update(problem_elem.attr)
yield item
def get_trace_items(xml):
for cov_elem in xml.children('coverage'):
item = {'type': 'coverage', 'name': cov_elem.attr['module'],
'file': cov_elem.attr['file'],
'percentage': cov_elem.attr['percentage']}
lines = 0
line_hits = []
for line_elem in cov_elem.children('line'):
lines += 1
line_hits.append(line_elem.attr['hits'])
item['lines'] = lines
item['line_hits'] = ' '.join(line_hits)
yield item
def get_unittest_items(xml):
for test_elem in xml.children('test'):
item = {'type': 'test'}
item.update(test_elem.attr)
for child_elem in test_elem.children():
item[child_elem.name] = child_elem.gettext()
yield item
qc = mgr.createQueryContext()
for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
doc = value.asDocument()
metaval = dbxml.XmlValue()
build, step = None, None
if doc.getMetaData('', 'build', metaval):
build = metaval.asNumber()
if doc.getMetaData('', 'step', metaval):
step = metaval.asString()
report_types = {'pylint': ('lint', get_pylint_items),
'trace': ('coverage', get_trace_items),
'unittest': ('test', get_unittest_items)}
xml = xmlio.parse(value.asString())
report_type = xml.attr['type']
category, get_items = report_types[report_type]
sys.stderr.write('.')
sys.stderr.flush()
items = list(get_items(xml))
cursor = db.cursor()
cursor.execute("SELECT bitten_report.id FROM bitten_report "
"WHERE build=%s AND step=%s AND category=%s",
(build, step, category))
rows = cursor.fetchall()
if rows:
# Duplicate report, skip
continue
cursor.execute("INSERT INTO bitten_report "
"(build,step,category,generator) VALUES (%s,%s,%s,%s)",
(build, step, category, report_type))
id = db.get_last_id(cursor, 'bitten_report')
for idx, item in enumerate(items):
cursor.executemany("INSERT INTO bitten_report_item "
"(report,item,name,value) VALUES (%s,%s,%s,%s)",
[(id, idx, key, value) for key, value
in item.items()])
sys.stderr.write('\n')
sys.stderr.flush()
xtn.abort()
container.close()
dbenv.close(0)
def coverage(ctxt, file_=None):
"""Extract data from Phing or PHPUnit code coverage report."""
assert file_, 'Missing required attribute "file"'
def _process_phing_coverage(ctxt, element, coverage):
for cls in element.children('class'):
statements = float(cls.attr['statementcount'])
covered = float(cls.attr['statementscovered'])
if statements:
percentage = covered / statements * 100
else:
percentage = 100
class_coverage = xmlio.Element('coverage',
name=cls.attr['name'],
lines=int(statements),
percentage=percentage)
source = list(cls.children())[0]
if 'sourcefile' in source.attr:
sourcefile = os.path.realpath(source.attr['sourcefile'])
if sourcefile.startswith(ctxt.basedir):
sourcefile = sourcefile[len(ctxt.basedir) + 1:]
sourcefile = sourcefile.replace(os.sep, '/')
class_coverage.attr['file'] = sourcefile
coverage.append(class_coverage)
def _process_phpunit_coverage(ctxt, element, coverage):
for cls in element._node.getElementsByTagName('class'):
sourcefile = cls.parentNode.getAttribute('name')
if not os.path.isabs(sourcefile):
sourcefile = os.path.join(ctxt.basedir, sourcefile)
if sourcefile.startswith(ctxt.basedir):
loc, ncloc = 0, 0.0
for line in cls.parentNode.getElementsByTagName('line'):
if str(line.getAttribute('type')) == 'stmt':
loc += 1
if int(line.getAttribute('count')) == 0:
ncloc += 1
if loc > 0:
percentage = 100 - (ncloc / loc * 100)
else:
percentage = 100
if sourcefile.startswith(ctxt.basedir):
sourcefile = sourcefile[len(ctxt.basedir) + 1:]
class_coverage = xmlio.Element('coverage',
name=cls.getAttribute('name'),
lines=int(loc),
percentage=int(percentage),
file=sourcefile.replace(
os.sep, '/'))
coverage.append(class_coverage)
try:
summary_file = file(ctxt.resolve(file_), 'r')
summary = xmlio.parse(summary_file)
coverage = xmlio.Fragment()
try:
for element in summary.children():
if element.name == 'package':
_process_phing_coverage(ctxt, element, coverage)
elif element.name == 'project':
_process_phpunit_coverage(ctxt, element, coverage)
finally:
summary_file.close()
ctxt.report('coverage', coverage)
except IOError, e:
ctxt.log('Error opening coverage summary file (%s)' % e)
def test_validate_step_with_command_children(self):
xml = xmlio.parse('<build><step id="test">'
'<somecmd><child1/><child2/></somecmd>'
'</step></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_validate_step_without_commands(self):
xml = xmlio.parse('<build><step id="test"/></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_validate_step_with_empty_id(self):
xml = xmlio.parse('<build><step id=""><cmd/></step></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_validate_child_not_step(self):
xml = xmlio.parse('<build><foo/></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_validate_bad_root(self):
xml = xmlio.parse('<foo></foo>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def cppunit(ctxt, file_=None, srcdir=None):
"""Collect CppUnit XML data.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path of the file containing the CppUnit results; may contain
globbing wildcards to match multiple files
:param srcdir: name of the directory containing the source files, used to
link the test results to the corresponding files
"""
assert file_, 'Missing required attribute "file"'
try:
fileobj = file(ctxt.resolve(file_), "r")
try:
total, failed = 0, 0
results = xmlio.Fragment()
for group in xmlio.parse(fileobj):
if group.name not in ("FailedTests", "SuccessfulTests"):
continue
for child in group.children():
test = xmlio.Element("test")
name = child.children("Name").next().gettext()
if "::" in name:
parts = name.split("::")
test.attr["fixture"] = "::".join(parts[:-1])
name = parts[-1]
test.attr["name"] = name
for location in child.children("Location"):
for file_elem in location.children("File"):
filepath = file_elem.gettext()
if srcdir is not None:
filepath = posixpath.join(srcdir, filepath)
test.attr["file"] = filepath
break
for line_elem in location.children("Line"):
test.attr["line"] = line_elem.gettext()
break
break
if child.name == "FailedTest":
for message in child.children("Message"):
test.append(xmlio.Element("traceback")[message.gettext()])
test.attr["status"] = "failure"
failed += 1
else:
test.attr["status"] = "success"
results.append(test)
total += 1
if failed:
ctxt.error("%d of %d test%s failed" % (failed, total, total != 1 and "s" or ""))
ctxt.report("test", results)
finally:
fileobj.close()
except IOError, e:
log.warning("Error opening CppUnit results file (%s)", e)
def cunit(ctxt, file_=None, srcdir=None):
"""Collect CUnit XML data.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path of the file containing the CUnit results; may contain
globbing wildcards to match multiple files
:param srcdir: name of the directory containing the source files, used to
link the test results to the corresponding files
"""
assert file_, 'Missing required attribute "file"'
try:
fileobj = file(ctxt.resolve(file_), "r")
try:
total, failed = 0, 0
results = xmlio.Fragment()
log_elem = xmlio.Fragment()
def info(msg):
log.info(msg)
log_elem.append(xmlio.Element("message", level="info")[msg])
def warning(msg):
log.warning(msg)
log_elem.append(xmlio.Element("message", level="warning")[msg])
def error(msg):
log.error(msg)
log_elem.append(xmlio.Element("message", level="error")[msg])
for node in xmlio.parse(fileobj):
if node.name != "CUNIT_RESULT_LISTING":
continue
for suiteRun in node.children("CUNIT_RUN_SUITE"):
for suite in suiteRun.children():
if suite.name not in ("CUNIT_RUN_SUITE_SUCCESS", "CUNIT_RUN_SUITE_FAILURE"):
warning("Unknown node: %s" % suite.name)
continue
suiteName = suite.children("SUITE_NAME").next().gettext()
info("%s [%s]" % ("*" * (57 - len(suiteName)), suiteName))
for record in suite.children("CUNIT_RUN_TEST_RECORD"):
for result in record.children():
if result.name not in ("CUNIT_RUN_TEST_SUCCESS", "CUNIT_RUN_TEST_FAILURE"):
continue
testName = result.children("TEST_NAME").next().gettext()
info("Running %s..." % testName)
test = xmlio.Element("test")
test.attr["fixture"] = suiteName
test.attr["name"] = testName
if result.name == "CUNIT_RUN_TEST_FAILURE":
error(
"%s(%d): %s"
% (
result.children("FILE_NAME").next().gettext(),
int(result.children("LINE_NUMBER").next().gettext()),
result.children("CONDITION").next().gettext(),
)
)
test.attr["status"] = "failure"
failed += 1
else:
test.attr["status"] = "success"
results.append(test)
total += 1
if failed:
ctxt.error("%d of %d test%s failed" % (failed, total, total != 1 and "s" or ""))
ctxt.report("test", results)
ctxt.log(log_elem)
finally:
fileobj.close()
except IOError, e:
log.warning("Error opening CUnit results file (%s)", e)
def _gather(self, method, url, body=None, headers=None):
self.results.append(xmlio.parse(body))
return DummyResponse(201)
max_rev = req.args.get('max_rev') or None
try:
node = repos.get_node(path, max_rev)
assert node.isdir, '%s is not a directory' % node.path
except (AssertionError, TracError), e:
raise TracError(unicode(e), 'Invalid Repository Path')
if req.args.get('min_rev'):
try:
repos.get_node(path, req.args.get('min_rev'))
except TracError, e:
raise TracError(unicode(e), 'Invalid Oldest Revision')
recipe_xml = req.args.get('recipe', '')
if recipe_xml:
try:
Recipe(xmlio.parse(recipe_xml)).validate()
except xmlio.ParseError, e:
raise TracError('Failure parsing recipe: %s' % e,
'Invalid Recipe')
except InvalidRecipeError, e:
raise TracError(unicode(e), 'Invalid Recipe')
config.name = name
config.path = repos.normalize_path(path)
config.recipe = recipe_xml
config.min_rev = req.args.get('min_rev')
config.max_rev = req.args.get('max_rev')
config.label = req.args.get('label', config.name)
config.description = req.args.get('description', '')
if config.exists:
def test_transform(self):
src_file = file(self.ctxt.resolve('src.xml'), 'w')
try:
src_file.write("""<doc>
<title>Document Title</title>
<section>
<title>Section Title</title>
<para>This is a test.</para>
<note>This is a note.</note>
</section>
</doc>
""")
finally:
src_file.close()
style_file = file(self.ctxt.resolve('style.xsl'), 'w')
try:
style_file.write("""<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns="http://www.w3.org/TR/xhtml1/strict">
<xsl:template match="doc">
<html>
<head>
<title><xsl:value-of select="title"/></title>
</head>
<body>
<xsl:apply-templates/>
</body>
</html>
</xsl:template>
<xsl:template match="doc/title">
<h1><xsl:apply-templates/></h1>
</xsl:template>
<xsl:template match="section/title">
<h2><xsl:apply-templates/></h2>
</xsl:template>
<xsl:template match="para">
<p><xsl:apply-templates/></p>
</xsl:template>
<xsl:template match="note">
<p class="note"><b>NOTE: </b><xsl:apply-templates/></p>
</xsl:template>
</xsl:stylesheet>
""")
finally:
style_file.close()
xmltools.transform(self.ctxt, src='src.xml', dest='dest.xml',
stylesheet='style.xsl')
dest_file = file(self.ctxt.resolve('dest.xml'))
try:
dest = xmlio.parse(dest_file)
finally:
dest_file.close()
self.assertEqual('html', dest.name)
self.assertEqual('http://www.w3.org/TR/xhtml1/strict', dest.namespace)
children = list(dest.children())
self.assertEqual(2, len(children))
self.assertEqual('head', children[0].name)
head_children = list(children[0].children())
self.assertEqual(1, len(head_children))
self.assertEqual('title', head_children[0].name)
self.assertEqual('Document Title', head_children[0].gettext())
self.assertEqual('body', children[1].name)
body_children = list(children[1].children())
self.assertEqual(4, len(body_children))
self.assertEqual('h1', body_children[0].name)
self.assertEqual('Document Title', body_children[0].gettext())
self.assertEqual('h2', body_children[1].name)
self.assertEqual('Section Title', body_children[1].gettext())
self.assertEqual('p', body_children[2].name)
self.assertEqual('This is a test.', body_children[2].gettext())
self.assertEqual('p', body_children[3].name)
self.assertEqual('note', body_children[3].attr['class'])
self.assertEqual('This is a note.', body_children[3].gettext())
class BuildConfigurationsAdminPageProvider(Component):
"""Web administration panel for configuring the build master."""
implements(IAdminPanelProvider)
# IAdminPanelProvider methods
def get_admin_panels(self, req):
if req.perm.has_permission('BUILD_MODIFY'):
yield ('bitten', 'Builds', 'configs', 'Configurations')
def render_admin_panel(self, req, cat, page, path_info):
data = {}
# Analyze url
try:
config_name, platform_id = path_info.split('/', 1)
except:
config_name = path_info
platform_id = None
if config_name: # Existing build config
warnings = []
if platform_id or (
# Editing or creating one of the config's target platforms
req.method == 'POST' and 'new' in req.args):
if platform_id: # Editing target platform
platform_id = int(platform_id)
platform = TargetPlatform.fetch(self.env, platform_id)
if req.method == 'POST':
if 'cancel' in req.args or \
self._update_platform(req, platform):
req.redirect(
req.abs_href.admin(cat, page, config_name))
else: # creating target platform
platform = self._create_platform(req, config_name)
req.redirect(
req.abs_href.admin(cat, page, config_name,
platform.id))
# Set up template variables
data['platform'] = {
'id':
platform.id,
'name':
platform.name,
'exists':
platform.exists,
'rules': [{
'property': propname,
'pattern': pattern
} for propname, pattern in platform.rules] or [('', '')]
}
else: # Editing existing build config itself
config = BuildConfig.fetch(self.env, config_name)
platforms = list(
TargetPlatform.select(self.env, config=config.name))
if req.method == 'POST':
if 'remove' in req.args: # Remove selected platforms
self._remove_platforms(req)
add_notice(req, "Target Platform(s) Removed.")
req.redirect(req.abs_href.admin(
cat, page, config.name))
elif 'save' in req.args: # Save this build config
warnings = self._update_config(req, config)
if not warnings:
add_notice(req, "Configuration Saved.")
req.redirect(req.abs_href.admin(
cat, page, config.name))
for warning in warnings:
add_warning(req, warning)
# FIXME: Deprecation notice for old namespace.
# Remove notice code when migration to new namespace is complete
if 'http://bitten.cmlenz.net/tools/' in config.recipe:
add_notice(
req, "Recipe uses a deprecated namespace. "
"Replace 'http://bitten.cmlenz.net/tools/' with "
"'http://bitten.edgewall.org/tools/'.")
# Add a notice if configuration is not active
if not warnings and not config.active and config.recipe:
add_notice(
req, "Configuration is not active. Activate "
"from main 'Configurations' listing to enable it.")
# Prepare template variables
data['config'] = {
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'branch':
config.branch,
'description':
config.description,
'recipe':
config.recipe,
'platforms': [{
'name':
platform.name,
'id':
platform.id,
'href':
req.href.admin('bitten', 'configs', config.name,
platform.id),
'rules': [{
'property': propname,
'pattern': pattern
} for propname, pattern in platform.rules]
} for platform in platforms]
}
else: # At the top level build config list
if req.method == 'POST':
if 'add' in req.args: # Add build config
config = self._create_config(req)
req.redirect(req.abs_href.admin(cat, page, config.name))
elif 'remove' in req.args: # Remove selected build configs
self._remove_configs(req)
elif 'apply' in req.args: # Update active state of configs
self._activate_configs(req)
req.redirect(req.abs_href.admin(cat, page))
# Prepare template variables
configs = []
for config in BuildConfig.select(self.env, include_inactive=True):
configs.append({
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'branch':
config.branch,
'href':
req.href.admin('bitten', 'configs', config.name),
'recipe':
config.recipe and True or False
})
data['configs'] = configs
add_stylesheet(req, 'bitten/admin.css')
add_script(req, 'common/js/suggest.js')
return 'bitten_admin_configs.html', data
# Internal methods
def _activate_configs(self, req):
req.perm.assert_permission('BUILD_MODIFY')
active = req.args.get('active') or []
active = isinstance(active, list) and active or [active]
db = self.env.get_db_cnx()
for config in list(
BuildConfig.select(self.env, db=db, include_inactive=True)):
config.active = config.name in active
config.update(db=db)
db.commit()
def _create_config(self, req):
req.perm.assert_permission('BUILD_CREATE')
config = BuildConfig(self.env)
warnings = self._update_config(req, config)
if warnings:
if len(warnings) == 1:
raise TracError(warnings[0], 'Add Configuration')
else:
raise TracError('Errors: %s' % ' '.join(warnings),
'Add Configuration')
return config
def _remove_configs(self, req):
req.perm.assert_permission('BUILD_DELETE')
sel = req.args.get('sel')
if not sel:
raise TracError('No configuration selected')
sel = isinstance(sel, list) and sel or [sel]
db = self.env.get_db_cnx()
for name in sel:
config = BuildConfig.fetch(self.env, name, db=db)
if not config:
raise TracError('Configuration %r not found' % name)
config.delete(db=db)
db.commit()
def _update_config(self, req, config):
warnings = []
req.perm.assert_permission('BUILD_MODIFY')
name = req.args.get('name')
if not name:
warnings.append('Missing required field "name".')
if name and not re.match(r'^[\w.-]+$', name):
warnings.append('The field "name" may only contain letters, '
'digits, periods, or dashes.')
branch = req.args.get('branch', '')
repos = self.env.get_repository(req.authname)
try:
if not (branch in [b[0] for b in repos.git.get_branches()
]) and branch != 'master':
raise TracError('Invalid Repository Branch "%s".' % branch)
except (AssertionError, TracError), e:
warnings.append('Invalid Repository Branch "%s".' % branch)
recipe_xml = req.args.get('recipe', '')
if recipe_xml:
try:
Recipe(xmlio.parse(recipe_xml)).validate()
except xmlio.ParseError, e:
warnings.append('Failure parsing recipe: %s.' % unicode(e))
except InvalidRecipeError, e:
warnings.append('Invalid Recipe: %s.' % unicode(e))
class BuildMaster(Component):
"""Trac request handler implementation for the build master."""
implements(IRequestHandler)
# Configuration options
adjust_timestamps = BoolOption(
'bitten',
'adjust_timestamps',
False,
doc="""Whether the timestamps of builds should be adjusted to be close
to the timestamps of the corresponding changesets.""")
build_all = BoolOption(
'bitten',
'build_all',
False,
doc="""Whether to request builds of older revisions even if a younger
revision has already been built.""")
stabilize_wait = IntOption(
'bitten',
'stabilize_wait',
0,
doc="""The time in seconds to wait for the repository to stabilize before
queuing up a new build. This allows time for developers to check in
a group of related changes back to back without spawning multiple
builds.""")
slave_timeout = IntOption(
'bitten',
'slave_timeout',
3600,
doc="""The time in seconds after which a build is cancelled if the slave
does not report progress.""")
logs_dir = Option(
'bitten',
'logs_dir',
"log/bitten",
doc=
"""The directory on the server in which client log files will be stored."""
)
quick_status = BoolOption(
'bitten',
'quick_status',
False,
doc="""Whether to show the current build status within the Trac main
navigation bar. '''Note:''' The feature requires expensive database and
repository checks for every page request, and should not be enabled
if the project has a large repository or uses a non-Subversion
repository such as Mercurial or Git.""")
def __init__(self):
self.env.systeminfo.append(
('Bitten', __import__('bitten', ['__version__']).__version__))
# IRequestHandler methods
def match_request(self, req):
match = re.match(r'/builds(?:/(\d+)(?:/(\w+)/([^/]+)?)?)?$',
req.path_info)
if match:
if match.group(1):
req.args['id'] = match.group(1)
req.args['collection'] = match.group(2)
req.args['member'] = match.group(3)
return True
def process_request(self, req):
req.perm.assert_permission('BUILD_EXEC')
if 'trac_auth' in req.incookie:
slave_token = req.incookie['trac_auth'].value
else:
slave_token = req.session.sid
if 'id' not in req.args:
if req.method != 'POST':
self._send_response(
req, body='Only POST allowed for build creation.')
return self._process_build_creation(req, slave_token)
build = Build.fetch(self.env, req.args['id'])
if not build:
self._send_error(req, HTTP_NOT_FOUND,
'No such build (%s)' % req.args['id'])
build_token = build.slave_info.get('token', '')
if build_token != slave_token:
self._send_error(req, HTTP_CONFLICT,
'Token mismatch (wrong slave): slave=%s, build=%s' \
% (slave_token, build_token))
config = BuildConfig.fetch(self.env, build.config)
if not req.args['collection']:
if req.method == 'DELETE':
return self._process_build_cancellation(req, config, build)
else:
return self._process_build_initiation(req, config, build)
if req.method != 'POST':
self._send_error(req, HTTP_METHOD_NOT_ALLOWED,
'Method %s not allowed' % req.method)
if req.args['collection'] == 'steps':
return self._process_build_step(req, config, build)
elif req.args['collection'] == 'attach':
return self._process_attachment(req, config, build)
elif req.args['collection'] == 'keepalive':
return self._process_keepalive(req, config, build)
else:
self._send_error(
req, HTTP_NOT_FOUND,
"No such collection '%s'" % req.args['collection'])
# Internal methods
def _send_response(self, req, code=200, body='', headers=None):
""" Formats and sends the response, raising ``RequestDone``. """
if isinstance(body, unicode):
body = body.encode('utf-8')
req.send_response(code)
headers = headers or {}
headers.setdefault('Content-Length', len(body))
for header in headers:
req.send_header(header, headers[header])
req.write(body)
raise RequestDone
def _send_error(self, req, code=500, message=''):
""" Formats and sends the error, raising ``RequestDone``. """
headers = {
'Content-Type': 'text/plain',
'Content-Length': str(len(message))
}
self._send_response(req, code, body=message, headers=headers)
def _process_build_creation(self, req, slave_token):
queue = BuildQueue(self.env,
build_all=self.build_all,
stabilize_wait=self.stabilize_wait,
timeout=self.slave_timeout)
try:
queue.populate()
except AssertionError, e:
self.log.error(e.message, exc_info=True)
self._send_error(req, HTTP_BAD_REQUEST, e.message)
try:
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build initialization request: %s',
e,
exc_info=True)
self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
def _get_cases(fileobj):
for testsuite in xmlio.parse(fileobj).children('test-suite'):
for suite in _parse_suite(testsuite):
yield suite
def cunit (ctxt, file_=None, srcdir=None):
"""Collect CUnit XML data.
:param ctxt: the build context
:type ctxt: `Context`
:param file\_: path of the file containing the CUnit results; may contain
globbing wildcards to match multiple files
:param srcdir: name of the directory containing the source files, used to
link the test results to the corresponding files
"""
assert file_, 'Missing required attribute "file"'
try:
fileobj = file(ctxt.resolve(file_), 'r')
try:
total, failed = 0, 0
results = xmlio.Fragment()
log_elem = xmlio.Fragment()
def info (msg):
log.info (msg)
log_elem.append (xmlio.Element ('message', level='info')[msg])
def warning (msg):
log.warning (msg)
log_elem.append (xmlio.Element ('message', level='warning')[msg])
def error (msg):
log.error (msg)
log_elem.append (xmlio.Element ('message', level='error')[msg])
for node in xmlio.parse(fileobj):
if node.name != 'CUNIT_RESULT_LISTING':
continue
for suiteRun in node.children ('CUNIT_RUN_SUITE'):
for suite in suiteRun.children():
if suite.name not in ('CUNIT_RUN_SUITE_SUCCESS', 'CUNIT_RUN_SUITE_FAILURE'):
warning ("Unknown node: %s" % suite.name)
continue
suiteName = suite.children ('SUITE_NAME').next().gettext()
info ("%s [%s]" % ("*" * (57 - len (suiteName)), suiteName))
for record in suite.children ('CUNIT_RUN_TEST_RECORD'):
for result in record.children():
if result.name not in ('CUNIT_RUN_TEST_SUCCESS', 'CUNIT_RUN_TEST_FAILURE'):
continue
testName = result.children ('TEST_NAME').next().gettext()
info ("Running %s..." % testName);
test = xmlio.Element('test')
test.attr['fixture'] = suiteName
test.attr['name'] = testName
if result.name == 'CUNIT_RUN_TEST_FAILURE':
error ("%s(%d): %s"
% (result.children ('FILE_NAME').next().gettext(),
int (result.children ('LINE_NUMBER').next().gettext()),
result.children ('CONDITION').next().gettext()))
test.attr['status'] = 'failure'
failed += 1
else:
test.attr['status'] = 'success'
results.append(test)
total += 1
if failed:
ctxt.error('%d of %d test%s failed' % (failed, total,
total != 1 and 's' or ''))
ctxt.report('test', results)
ctxt.log (log_elem)
finally:
fileobj.close()
except IOError, e:
log.warning('Error opening CUnit results file (%s)', e)
