def reset_orphaned_builds(self):
"""Reset all in-progress builds to ``PENDING`` state if they've been
running so long that the configured timeout has been reached.
This is used to cleanup after slaves that have unexpectedly cancelled
a build without notifying the master, or are for some other reason not
reporting back status updates.
"""
if not self.timeout:
# If no timeout is set, none of the in-progress builds can be
# considered orphaned
return
db = self.env.get_db_cnx()
now = int(time.time())
for build in Build.select(self.env, status=Build.IN_PROGRESS, db=db):
if now - build.started < self.timeout:
# This build has not reached the timeout yet, assume it's still
# being executed
# FIXME: ideally, we'd base this check on the last activity on
# the build, not the start time
continue
build.status = Build.PENDING
build.slave = None
build.slave_info = {}
build.started = 0
for step in list(BuildStep.select(self.env, build=build.id,
db=db)):
step.delete(db=db)
build.update(db=db)
db.commit()
def template_data(self):
failed_steps = BuildStep.select(self.env,
build=self.build.id,
status=BuildStep.FAILURE)
change = self.get_changeset()
return {
'build': {
'id':
self.build.id,
'status':
self.readable_states[self.build.status],
'link':
self.build_link(),
'config':
self.build.config,
'slave':
self.build.slave,
'failed_steps': [{
'name':
step.name,
'description':
step.description,
'errors':
step.errors,
'log_messages':
self.get_all_log_messages_for_step(step),
} for step in failed_steps],
},
'change': {
'rev': change.rev,
'link': self.env.abs_href.changeset(change.rev),
'author': change.author,
},
}
def reset_orphaned_builds(self):
"""Reset all in-progress builds to ``PENDING`` state if they've been
running so long that the configured timeout has been reached.
This is used to cleanup after slaves that have unexpectedly cancelled
a build without notifying the master, or are for some other reason not
reporting back status updates.
"""
if not self.timeout:
# If no timeout is set, none of the in-progress builds can be
# considered orphaned
return
db = self.env.get_db_cnx()
now = int(time.time())
for build in Build.select(self.env, status=Build.IN_PROGRESS, db=db):
if now - build.started < self.timeout:
# This build has not reached the timeout yet, assume it's still
# being executed
# FIXME: ideally, we'd base this check on the last activity on
# the build, not the start time
continue
build.status = Build.PENDING
build.slave = None
build.slave_info = {}
build.started = 0
for step in list(BuildStep.select(self.env, build=build.id, db=db)):
step.delete(db=db)
build.update(db=db)
db.commit()
def template_data(self):
failed_steps = BuildStep.select(self.env, build=self.build.id,
status=BuildStep.FAILURE)
platform = TargetPlatform.fetch(self.env, id=self.build.platform)
reposname, repos, change = self.get_change()
return {
'build': {
'id': self.build.id,
'status': self.readable_states[self.build.status],
'link': self.build_link(),
'config': self.build.config,
'platform': getattr(platform, 'name', 'unknown'),
'slave': self.build.slave,
'failed_steps': [{
'name': step.name,
'description': step.description,
'errors': step.errors,
'log_messages': self.get_all_log_messages_for_step(step),
} for step in failed_steps],
},
'change': {
'rev': display_rev(repos, change.rev),
'link': self.env.abs_href.changeset(change.rev,
reposname != '(default)' and reposname or None),
'author': change.author,
},
}
def test_process_build_step_success_with_log(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.IP_ADDRESS] = '127.0.0.1';
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.cmlenz.net/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write)
module = BuildMaster(self.env)
assert module.match_request(req)
try:
module.process_request(req)
self.fail('Expected RequestDone')
except RequestDone:
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
logs = list(BuildLog.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(logs))
self.assertEqual('http://bitten.cmlenz.net/tools/python#unittest',
logs[0].generator)
self.assertEqual(2, len(logs[0].messages))
self.assertEqual((u'info', u'Doing stuff'), logs[0].messages[0])
self.assertEqual((u'error', u'Ouch that hurt'), logs[0].messages[1])
def process_request(self, req):
req.perm.require('BUILD_VIEW')
db = self.env.get_db_cnx()
build_id = int(req.args.get('id'))
build = Build.fetch(self.env, build_id, db=db)
assert build, 'Build %s does not exist' % build_id
if req.method == 'POST':
if req.args.get('action') == 'invalidate':
self._do_invalidate(req, build, db)
req.redirect(req.href.build(build.config, build.id))
add_link(req, 'up', req.href.build(build.config),
'Build Configuration')
data = {'title': 'Build %s - %s' % (build_id,
_status_title[build.status]),
'page_mode': 'view_build',
'build': {}}
config = BuildConfig.fetch(self.env, build.config, db=db)
data['build']['config'] = {
'name': config.label,
'href': req.href.build(config.name)
}
formatters = []
for formatter in self.log_formatters:
formatters.append(formatter.get_formatter(req, build))
summarizers = {} # keyed by report type
for summarizer in self.report_summarizers:
categories = summarizer.get_supported_categories()
summarizers.update(dict([(cat, summarizer) for cat in categories]))
data['build'].update(_get_build_data(self.env, req, build))
steps = []
for step in BuildStep.select(self.env, build=build.id, db=db):
steps.append({
'name': step.name, 'description': step.description,
'duration': pretty_timedelta(step.started, step.stopped),
'failed': step.status == BuildStep.FAILURE,
'errors': step.errors,
'log': self._render_log(req, build, formatters, step),
'reports': self._render_reports(req, config, build, summarizers,
step)
})
data['build']['steps'] = steps
data['build']['can_delete'] = ('BUILD_DELETE' in req.perm)
repos = self.env.get_repository(req.authname)
repos.authz.assert_permission(config.path)
chgset = repos.get_changeset(build.rev)
data['build']['chgset_author'] = chgset.author
add_script(req, 'bitten/tabset.js')
add_stylesheet(req, 'bitten/bitten.css')
return 'bitten_build.html', data, None
def test_process_build_step_failure_continue(self):
recipe = """<build>
<step id="foo" onerror="continue">
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123';
build.insert()
inbody = StringIO("""<result step="foo" status="failure"
time="2007-04-01T15:30:00.0000"
duration="3.45">
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.FAILURE, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.FAILURE, steps[0].status)
def _format_plaintext(self, event):
failed_steps = BuildStep.select(self.env,
build=event.target.id,
status=BuildStep.FAILURE)
change = self._get_changeset(event.target)
data = {
'build': {
'id':
event.target.id,
'status':
self.readable_states.get(event.target.status,
event.target.status),
'link':
self._build_link(event.target),
'config':
event.target.config,
'slave':
event.target.slave,
'failed_steps': [{
'name':
step.name,
'description':
step.description,
'errors':
step.errors,
'log_messages':
self._get_all_log_messages_for_step(event.target, step),
} for step in failed_steps],
},
'change': {
'rev': change.rev,
'link': self.env.abs_href.changeset(change.rev),
'author': change.author,
},
'project': {
'name': self.env.project_name,
'url': self.env.project_url or self.env.abs_href(),
'descr': self.env.project_description
}
}
chrome = Chrome(self.env)
dirs = []
for provider in chrome.template_providers:
dirs += provider.get_templates_dirs()
templates = TemplateLoader(dirs, variable_lookup='lenient')
template = templates.load('bitten_plaintext.txt', cls=NewTextTemplate)
if template:
stream = template.generate(**data)
output = stream.render('text')
return output
def test_process_build_step_wrong_slave(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123';
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth='))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(409, outheaders['Status'])
self.assertEqual('Token mismatch (wrong slave): slave=, build=123',
outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.IN_PROGRESS, build.status)
assert not build.stopped
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
def _do_invalidate(self, req, build, db):
self.log.info('Invalidating build %d', build.id)
for step in BuildStep.select(self.env, build=build.id, db=db):
step.delete(db=db)
build.slave = None
build.started = build.stopped = 0
build.status = Build.PENDING
build.slave_info = {}
build.update()
db.commit()
req.redirect(req.href.build(build.config))
def _do_invalidate(self, req, build, db):
self.log.info('Invalidating build %d', build.id)
for step in BuildStep.select(self.env, build=build.id, db=db):
step.delete(db=db)
build.slave = None
build.started = build.stopped = 0
build.status = Build.PENDING
build.slave_info = {}
build.update()
db.commit()
req.redirect(req.href.build(build.config))
def test_select(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.executemany("INSERT INTO bitten_step VALUES (%s,%s,%s,%s,%s,%s)",
[(1, 'test', 'Foo bar', BuildStep.SUCCESS, 1, 2),
(1, 'dist', 'Foo baz', BuildStep.FAILURE, 2, 3)])
steps = list(BuildStep.select(self.env, build=1))
self.assertEqual(1, steps[0].build)
self.assertEqual('test', steps[0].name)
self.assertEqual('Foo bar', steps[0].description)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
self.assertEqual(1, steps[1].build)
self.assertEqual('dist', steps[1].name)
self.assertEqual('Foo baz', steps[1].description)
self.assertEqual(BuildStep.FAILURE, steps[1].status)
def get_timeline_events(self, req, start, stop, filters):
if 'build' not in filters:
return
# Attachments (will be rendered by attachment module)
for event in AttachmentModule(self.env).get_timeline_events(
req, Resource('build'), start, stop):
yield event
start = to_timestamp(start)
stop = to_timestamp(stop)
add_stylesheet(req, 'bitten/bitten.css')
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"SELECT b.id,b.config,c.label,c.path, b.rev,p.name,"
"b.stopped,b.status FROM bitten_build AS b"
" INNER JOIN bitten_config AS c ON (c.name=b.config) "
" INNER JOIN bitten_platform AS p ON (p.id=b.platform) "
"WHERE b.stopped>=%s AND b.stopped<=%s "
"AND b.status IN (%s, %s) ORDER BY b.stopped",
(start, stop, Build.SUCCESS, Build.FAILURE))
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
event_kinds = {
Build.SUCCESS: 'successbuild',
Build.FAILURE: 'failedbuild'
}
for id_, config, label, path, rev, platform, stopped, status in cursor:
if not _has_permission(req.perm, repos, path, rev=rev):
continue
errors = []
if status == Build.FAILURE:
for step in BuildStep.select(self.env,
build=id_,
status=BuildStep.FAILURE,
db=db):
errors += [(step.name, error) for error in step.errors]
display_rev = repos.normalize_rev(rev)
yield (event_kinds[status], to_datetime(stopped, utc), None,
(id_, config, label, display_rev, platform, status, errors))
def _process_build_cancellation(self, req, config, build):
self.log.info('Build slave %r cancelled build %d', build.slave,
build.id)
build.status = Build.PENDING
build.slave = None
build.slave_info = {}
build.started = 0
db = self.env.get_db_cnx()
for step in list(BuildStep.select(self.env, build=build.id, db=db)):
step.delete(db=db)
build.update(db=db)
db.commit()
for listener in BuildSystem(self.env).listeners:
listener.build_aborted(build)
self._send_response(req, 204, '', {})
def _process_build_cancellation(self, req, config, build):
self.log.info('Build slave %r cancelled build %d', build.slave,
build.id)
build.status = Build.PENDING
build.slave = None
build.slave_info = {}
build.started = 0
db = self.env.get_db_cnx()
for step in list(BuildStep.select(self.env, build=build.id, db=db)):
step.delete(db=db)
build.update(db=db)
db.commit()
for listener in BuildSystem(self.env).listeners:
listener.build_aborted(build)
self._send_response(req, 204, '', {})
def test_process_build_step_wrong_slave(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.IP_ADDRESS] = '192.168.1.1';
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.cmlenz.net/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write)
module = BuildMaster(self.env)
assert module.match_request(req)
try:
module.process_request(req)
self.fail('Expected HTTPForbidden')
except HTTPForbidden, e:
self.assertEqual('Build 1 has been invalidated for host 127.0.0.1.', e.detail)
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.IN_PROGRESS, build.status)
assert not build.stopped
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(0, len(steps))
def get_timeline_events(self, req, start, stop, filters):
if 'build' not in filters:
return
# Attachments (will be rendered by attachment module)
for event in AttachmentModule(self.env).get_timeline_events(
req, Resource('build'), start, stop):
yield event
start = to_timestamp(start)
stop = to_timestamp(stop)
add_stylesheet(req, 'bitten/bitten.css')
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT b.id,b.config,c.label,c.path, b.rev,p.name,"
"b.stopped,b.status FROM bitten_build AS b"
" INNER JOIN bitten_config AS c ON (c.name=b.config) "
" INNER JOIN bitten_platform AS p ON (p.id=b.platform) "
"WHERE b.stopped>=%s AND b.stopped<=%s "
"AND b.status IN (%s, %s) ORDER BY b.stopped",
(start, stop, Build.SUCCESS, Build.FAILURE))
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
event_kinds = {Build.SUCCESS: 'successbuild',
Build.FAILURE: 'failedbuild'}
for id_, config, label, path, rev, platform, stopped, status in cursor:
if not _has_permission(req.perm, repos, path, rev=rev):
continue
errors = []
if status == Build.FAILURE:
for step in BuildStep.select(self.env, build=id_,
status=BuildStep.FAILURE,
db=db):
errors += [(step.name, error) for error
in step.errors]
display_rev = repos.normalize_rev(rev)
yield (event_kinds[status], to_datetime(stopped, utc), None,
(id_, config, label, display_rev, platform, status,
errors))
def _format_plaintext(self, event):
failed_steps = BuildStep.select(self.env, build=event.target.id,
status=BuildStep.FAILURE)
change = self._get_changeset(event.target)
data = {
'build': {
'id': event.target.id,
'status': self.readable_states.get(
event.target.status, event.target.status
),
'link': self._build_link(event.target),
'config': event.target.config,
'slave': event.target.slave,
'failed_steps': [{
'name': step.name,
'description': step.description,
'errors': step.errors,
'log_messages':
self._get_all_log_messages_for_step(event.target,
step),
} for step in failed_steps],
},
'change': {
'rev': change.rev,
'link': self.env.abs_href.changeset(change.rev),
'author': change.author,
},
'project': {
'name': self.env.project_name,
'url': self.env.project_url or self.env.abs_href(),
'descr': self.env.project_description
}
}
chrome = Chrome(self.env)
dirs = []
for provider in chrome.template_providers:
dirs += provider.get_templates_dirs()
templates = TemplateLoader(dirs, variable_lookup='lenient')
template = templates.load('bitten_plaintext.txt',
cls=NewTextTemplate)
if template:
stream = template.generate(**data)
return stream.render('text')
def template_data(self):
failed_steps = BuildStep.select(self.env,
build=self.build.id,
status=BuildStep.FAILURE)
platform = TargetPlatform.fetch(self.env, id=self.build.platform)
reposname, repos, change = self.get_change()
return {
'build': {
'id':
self.build.id,
'status':
self.readable_states[self.build.status],
'link':
self.build_link(),
'config':
self.build.config,
'platform':
getattr(platform, 'name', 'unknown'),
'slave':
self.build.slave,
'failed_steps': [{
'name':
step.name,
'description':
step.description,
'errors':
step.errors,
'log_messages':
self.get_all_log_messages_for_step(step),
} for step in failed_steps],
},
'change': {
'rev':
display_rev(repos, change.rev),
'link':
self.env.abs_href.changeset(
change.rev, reposname != '(default)' and reposname
or None),
'author':
change.author,
},
}
def reset_orphaned_builds(self):
"""Reset all in-progress builds to ``PENDING`` state if they've been
running so long that the configured timeout has been reached.
This is used to cleanup after slaves that have unexpectedly cancelled
a build without notifying the master, or are for some other reason not
reporting back status updates.
"""
if not self.timeout:
# If no timeout is set, none of the in-progress builds can be
# considered orphaned
return
db = self.env.get_db_cnx()
now = int(time.time())
for build in Build.select(self.env, status=Build.IN_PROGRESS, db=db):
if now - build.last_activity < self.timeout:
# This build has not reached the timeout yet, assume it's still
# being executed
continue
self.log.info('Orphaning build %d. Last activity was %s (%s)' % \
(build.id, format_datetime(build.last_activity),
pretty_timedelta(build.last_activity)))
build.status = Build.PENDING
build.slave = None
build.slave_info = {}
build.started = 0
build.stopped = 0
build.last_activity = 0
for step in list(BuildStep.select(self.env, build=build.id,
db=db)):
step.delete(db=db)
build.update(db=db)
Attachment.delete_all(self.env, 'build', build.resource.id, db)
db.commit()
def reset_orphaned_builds(self):
"""Reset all in-progress builds to ``PENDING`` state if they've been
running so long that the configured timeout has been reached.
This is used to cleanup after slaves that have unexpectedly cancelled
a build without notifying the master, or are for some other reason not
reporting back status updates.
"""
if not self.timeout:
# If no timeout is set, none of the in-progress builds can be
# considered orphaned
return
db = self.env.get_db_cnx()
now = int(time.time())
for build in Build.select(self.env, status=Build.IN_PROGRESS, db=db):
if now - build.last_activity < self.timeout:
# This build has not reached the timeout yet, assume it's still
# being executed
continue
self.log.info('Orphaning build %d. Last activity was %s (%s)' % \
(build.id, format_datetime(build.last_activity),
pretty_timedelta(build.last_activity)))
build.status = Build.PENDING
build.slave = None
build.slave_info = {}
build.started = 0
build.stopped = 0
build.last_activity = 0
for step in list(BuildStep.select(self.env, build=build.id, db=db)):
step.delete(db=db)
build.update(db=db)
Attachment.delete_all(self.env, 'build', build.resource.id, db)
db.commit()
def _format_plaintext(self, event):
failed_steps = BuildStep.select(self.env, build=event.target.id, status=BuildStep.FAILURE)
change = self._get_changeset(event.target)
data = {
"build": {
"id": event.target.id,
"status": self.readable_states.get(event.target.status, event.target.status),
"link": self._build_link(event.target),
"config": event.target.config,
"slave": event.target.slave,
"failed_steps": [
{
"name": step.name,
"description": step.description,
"errors": step.errors,
"log_messages": self._get_all_log_messages_for_step(event.target, step),
}
for step in failed_steps
],
},
"change": {"rev": change.rev, "link": self.env.abs_href.changeset(change.rev), "author": change.author},
"project": {
"name": self.env.project_name,
"url": self.env.project_url or self.env.abs_href(),
"descr": self.env.project_description,
},
}
chrome = Chrome(self.env)
dirs = []
for provider in chrome.template_providers:
dirs += provider.get_templates_dirs()
templates = TemplateLoader(dirs, variable_lookup="lenient")
template = templates.load("bitten_plaintext.txt", cls=NewTextTemplate)
if template:
stream = template.generate(**data)
output = stream.render("text")
return output
def template_data(self):
failed_steps = BuildStep.select(self.env, build=self.build.id,
status=BuildStep.FAILURE)
change = self.get_changeset()
return {
'build': {
'id': self.build.id,
'status': self.readable_states[self.build.status],
'link': self.build_link(),
'config': self.build.config,
'slave': self.build.slave,
'failed_steps': [{
'name': step.name,
'description': step.description,
'errors': step.errors,
'log_messages': self.get_all_log_messages_for_step(step),
} for step in failed_steps],
},
'change': {
'rev': change.rev,
'link': self.env.abs_href.changeset(change.rev),
'author': change.author,
},
}
def get_failed_steps(self, build):
build_steps = BuildStep.select(self.env,
build=build.id,
status=BuildStep.FAILURE)
return build_steps
def _render_inprogress(self, req):
data = {'title': 'In Progress Builds', 'page_mode': 'view-inprogress'}
db = self.env.get_db_cnx()
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
configs = []
for config in BuildConfig.select(self.env, include_inactive=False):
rev = config.max_rev or repos.youngest_rev
try:
if not _has_permission(req.perm, repos, config.path, rev=rev):
continue
except NoSuchNode:
add_warning(req, "Configuration '%s' points to non-existing "
"path '/%s' at revision '%s'. Configuration skipped." \
% (config.name, config.path, rev))
continue
self.log.debug(config.name)
if not config.active:
continue
in_progress_builds = Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
db=db)
current_builds = 0
builds = []
# sort correctly by revision.
for build in sorted(
in_progress_builds,
cmp=lambda x, y: int(y.rev_time) - int(x.rev_time)):
rev = build.rev
build_data = _get_build_data(self.env, req, build)
build_data['rev'] = rev
build_data['rev_href'] = req.href.changeset(rev)
platform = TargetPlatform.fetch(self.env, build.platform)
build_data['platform'] = platform.name
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id, db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds.append(build_data)
current_builds += 1
if current_builds == 0:
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
configs.append({
'name': config.name,
'label': config.label or config.name,
'active': config.active,
'path': config.path,
'description': description,
'href': req.href.build(config.name),
'builds': builds
})
data['configs'] = sorted(configs, key=lambda x: x['label'].lower())
return data
def test_process_build_step_success_with_report(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123';
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<report category="test"
generator="http://bitten.edgewall.org/tools/python#unittest">
<test fixture="my.Fixture" file="my/test/file.py">
<stdout>Doing my thing</stdout>
</test>
</report>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
reports = list(Report.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(reports))
self.assertEqual('test', reports[0].category)
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
reports[0].generator)
self.assertEqual(1, len(reports[0].items))
self.assertEqual({
'fixture': 'my.Fixture',
'file': 'my/test/file.py',
'stdout': 'Doing my thing',
'type': 'test',
}, reports[0].items[0])
def _render_inprogress(self, req):
data = {'title': 'In Progress Builds',
'page_mode': 'view-inprogress'}
db = self.env.get_db_cnx()
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
configs = []
for config in BuildConfig.select(self.env, include_inactive=False):
rev = config.max_rev or repos.youngest_rev
try:
if not _has_permission(req.perm, repos, config.path, rev=rev):
continue
except NoSuchNode:
add_warning(req, "Configuration '%s' points to non-existing "
"path '/%s' at revision '%s'. Configuration skipped." \
% (config.name, config.path, rev))
continue
self.log.debug(config.name)
if not config.active:
continue
in_progress_builds = Build.select(self.env, config=config.name,
status=Build.IN_PROGRESS, db=db)
current_builds = 0
builds = []
# sort correctly by revision.
for build in sorted(in_progress_builds,
cmp=lambda x, y: int(y.rev_time) - int(x.rev_time)):
rev = build.rev
build_data = _get_build_data(self.env, req, build)
build_data['rev'] = rev
build_data['rev_href'] = req.href.changeset(rev)
platform = TargetPlatform.fetch(self.env, build.platform)
build_data['platform'] = platform.name
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id, db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds.append(build_data)
current_builds += 1
if current_builds == 0:
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
configs.append({
'name': config.name, 'label': config.label or config.name,
'active': config.active, 'path': config.path,
'description': description,
'href': req.href.build(config.name),
'builds': builds
})
data['configs'] = sorted(configs, key=lambda x:x['label'].lower())
return data
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
rev = config.max_rev or repos.youngest_rev
try:
_has_permission(req.perm,
repos,
config.path,
rev=rev,
raise_error=True)
except NoSuchNode:
raise TracError("Permission checking against repository path %s "
"at revision %s failed." % (config.path, rev))
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(
Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))
data['config'] = {
'name': config.name,
'label': config.label,
'path': config.path,
'min_rev': config.min_rev,
'min_rev_href': req.href.changeset(config.min_rev),
'max_rev': config.max_rev,
'max_rev_href': req.href.changeset(config.max_rev),
'active': config.active,
'description': description,
'browser_href': req.href.browser(config.path),
'builds_pending': len(pending_builds),
'builds_inprogress': len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
platforms = list(
TargetPlatform.select(self.env, config=config_name, db=db))
data['config']['platforms'] = [{
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
} for platform in platforms]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
report_categories = list(
self._report_categories_for_config(config))
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
if category in report_categories:
chart_generators.append({
'href':
req.href.build(config.name, 'chart/' + category),
'category':
category,
'style':
self.config.get('bitten', 'chart_style'),
})
data['config']['charts'] = chart_generators
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
revisions = []
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
if rev not in builds:
revisions.append(rev)
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
builds[rev].setdefault('display_rev', repos.normalize_rev(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env,
build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
data['config']['revisions'] = revisions
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
if arity(prevnext_nav) == 4: # Trac 0.12 compat, see #450
prevnext_nav(req, 'Previous Page', 'Next Page')
else:
prevnext_nav(req, 'Page')
return data
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
repos = self.env.get_repository(req.authname)
if hasattr(repos, 'sync'):
repos.sync()
repos.authz.assert_permission(config.path)
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS))
data['config'] = {
'name': config.name, 'label': config.label, 'path': config.path,
'min_rev': config.min_rev,
'min_rev_href': req.href.changeset(config.min_rev),
'max_rev': config.max_rev,
'max_rev_href': req.href.changeset(config.max_rev),
'active': config.active, 'description': description,
'browser_href': req.href.browser(config.path),
'builds_pending' : len(pending_builds),
'builds_inprogress' : len(inprogress_builds)
}
platforms = list(TargetPlatform.select(self.env, config=config_name,
db=db))
data['config']['platforms'] = [
{ 'name': platform.name,
'id': platform.id,
'builds_pending': len(list(Build.select(self.env, config=config.name, status=Build.PENDING, platform=platform.id))),
'builds_inprogress': len(list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS, platform=platform.id)))
}
for platform in platforms
]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
chart_generators.append({
'href': req.href.build(config.name, 'chart/' + category)
})
data['config']['charts'] = chart_generators
charts_license = self.config.get('bitten', 'charts_license')
if charts_license:
data['config']['charts_license'] = charts_license
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(req.authname)
if hasattr(repos, 'sync'):
repos.sync()
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': datetime.fromtimestamp(step.stopped) - \
datetime.fromtimestamp(step.started),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
prevnext_nav(req, 'Page')
return data
def process_request(self, req):
req.perm.require('BUILD_VIEW')
db = self.env.get_db_cnx()
build_id = int(req.args.get('id'))
build = Build.fetch(self.env, build_id, db=db)
if not build:
raise HTTPNotFound("Build '%s' does not exist." \
% build_id)
if req.method == 'POST':
if req.args.get('action') == 'invalidate':
self._do_invalidate(req, build, db)
req.redirect(req.href.build(build.config, build.id))
add_link(req, 'up', req.href.build(build.config),
'Build Configuration')
data = {
'title': 'Build %s - %s' % (build_id, _status_title[build.status]),
'page_mode': 'view_build',
'build': {}
}
config = BuildConfig.fetch(self.env, build.config, db=db)
data['build']['config'] = {
'name': config.label or config.name,
'href': req.href.build(config.name)
}
context = Context.from_request(req, build.resource)
data['context'] = context
data['build']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
formatters = []
for formatter in self.log_formatters:
formatters.append(formatter.get_formatter(req, build))
summarizers = {} # keyed by report type
for summarizer in self.report_summarizers:
categories = summarizer.get_supported_categories()
summarizers.update(dict([(cat, summarizer) for cat in categories]))
data['build'].update(_get_build_data(self.env, req, build))
steps = []
for step in BuildStep.select(self.env, build=build.id, db=db):
steps.append({
'name':
step.name,
'description':
step.description,
'duration':
pretty_timedelta(step.started, step.stopped),
'failed':
step.status == BuildStep.FAILURE,
'errors':
step.errors,
'log':
self._render_log(req, build, formatters, step),
'reports':
self._render_reports(req, config, build, summarizers, step)
})
data['build']['steps'] = steps
data['build']['can_delete'] = ('BUILD_DELETE' in req.perm \
and build.status != build.PENDING)
repos = self.env.get_repository(req.authname)
repos.authz.assert_permission(config.branch)
chgset = repos.get_changeset(build.rev)
data['build']['chgset_author'] = chgset.author
add_script(req, 'common/js/folding.js')
add_script(req, 'bitten/tabset.js')
add_stylesheet(req, 'bitten/bitten.css')
return 'bitten_build.html', data, None
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
rev = config.max_rev or repos.youngest_rev
try:
_has_permission(req.perm, repos, config.path, rev=rev,
raise_error=True)
except NoSuchNode:
raise TracError("Permission checking against repository path %s "
"at revision %s failed." % (config.path, rev))
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(Build.select(self.env,
config=config.name, status=Build.PENDING))
inprogress_builds = list(Build.select(self.env,
config=config.name, status=Build.IN_PROGRESS))
data['config'] = {
'name': config.name, 'label': config.label, 'path': config.path,
'min_rev': config.min_rev,
'min_rev_href': req.href.changeset(config.min_rev),
'max_rev': config.max_rev,
'max_rev_href': req.href.changeset(config.max_rev),
'active': config.active, 'description': description,
'browser_href': req.href.browser(config.path),
'builds_pending' : len(pending_builds),
'builds_inprogress' : len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(self.env).attachment_data(context)
platforms = list(TargetPlatform.select(self.env, config=config_name,
db=db))
data['config']['platforms'] = [
{ 'name': platform.name,
'id': platform.id,
'builds_pending': len(list(Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress': len(list(Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
}
for platform in platforms
]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
report_categories = list(self._report_categories_for_config(config))
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
if category in report_categories:
chart_generators.append({
'href': req.href.build(config.name, 'chart/' + category),
'category': category,
'style': self.config.get('bitten', 'chart_style'),
})
data['config']['charts'] = chart_generators
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
revisions = []
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
if rev not in builds:
revisions.append(rev)
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
builds[rev].setdefault('display_rev', repos.normalize_rev(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
data['config']['revisions'] = revisions
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
if arity(prevnext_nav) == 4: # Trac 0.12 compat, see #450
prevnext_nav(req, 'Previous Page', 'Next Page')
else:
prevnext_nav (req, 'Page')
return data
def _render_inprogress(self, req):
data = {'title': 'In Progress Builds', 'page_mode': 'view-inprogress'}
db = self.env.get_db_cnx()
repos = self.env.get_repository(req.authname)
configs = []
for config in BuildConfig.select(self.env, include_inactive=False):
if not repos.authz.has_permission(config.branch):
continue
self.log.debug(config.name)
if not config.active:
continue
in_progress_builds = Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
db=db)
current_builds = 0
builds = []
# sort correctly by revision.
for build in sorted(in_progress_builds,
cmp=lambda x, y: int(y.rev) - int(x.rev)):
rev = build.rev
build_data = _get_build_data(self.env, req, build)
build_data['rev'] = rev
build_data['rev_href'] = req.href.changeset(rev)
platform = TargetPlatform.fetch(self.env, build.platform)
build_data['platform'] = platform.name
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id, db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped, utc) - \
to_datetime(step.started, utc),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds.append(build_data)
current_builds += 1
if current_builds == 0:
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
configs.append({
'name': config.name,
'label': config.label or config.name,
'active': config.active,
'branch': config.branch,
'description': description,
'href': req.href.build(config.name),
'builds': builds
})
data['configs'] = configs
return data
def test_process_build_step_success_with_log(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
logs = list(BuildLog.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(logs))
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
logs[0].generator)
self.assertEqual(2, len(logs[0].messages))
self.assertEqual((u'info', u'Doing stuff'), logs[0].messages[0])
self.assertEqual((u'error', u'Ouch that hurt'), logs[0].messages[1])
def process_request(self, req):
req.perm.require('BUILD_VIEW')
db = self.env.get_db_cnx()
build_id = int(req.args.get('id'))
build = Build.fetch(self.env, build_id, db=db)
if not build:
raise HTTPNotFound("Build '%s' does not exist." \
% build_id)
if req.method == 'POST':
if req.args.get('action') == 'invalidate':
self._do_invalidate(req, build, db)
req.redirect(req.href.build(build.config, build.id))
add_link(req, 'up', req.href.build(build.config),
'Build Configuration')
data = {'title': 'Build %s - %s' % (build_id,
_status_title[build.status]),
'page_mode': 'view_build',
'build': {}}
config = BuildConfig.fetch(self.env, build.config, db=db)
data['build']['config'] = {
'name': config.label or config.name,
'href': req.href.build(config.name)
}
context = Context.from_request(req, build.resource)
data['context'] = context
data['build']['attachments'] = AttachmentModule(self.env).attachment_data(context)
formatters = []
for formatter in self.log_formatters:
formatters.append(formatter.get_formatter(req, build))
summarizers = {} # keyed by report type
for summarizer in self.report_summarizers:
categories = summarizer.get_supported_categories()
summarizers.update(dict([(cat, summarizer) for cat in categories]))
data['build'].update(_get_build_data(self.env, req, build))
steps = []
for step in BuildStep.select(self.env, build=build.id, db=db):
steps.append({
'name': step.name, 'description': step.description,
'duration': pretty_timedelta(step.started, step.stopped or int(time.time())),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'log': self._render_log(req, build, formatters, step),
'reports': self._render_reports(req, config, build, summarizers,
step)
})
data['build']['steps'] = steps
data['build']['can_delete'] = ('BUILD_DELETE' in req.perm \
and build.status != build.PENDING)
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
_has_permission(req.perm, repos, config.path, rev=build.rev, raise_error=True)
chgset = repos.get_changeset(build.rev)
data['build']['chgset_author'] = chgset.author
data['build']['display_rev'] = repos.normalize_rev(build.rev)
add_script(req, 'common/js/folding.js')
add_script(req, 'bitten/tabset.js')
add_script(req, 'bitten/jquery.flot.js')
add_stylesheet(req, 'bitten/bitten.css')
return 'bitten_build.html', data, None
def test_process_build_step_success_with_report(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<report category="test"
generator="http://bitten.edgewall.org/tools/python#unittest">
<test fixture="my.Fixture" file="my/test/file.py">
<stdout>Doing my thing</stdout>
</test>
</report>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
reports = list(Report.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(reports))
self.assertEqual('test', reports[0].category)
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
reports[0].generator)
self.assertEqual(1, len(reports[0].items))
self.assertEqual(
{
'fixture': 'my.Fixture',
'file': 'my/test/file.py',
'stdout': 'Doing my thing',
'type': 'test',
}, reports[0].items[0])
def test_process_build_step_invalidated_build(self):
recipe = """<build>
<step id="foo">
</step>
<step id="foo2">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.IN_PROGRESS, build.status)
assert not build.stopped
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(2, len(steps))
# invalidate the build.
build = Build.fetch(self.env, build.id)
build.slave = None
build.status = Build.PENDING
build.slave_info = {}
for step in list(BuildStep.select(self.env, build=build.id)):
step.delete()
build.update()
# have this slave submit more data.
inbody = StringIO("""<result step="foo2" status="success"
time="2007-04-01T15:45:00.0000"
duration="4">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">This is a step after invalidation</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEquals(409, outheaders['Status'])
self.assertEquals('Token mismatch (wrong slave): slave=123, build=',
outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.PENDING, build.status)
def test_process_build_step_failure_continue(self):
recipe = """<build>
<step id="foo" onerror="continue">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="failure"
time="2007-04-01T15:30:00.0000"
duration="3.45">
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.FAILURE, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.FAILURE, steps[0].status)
def test_process_build_step_success_with_attach(self):
# Parse input and create attachments for config + build
recipe = """<build>
<step id="foo">
<attach file="bar.txt" description="bar bar"/>
<attach file="baz.txt" description="baz baz" resource="config"/>
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123';
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<attach>
<file filename="bar.txt"
description="bar bar">aGVsbG8gYmFy\n</file>
</attach>
<attach>
<file filename="baz.txt" description="baz baz"
resource="config">aGVsbG8gYmF6\n</file>
</attach>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
authname='hal',
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
from trac.attachment import Attachment
config_attachments = list(Attachment.select(self.env, 'build', 'test'))
build_attachments = list(Attachment.select(self.env, 'build', 'test/1'))
self.assertEquals(1, len(build_attachments))
self.assertEquals('hal', build_attachments[0].author)
self.assertEquals('bar bar', build_attachments[0].description)
self.assertEquals('bar.txt', build_attachments[0].filename)
self.assertEquals('hello bar',
build_attachments[0].open().read())
self.assertEquals(1, len(config_attachments))
self.assertEquals('hal', config_attachments[0].author)
self.assertEquals('baz baz', config_attachments[0].description)
self.assertEquals('baz.txt', config_attachments[0].filename)
self.assertEquals('hello baz',
config_attachments[0].open().read())
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(req.authname)
repos.authz.assert_permission(config.branch)
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(
Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))
rev = ''
for b in repos.git.get_branches():
if b[0] == config.branch:
rev = b[1]
break
data['config'] = {
'name': config.name,
'label': config.label,
'branch': config.branch,
'active': config.active,
'description': description,
'browser_href': req.href.browser(rev=rev),
'builds_pending': len(pending_builds),
'builds_inprogress': len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
platforms = list(
TargetPlatform.select(self.env, config=config_name, db=db))
data['config']['platforms'] = [{
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
} for platform in platforms]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
chart_generators.append({
'href':
req.href.build(config.name, 'chart/' + category)
})
data['config']['charts'] = chart_generators
charts_license = self.config.get('bitten', 'charts_license')
if charts_license:
data['config']['charts_license'] = charts_license
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(req.authname)
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env,
build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped, utc) - \
to_datetime(step.started, utc),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
prevnext_nav(req, 'Page')
return data
def _render_inprogress(self, req):
data = {'title': 'In Progress Builds',
'page_mode': 'view-inprogress'}
db = self.env.get_db_cnx()
repos = self.env.get_repository(req.authname)
if hasattr(repos, 'sync'):
repos.sync()
configs = []
for config in BuildConfig.select(self.env, include_inactive=False):
if not repos.authz.has_permission(config.path):
continue
self.log.debug(config.name)
if not config.active:
continue
in_progress_builds = Build.select(self.env, config=config.name,
status=Build.IN_PROGRESS, db=db)
current_builds = 0
builds = []
# sort correctly by revision.
for build in sorted(in_progress_builds,
cmp=lambda x, y: int(y.rev) - int(x.rev)):
rev = build.rev
build_data = _get_build_data(self.env, req, build)
build_data['rev'] = rev
build_data['rev_href'] = req.href.changeset(rev)
platform = TargetPlatform.fetch(self.env, build.platform)
build_data['platform'] = platform.name
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id, db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': datetime.fromtimestamp(step.stopped) - \
datetime.fromtimestamp(step.started),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds.append(build_data)
current_builds += 1
if current_builds == 0:
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
configs.append({
'name': config.name, 'label': config.label or config.name,
'active': config.active, 'path': config.path,
'description': description,
'href': req.href.build(config.name),
'builds': builds
})
data['configs'] = configs
return data
def get_timeline_events(self, req, start, stop, filters):
if 'build' not in filters:
return
if isinstance(start, datetime): # Trac>=0.11
from trac.util.datefmt import to_timestamp
start = to_timestamp(start)
stop = to_timestamp(stop)
add_stylesheet(req, 'bitten/bitten.css')
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT b.id,b.config,c.label,b.rev,p.name,"
"b.stopped,b.status FROM bitten_build AS b"
" INNER JOIN bitten_config AS c ON (c.name=b.config) "
" INNER JOIN bitten_platform AS p ON (p.id=b.platform) "
"WHERE b.stopped>=%s AND b.stopped<=%s "
"AND b.status IN (%s, %s) ORDER BY b.stopped",
(start, stop, Build.SUCCESS, Build.FAILURE))
repos = self.env.get_repository(req.authname)
if hasattr(repos, 'sync'):
repos.sync()
event_kinds = {Build.SUCCESS: 'successbuild',
Build.FAILURE: 'failedbuild'}
for id, config, label, rev, platform, stopped, status in cursor:
config_object = BuildConfig.fetch(self.env, config, db=db)
if not repos.authz.has_permission(config_object.path):
continue
errors = []
if status == Build.FAILURE:
for step in BuildStep.select(self.env, build=id,
status=BuildStep.FAILURE,
db=db):
errors += [(step.name, error) for error
in step.errors]
title = tag('Build of ', tag.em('%s [%s]' % (label, rev)),
' on %s %s' % (platform, _status_label[status]))
message = ''
if req.args.get('format') == 'rss':
href = req.abs_href.build(config, id)
if errors:
buf = StringIO()
prev_step = None
for step, error in errors:
if step != prev_step:
if prev_step is not None:
buf.write('</ul>')
buf.write('<p>Step %s failed:</p><ul>' \
% escape(step))
prev_step = step
buf.write('<li>%s</li>' % escape(error))
buf.write('</ul>')
message = Markup(buf.getvalue())
else:
href = req.href.build(config, id)
if errors:
steps = []
for step, error in errors:
if step not in steps:
steps.append(step)
steps = [Markup('<em>%s</em>') % step for step in steps]
if len(steps) < 2:
message = steps[0]
elif len(steps) == 2:
message = Markup(' and ').join(steps)
elif len(steps) > 2:
message = Markup(', ').join(steps[:-1]) + ', and ' + \
steps[-1]
message = Markup('Step%s %s failed') % (
len(steps) != 1 and 's' or '', message
)
yield event_kinds[status], href, title, stopped, None, message