def test_bitten_keeps_order_of_revisions_from_versioncontrol(self):
# Trac's API specifies that they are sorted chronological (backwards)
# We must not assume that these revision numbers can be sorted later on,
# for example the mercurial plugin will return the revisions as strings
# (e.g. '880:4c19fa95fb9e')
config = BuildConfig(self.env, name='test', path='trunk')
config.insert()
platform = TargetPlatform(self.env, config='test', name='any')
platform.insert()
PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW')
req = Mock(method='GET', base_path='', cgi_location='',
path_info='/build/'+config.name, href=Href('/trac'), args={},
chrome={}, authname='joe',
perm=PermissionCache(self.env, 'joe'))
# revisions are intentionally not sorted in any way - bitten should just keep them!
revision_ids = [5, 8, 2]
revision_list = [('trunk', revision, 'edit') for revision in revision_ids]
root = Mock(get_entries=lambda: ['foo'], get_history=lambda: revision_list)
self.repos.get_node=lambda path, rev=None: root
self.repos.youngest_rev=5
self.repos.get_changeset=lambda rev: Mock(author='joe', date=99)
module = BuildConfigController(self.env)
assert module.match_request(req)
_, data, _ = module.process_request(req)
actual_revision_ids = data['config']['revisions']
self.assertEquals(revision_ids, actual_revision_ids)
def test_view_config_paging(self):
config = BuildConfig(self.env, name='test', path='trunk')
config.insert()
platform = TargetPlatform(self.env, config='test', name='any')
platform.insert()
PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW')
req = Mock(method='GET', base_path='', cgi_location='',
path_info='/build/test', href=Href('/trac'), args={},
chrome={}, authname='joe',
perm=PermissionCache(self.env, 'joe'))
root = Mock(get_entries=lambda: ['foo'],
get_history=lambda: [('trunk', rev, 'edit') for rev in
range(123, 110, -1)])
self.repos = Mock(get_node=lambda path, rev=None: root,
sync=lambda: None, normalize_path=lambda path: path)
self.repos.authz = Mock(has_permission=lambda path: True, assert_permission=lambda path: None)
module = BuildConfigController(self.env)
assert module.match_request(req)
_, data, _ = module.process_request(req)
if req.chrome:
self.assertEqual('/trac/build/test?page=2',
req.chrome['links']['next'][0]['href'])
def test_view_config(self):
config = BuildConfig(self.env, name='test', path='trunk')
config.insert()
platform = TargetPlatform(self.env, config='test', name='any')
platform.insert()
PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW')
req = Mock(method='GET', base_path='', cgi_location='',
path_info='/build/test', href=Href('/trac'), args={},
chrome={}, authname='joe',
perm=PermissionCache(self.env, 'joe'))
root = Mock(get_entries=lambda: ['foo'],
get_history=lambda: [('trunk', rev, 'edit') for rev in
range(123, 111, -1)])
self.repos.get_node=lambda path, rev=None: root
self.repos.youngest_rev=123
self.repos.get_changeset=lambda rev: Mock(author='joe', date=99)
module = BuildConfigController(self.env)
assert module.match_request(req)
_, data, _ = module.process_request(req)
self.assertEqual('view_config', data['page_mode'])
assert not 'next' in req.chrome['links']
self.assertEquals(Resource('build', 'test'), data['context'].resource)
self.assertEquals([], data['config']['attachments']['attachments'])
self.assertEquals('/trac/attachment/build/test/',
data['config']['attachments']['attach_href'])
def test_cancel_build(self):
config = BuildConfig(self.env, 'test', path='somepath', active=True,
recipe='<build></build>')
config.insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
status=Build.IN_PROGRESS, started=42)
build.insert()
outheaders = {}
outbody = StringIO()
req = Mock(method='DELETE', base_path='',
path_info='/builds/%d' % build.id,
href=Href('/trac'), remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth='))
module = BuildMaster(self.env)
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(204, outheaders['Status'])
self.assertEqual('', outbody.getvalue())
# Make sure the started timestamp has been set
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.PENDING, build.status)
assert not build.started
def test_process_deactivate_config(self):
BuildConfig(self.env, name='foo', path='branches/foo',
active=True).insert()
BuildConfig(self.env, name='bar', path='branches/bar',
active=True).insert()
redirected_to = []
def redirect(url):
redirected_to.append(url)
raise RequestDone
req = Mock(method='POST', perm=PermissionCache(self.env, 'joe'),
abs_href=Href('http://example.org/'), redirect=redirect,
authname='joe',
args={'apply': ''})
provider = BuildConfigurationsAdminPageProvider(self.env)
try:
provider.render_admin_panel(req, 'bitten', 'configs', '')
self.fail('Expected RequestDone')
except RequestDone:
self.assertEqual('http://example.org/admin/bitten/configs',
redirected_to[0])
config = BuildConfig.fetch(self.env, name='foo')
self.assertEqual(False, config.active)
config = BuildConfig.fetch(self.env, name='bar')
self.assertEqual(False, config.active)
def test_insert(self):
config = BuildConfig(self.env, name='test', path='trunk', label='Test')
config.insert()
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT name,path,label,active,description "
"FROM bitten_config")
self.assertEqual(('test', 'trunk', 'Test', 0, ''), cursor.fetchone())
def test_should_delete_build_platform_dont_exist(self):
messages = []
self.env.log = Mock(info=lambda msg, *args: messages.append(msg))
config = BuildConfig(self.env, 'test', active=True)
config.insert()
build = Build(self.env, config=config.name, rev=42,
platform="no-stuff", rev_time=123456)
build.insert()
queue = BuildQueue(self.env, build_all=True)
self.assertEqual(True, queue.should_delete_build(build, self.repos))
self.assert_("platform no longer exists" in messages[0])
def test_should_delete_build_config_deactivated(self):
messages = []
self.env.log = Mock(info=lambda msg, *args: messages.append(msg))
config = BuildConfig(self.env, 'test', active=False)
config.insert()
platform = TargetPlatform(self.env, config='test', name='stuff')
platform.insert()
build = Build(self.env, config=config.name, rev=42,
platform=platform.id, rev_time=123456)
build.insert()
queue = BuildQueue(self.env, build_all=True)
self.assertEqual(True, queue.should_delete_build(build, self.repos))
self.assert_("configuration is deactivated" in messages[0])
def test_process_attach_build(self):
body, content_type = encode_multipart_formdata({
'description': 'baz baz',
'file': ('baz.txt', 'hello baz'),
'__FORM_TOKEN': '123456'})
args = {}
for k, v in dict(cgi.FieldStorage(fp=StringIO(body), environ={
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': content_type})
).items():
if v.filename:
args[k] = v
else:
args[k] = v.value
args.update({'collection': 'attach', 'member': 'build'})
self.assertTrue('file' in args)
outheaders = {}
outbody = StringIO()
req = Mock(args=args, form_token='123456', authname='hal',
remote_addr='127.0.0.1',
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write)
config = BuildConfig(self.env, 'test', path='somepath', active=True,
recipe='')
config.insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.insert()
module = BuildMaster(self.env)
self.assertRaises(RequestDone, module._process_attachment,
req, config, build)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('18', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Attachment created', outbody.getvalue())
build_atts = list(Attachment.select(self.env, 'build', 'test/1'))
self.assertEquals(1, len(build_atts))
self.assertEquals('hal', build_atts[0].author)
self.assertEquals('baz baz', build_atts[0].description)
self.assertEquals('baz.txt', build_atts[0].filename)
self.assertEquals('hello baz',
build_atts[0].open().read())
def process_request(self, req):
req.perm.assert_permission('BUILD_EXEC')
if 'id' not in req.args:
if req.method != 'POST':
raise HTTPMethodNotAllowed('Method not allowed')
return self._process_build_creation(req)
build = Build.fetch(self.env, req.args['id'])
if not build:
raise HTTPNotFound('No such build')
config = BuildConfig.fetch(self.env, build.config)
if not req.args['collection']:
if req.method == 'DELETE':
return self._process_build_cancellation(req, config, build)
else:
return self._process_build_initiation(req, config, build)
if req.method != 'POST':
raise HTTPMethodNotAllowed('Method not allowed')
if req.args['collection'] == 'steps':
return self._process_build_step(req, config, build)
else:
raise HTTPNotFound('No such collection')
def should_delete_build(self, build, repos):
config = BuildConfig.fetch(self.env, build.config)
platform = TargetPlatform.fetch(self.env, build.platform)
# Platform may or may not exist anymore - get safe name for logging
platform_name = platform and platform.name \
or 'unknown platform "%s"' % build.platform
# Drop build if platform no longer exists
if not platform:
self.log.info('Dropping build of configuration "%s" at '
'revision [%s] on %s because the platform no longer '
'exists', config.name, build.rev, platform_name)
return True
# Ignore pending builds for deactived build configs
if not config.active:
self.log.info('Dropping build of configuration "%s" at '
'revision [%s] on %s because the configuration is '
'deactivated', config.name, build.rev, platform_name)
return True
# If not 'build_all', drop if a more recent revision is available
if not self.build_all and \
len(list(Build.select(self.env, config=build.config,
min_rev_time=build.rev_time, platform=build.platform))) > 1:
self.log.info('Dropping build of configuration "%s" at revision [%s] '
'on "%s" because a more recent build exists',
config.name, build.rev, platform_name)
return True
return False
def get_annotation_data(self, context):
add_stylesheet(context.req, 'bitten/bitten_coverage.css')
resource = context.resource
self.log.debug("Looking for coverage report for %s@%s..." % (
resource.id, str(resource.version)))
builds = Build.select(self.env, rev=resource.version)
reports = []
for build in builds:
config = BuildConfig.fetch(self.env, build.config)
if not resource.id.startswith(config.branch):
continue
reports = Report.select(self.env, build=build.id,
category='coverage')
branch_in_config = resource.id[len(config.branch):]
for report in reports:
for item in report.items:
if item.get('file') == branch_in_config:
coverage = item.get('line_hits', '').split()
if coverage:
# Return first result with line data
self.log.debug(
"Coverage annotate for %s@%s: %s" % \
(resource.id, resource.version, coverage))
return coverage
return []
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.*'])
self.env.path = tempfile.mkdtemp()
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
self.config = BuildConfig(self.env, name='test', path='somepath')
self.config.insert(db=db)
self.platform = TargetPlatform(self.env, config='test', name='Foo')
self.platform.insert(db=db)
db.commit()
# Hook up a dummy repository
self.repos = Mock()
self.env.get_repository = lambda authname=None: self.repos # 0.11
try: # 0.12+
from trac.core import Component, implements
from trac.versioncontrol.api import IRepositoryConnector, \
IRepositoryProvider
class DummyRepos(Component):
implements(IRepositoryConnector, IRepositoryProvider)
def get_supported_types(self):
yield ('dummy', 9)
def get_repository(this, repos_type, repos_dir, params):
return self.repos # Note: 'this' vs 'self' usage
def get_repositories(self):
yield ('', {'dir': 'dummy_dir', 'type': 'dummy'})
self.dummy = DummyRepos
except ImportError:
self.dummy = None # not supported, will use get_repository()
def test_process_update_config(self):
BuildConfig(self.env, name='foo', label='Foo', path='branches/foo',
active=True).insert()
redirected_to = []
def redirect(url):
redirected_to.append(url)
raise RequestDone
req = Mock(method='POST', perm=PermissionCache(self.env, 'joe'),
abs_href=Href('http://example.org/'), redirect=redirect,
authname='joe', chrome={'warnings': [], 'notices': []},
href=Href('/'),
args={'save': '', 'name': 'foo', 'label': 'Foobar',
'description': 'Thanks for all the fish!'})
provider = BuildConfigurationsAdminPageProvider(self.env)
try:
provider.render_admin_panel(req, 'bitten', 'configs', 'foo')
self.fail('Expected RequestDone')
except RequestDone:
self.assertEqual(['Configuration Saved.'], req.chrome['notices'])
self.assertEqual('http://example.org/admin/bitten/configs/foo',
redirected_to[0])
config = BuildConfig.fetch(self.env, name='foo')
self.assertEqual('Foobar', config.label)
self.assertEqual('Thanks for all the fish!', config.description)
def should_delete_build(self, build, repos):
# Ignore pending builds for deactived build configs
config = BuildConfig.fetch(self.env, build.config)
if not config.active:
target_platform = TargetPlatform.fetch(self.env, build.platform)
if target_platform:
target_platform_name = '"%s"' % (target_platform.name,)
else:
target_platform_name = 'unknown platform "%s"' % (build.platform,)
log.info('Dropping build of configuration "%s" at '
'revision [%s] on %s because the configuration is '
'deactivated', config.name, build.rev,
target_platform_name)
return True
# Stay within the revision limits of the build config
if (config.min_rev and repos.rev_older_than(build.rev,
config.min_rev)) \
or (config.max_rev and repos.rev_older_than(config.max_rev,
build.rev)):
# This minimum and/or maximum revision has changed since
# this build was enqueued, so drop it
log.info('Dropping build of configuration "%s" at revision [%s] on '
'"%s" because it is outside of the revision range of the '
'configuration', config.name, build.rev,
TargetPlatform.fetch(self.env, build.platform).name)
return True
return False
def get_navigation_items(self, req):
"""Return the navigation item for access the build status overview from
the Trac navigation bar."""
if 'BUILD_VIEW' in req.perm:
status = ''
if BuildMaster(self.env).quick_status:
repos = self.env.get_repository(req.authname)
if hasattr(repos, 'sync'):
repos.sync()
for config in BuildConfig.select(self.env,
include_inactive=False):
prev_rev = None
for platform, rev, build in collect_changes(repos, config):
if rev != prev_rev:
if prev_rev is not None:
break
prev_rev = rev
if build:
build_data = _get_build_data(self.env, req, build)
if build_data['status'] == 'failed':
status='bittenfailed'
break
if build_data['status'] == 'in progress':
status='bitteninprogress'
elif not status:
if (build_data['status'] == 'completed'):
status='bittencompleted'
if not status:
status='bittenpending'
yield ('mainnav', 'build',
tag.a('Builds Status', href=req.href.build(), accesskey=5,
class_=status))
def populate(self):
"""Add a build for the next change on each build configuration to the
queue.
The next change is the latest repository check-in for which there isn't
a corresponding build on each target platform. Repeatedly calling this
method will eventually result in the entire change history of the build
configuration being in the build queue.
"""
repos = self.env.get_repository()
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
db = self.env.get_db_cnx()
builds = []
for config in BuildConfig.select(self.env, db=db):
platforms = []
for platform, rev, build in collect_changes(repos, config, db):
if not self.build_all and platform.id in platforms:
# We've seen this platform already, so these are older
# builds that should only be built if built_all=True
self.log.debug('Ignoring older revisions for configuration '
'%r on %r', config.name, platform.name)
break
platforms.append(platform.id)
if build is None:
self.log.info('Enqueuing build of configuration "%s" at '
'revision [%s] on %s', config.name, rev,
platform.name)
rev_time = to_timestamp(repos.get_changeset(rev).date)
age = int(time.time()) - rev_time
if self.stabilize_wait and age < self.stabilize_wait:
self.log.info('Delaying build of revision %s until %s '
'seconds pass. Current age is: %s '
'seconds' % (rev, self.stabilize_wait,
age))
continue
build = Build(self.env, config=config.name,
platform=platform.id, rev=str(rev),
rev_time=rev_time)
builds.append(build)
for build in builds:
try:
build.insert(db=db)
db.commit()
except Exception, e:
# really only want to catch IntegrityErrors raised when
# a second slave attempts to add builds with the same
# (config, platform, rev) as an existing build.
self.log.info('Failed to insert build of configuration "%s" '
'at revision [%s] on platform [%s]: %s',
build.config, build.rev, build.platform, e)
db.rollback()
def test_raise_404(self):
PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW')
module = BuildController(self.env)
config = BuildConfig(self.env, name='existing', path='trunk')
config.insert()
req = Mock(method='GET', base_path='', cgi_location='',
path_info='/build/existing/42', href=Href('/trac'), args={},
chrome={}, authname='joe',
perm=PermissionCache(self.env, 'joe'))
self.failUnless(module.match_request(req))
try:
module.process_request(req)
except Exception, e:
self.failUnless(isinstance(e, HTTPNotFound))
self.assertEquals(str(e),
"404 Not Found (Build '42' does not exist.)")
return
def resource_exists(self, resource):
config_name, build_id = self._parse_resource(resource.id)
if build_id:
build = Build.fetch(self.env, build_id)
return build and build.exists or False
elif config_name:
config = BuildConfig.fetch(self.env, config_name)
return config and config.exists or False
return False
def test_should_delete_build_old_with_not_buildall(self):
messages = []
self.env.log = Mock(info=lambda msg, *args: messages.append(msg))
config = BuildConfig(self.env, 'test', active=True)
config.insert()
platform = TargetPlatform(self.env, config='test', name='stuff')
platform.insert()
build1 = Build(self.env, config=config.name, rev=42,
platform=platform.id, rev_time=123456)
build1.insert()
build2 = Build(self.env, config=config.name, rev=43,
platform=platform.id, rev_time=123457,
slave='slave')
build2.insert()
queue = BuildQueue(self.env, build_all=False)
self.assertEqual(True, queue.should_delete_build(build1, self.repos))
self.assert_("more recent build exists" in messages[0])
def test_update_no_name(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_config (name,path,label,active) "
"VALUES (%s,%s,%s,%s)", ('test', 'trunk', 'Test', 0))
config = BuildConfig.fetch(self.env, 'test')
config.name = None
self.assertRaises(AssertionError, config.update)
def process_request(self, req):
req.perm.require('BUILD_VIEW')
db = self.env.get_db_cnx()
build_id = int(req.args.get('id'))
build = Build.fetch(self.env, build_id, db=db)
assert build, 'Build %s does not exist' % build_id
if req.method == 'POST':
if req.args.get('action') == 'invalidate':
self._do_invalidate(req, build, db)
req.redirect(req.href.build(build.config, build.id))
add_link(req, 'up', req.href.build(build.config),
'Build Configuration')
data = {'title': 'Build %s - %s' % (build_id,
_status_title[build.status]),
'page_mode': 'view_build',
'build': {}}
config = BuildConfig.fetch(self.env, build.config, db=db)
data['build']['config'] = {
'name': config.label,
'href': req.href.build(config.name)
}
formatters = []
for formatter in self.log_formatters:
formatters.append(formatter.get_formatter(req, build))
summarizers = {} # keyed by report type
for summarizer in self.report_summarizers:
categories = summarizer.get_supported_categories()
summarizers.update(dict([(cat, summarizer) for cat in categories]))
data['build'].update(_get_build_data(self.env, req, build))
steps = []
for step in BuildStep.select(self.env, build=build.id, db=db):
steps.append({
'name': step.name, 'description': step.description,
'duration': pretty_timedelta(step.started, step.stopped),
'failed': step.status == BuildStep.FAILURE,
'errors': step.errors,
'log': self._render_log(req, build, formatters, step),
'reports': self._render_reports(req, config, build, summarizers,
step)
})
data['build']['steps'] = steps
data['build']['can_delete'] = ('BUILD_DELETE' in req.perm)
repos = self.env.get_repository(req.authname)
repos.authz.assert_permission(config.path)
chgset = repos.get_changeset(build.rev)
data['build']['chgset_author'] = chgset.author
add_script(req, 'bitten/tabset.js')
add_stylesheet(req, 'bitten/bitten.css')
return 'bitten_build.html', data, None
def test_initiate_build(self):
config = BuildConfig(self.env, 'test', path='somepath', active=True,
recipe='<build><step id="s1"></step></build>')
config.insert()
platform = TargetPlatform(self.env, config='test', name="Unix")
platform.rules.append(('family', 'posix'))
platform.insert()
build = Build(self.env, 'test', '123', platform.id, slave='hal',
rev_time=42)
build.insert()
outheaders = {}
outbody = StringIO()
req = Mock(method='GET', base_path='',
path_info='/builds/%d' % build.id,
href=Href('/trac'), remote_addr='127.0.0.1', args={},
authname='hal',
perm=PermissionCache(self.env, 'hal'),
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
form_token="12345",
incookie=Cookie('trac_auth='))
module = BuildMaster(self.env)
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(200, outheaders['Status'])
self.assertEqual('163', outheaders['Content-Length'])
self.assertEqual('application/x-bitten+xml',
outheaders['Content-Type'])
self.assertEqual('attachment; filename=recipe_test_r123.xml',
outheaders['Content-Disposition'])
self.assertEqual('<build build="1" config="test" form_token="12345" '
'name="hal" path="somepath" platform="Unix" '
'reponame="" repopath="somepath" '
'revision="123"><step id="s1"/></build>',
outbody.getvalue())
# Make sure the started timestamp has been set
build = Build.fetch(self.env, build.id)
assert build.started
def get_annotation_data(self, context):
"""add annotation data for lint"""
context.perm.require('BUILD_VIEW')
add_stylesheet(context.req, 'bitten/bitten_coverage.css')
add_stylesheet(context.req, 'bitten/bitten_lintannotator.css')
resource = context.resource
# attempt to use the version passed in with the request,
# otherwise fall back to the latest version of this file.
try:
version = context.req.args['rev']
except (KeyError, TypeError):
version = resource.version
self.log.debug('no version passed to get_annotation_data')
builds = Build.select(self.env, rev=version)
self.log.debug("Looking for lint report for %s@%s [%s]..." % (
resource.id, str(resource.version), version))
self.itemid = 0
data = {}
reports = None
for build in builds:
config = BuildConfig.fetch(self.env, build.config)
if not resource.id.lstrip('/').startswith(config.path.lstrip('/')):
self.log.debug('Skip build %s' % build)
continue
path_in_config = resource.id[len(config.path)+1:].lstrip('/')
reports = Report.select(self.env, build=build.id, category='lint')
for report in reports:
for item in report.items:
if item.get('file') == path_in_config:
line = item.get('line')
if line:
problem = {'category': item.get('category', ''),
'tag': item.get('tag', ''),
'bid': build.id,
'rbuild': report.build,
'rstep': report.step, 'rid': report.id}
data.setdefault(int(line), []).append(problem)
if data:
self.log.debug("Lint annotate for %s@%s: %s results" % \
(resource.id, resource.version, len(data)))
return data
if not builds:
self.log.debug("No builds found")
elif not reports:
self.log.debug("No reports found")
else:
self.log.debug("No item of any report matched (%s)" % reports)
return None
def populate(self):
"""Add a build for the next change on each build configuration to the
queue.
The next change is the latest repository check-in for which there isn't
a corresponding build on each target platform. Repeatedly calling this
method will eventually result in the entire change history of the build
configuration being in the build queue.
"""
repos = self.env.get_repository()
if hasattr(repos, 'sync'):
repos.sync()
db = self.env.get_db_cnx()
builds = []
for config in BuildConfig.select(self.env, db=db):
platforms = []
for platform, rev, build in collect_changes(repos, config, db):
if not self.build_all and platform.id in platforms:
# We've seen this platform already, so these are older
# builds that should only be built if built_all=True
self.log.debug('Ignoring older revisions for configuration '
'%r on %r', config.name, platform.name)
break
platforms.append(platform.id)
if build is None:
self.log.info('Enqueuing build of configuration "%s" at '
'revision [%s] on %s', config.name, rev,
platform.name)
rev_time = repos.get_changeset(rev).date
if isinstance(rev_time, datetime): # Trac>=0.11
from trac.util.datefmt import to_timestamp
rev_time = to_timestamp(rev_time)
age = int(time.time()) - rev_time
if self.stabilize_wait and age < self.stabilize_wait:
self.log.info('Delaying build of revision %s until %s '
'seconds pass. Current age is: %s '
'seconds' % (rev, self.stabilize_wait,
age))
continue
build = Build(self.env, config=config.name,
platform=platform.id, rev=str(rev),
rev_time=rev_time)
builds.append(build)
for build in builds:
build.insert(db=db)
db.commit()
def _format_link(formatter, ns, name, label):
build = Build.fetch(self.env, int(name))
if build:
config = BuildConfig.fetch(self.env, build.config)
title = "Build %d ([%s] of %s) by %s" % (build.id, build.rev, config.label, build.slave)
return '<a class="build" href="%s" title="%s">%s</a>' % (
formatter.href.build(build.config, build.id),
title,
label,
)
return label
def test_fetch(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_config (name,path,label,active) "
"VALUES (%s,%s,%s,%s)", ('test', 'trunk', 'Test', 0))
config = BuildConfig.fetch(self.env, name='test')
assert config.exists
self.assertEqual('test', config.name)
self.assertEqual('trunk', config.path)
self.assertEqual('Test', config.label)
self.assertEqual(False, config.active)
def _activate_configs(self, req):
req.perm.assert_permission('BUILD_MODIFY')
active = req.args.get('active') or []
active = isinstance(active, list) and active or [active]
db = self.env.get_db_cnx()
for config in list(BuildConfig.select(self.env, db=db,
include_inactive=True)):
config.active = config.name in active
config.update(db=db)
db.commit()
def test_delete(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_config (name,path,label,active) "
"VALUES (%s,%s,%s,%s)", ('test', 'trunk', 'Test', 0))
config = BuildConfig.fetch(self.env, 'test')
config.delete()
self.assertEqual(False, config.exists)
cursor.execute("SELECT * FROM bitten_config WHERE name=%s", ('test',))
self.assertEqual(None, cursor.fetchone())
def test_should_delete_build_outside_revision_range(self):
messages = []
self.env.log = Mock(info=lambda msg, *args: messages.append(msg))
self.repos.rev_older_than = lambda rev1, rev2: rev1 < rev2
config = BuildConfig(self.env, 'test', active=True, min_rev=120,
max_rev=123)
config.insert()
platform = TargetPlatform(self.env, config='test', name='stuff')
platform.insert()
build1 = Build(self.env, config=config.name, rev=42,
platform=platform.id, rev_time=123456)
build1.insert()
build2 = Build(self.env, config=config.name, rev=10042,
platform=platform.id, rev_time=123456)
build2.insert()
queue = BuildQueue(self.env, build_all=True)
self.assertEqual(True, queue.should_delete_build(build1, self.repos))
self.assertEqual(True, queue.should_delete_build(build2, self.repos))
self.assert_("outside of the revision range" in messages[0])
self.assert_("outside of the revision range" in messages[1])
def test_new(self):
config = BuildConfig(self.env, name='test')
assert not config.exists
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(req.authname)
repos.authz.assert_permission(config.branch)
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(
Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))
rev = ''
for b in repos.git.get_branches():
if b[0] == config.branch:
rev = b[1]
break
data['config'] = {
'name': config.name,
'label': config.label,
'branch': config.branch,
'active': config.active,
'description': description,
'browser_href': req.href.browser(rev=rev),
'builds_pending': len(pending_builds),
'builds_inprogress': len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
platforms = list(
TargetPlatform.select(self.env, config=config_name, db=db))
data['config']['platforms'] = [{
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
} for platform in platforms]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
chart_generators.append({
'href':
req.href.build(config.name, 'chart/' + category)
})
data['config']['charts'] = chart_generators
charts_license = self.config.get('bitten', 'charts_license')
if charts_license:
data['config']['charts_license'] = charts_license
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(req.authname)
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env,
build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped, utc) - \
to_datetime(step.started, utc),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
prevnext_nav(req, 'Page')
return data
def _render_inprogress(self, req):
data = {'title': 'In Progress Builds', 'page_mode': 'view-inprogress'}
db = self.env.get_db_cnx()
repos = self.env.get_repository(req.authname)
configs = []
for config in BuildConfig.select(self.env, include_inactive=False):
if not repos.authz.has_permission(config.branch):
continue
self.log.debug(config.name)
if not config.active:
continue
in_progress_builds = Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
db=db)
current_builds = 0
builds = []
# sort correctly by revision.
for build in sorted(in_progress_builds,
cmp=lambda x, y: int(y.rev) - int(x.rev)):
rev = build.rev
build_data = _get_build_data(self.env, req, build)
build_data['rev'] = rev
build_data['rev_href'] = req.href.changeset(rev)
platform = TargetPlatform.fetch(self.env, build.platform)
build_data['platform'] = platform.name
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id, db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped, utc) - \
to_datetime(step.started, utc),
'failed': not step.successful,
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds.append(build_data)
current_builds += 1
if current_builds == 0:
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
configs.append({
'name': config.name,
'label': config.label or config.name,
'active': config.active,
'branch': config.branch,
'description': description,
'href': req.href.build(config.name),
'builds': builds
})
data['configs'] = configs
return data
def _render_overview(self, req):
data = {'title': 'Build Status'}
show_all = False
if req.args.get('show') == 'all':
show_all = True
data['show_all'] = show_all
repos = self.env.get_repository(req.authname)
configs = []
for config in BuildConfig.select(self.env, include_inactive=show_all):
if not repos.authz.has_permission(config.branch):
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
platforms_data = []
for platform in TargetPlatform.select(self.env,
config=config.name):
pd = {
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
}
platforms_data.append(pd)
config_data = {
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'branch':
config.branch,
'description':
description,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))),
'href':
req.href.build(config.name),
'builds': [],
'platforms':
platforms_data
}
configs.append(config_data)
if not config.active:
continue
prev_rev = None
for platform, rev, build in collect_changes(repos, config):
if rev != prev_rev:
if prev_rev is None:
chgset = repos.get_changeset(rev)
config_data['youngest_rev'] = {
'id':
rev,
'href':
req.href.changeset(rev),
'author':
chgset.author or 'anonymous',
'date':
format_datetime(chgset.date),
'message':
wiki_to_oneliner(shorten_line(chgset.message),
self.env,
req=req)
}
else:
break
prev_rev = rev
if build:
build_data = _get_build_data(self.env, req, build)
build_data['platform'] = platform.name
config_data['builds'].append(build_data)
else:
config_data['builds'].append({
'platform': platform.name,
'status': 'pending'
})
data['configs'] = configs
data['page_mode'] = 'overview'
in_progress_builds = Build.select(self.env, status=Build.IN_PROGRESS)
pending_builds = Build.select(self.env, status=Build.PENDING)
data['builds_pending'] = len(list(pending_builds))
data['builds_inprogress'] = len(list(in_progress_builds))
add_link(req, 'views', req.href.build(view='inprogress'),
'In Progress Builds')
add_ctxtnav(req, 'In Progress Builds',
req.href.build(view='inprogress'))
return data
class CollectChangesTestCase(unittest.TestCase):
"""
Unit tests for the `bitten.queue.collect_changes` function.
"""
def setUp(self):
self.env = EnvironmentStub(enable=['trac.*', 'bitten.*'])
self.env.path = tempfile.mkdtemp()
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
self.config = BuildConfig(self.env, name='test', path='somepath')
self.config.insert(db=db)
self.platform = TargetPlatform(self.env, config='test', name='Foo')
self.platform.insert(db=db)
db.commit()
# Hook up a dummy repository
self.repos = Mock()
self.env.get_repository = lambda authname=None: self.repos # 0.11
try: # 0.12+
from trac.core import Component, implements
from trac.versioncontrol.api import IRepositoryConnector, \
IRepositoryProvider
class DummyRepos(Component):
implements(IRepositoryConnector, IRepositoryProvider)
def get_supported_types(self):
yield ('dummy', 9)
def get_repository(this, repos_type, repos_dir, params):
return self.repos # Note: 'this' vs 'self' usage
def get_repositories(self):
yield ('', {'dir': 'dummy_dir', 'type': 'dummy'})
self.dummy = DummyRepos
except ImportError:
self.dummy = None # not supported, will use get_repository()
def tearDown(self):
if self.dummy: # remove from components list + interfaces dict
self.env.__metaclass__._components.remove(self.dummy)
for key in self.env.__metaclass__._registry.keys():
if self.dummy in self.env.__metaclass__._registry[key]:
self.env.__metaclass__._registry[key].remove(self.dummy)
shutil.rmtree(self.env.path)
def test_stop_on_copy(self):
self.repos.get_node=lambda path, rev=None: Mock(
get_history=lambda: [('otherpath', 123, 'copy')])
self.repos.normalize_path=lambda path: path
retval = list(collect_changes(self.config))
self.assertEqual(0, len(retval))
def test_stop_on_minrev(self):
self.repos.get_node=lambda path, rev=None: Mock(
get_entries=lambda: [Mock(), Mock()],
get_history=lambda: [('somepath', 123, 'edit'),
('somepath', 121, 'edit'),
('somepath', 120, 'edit')])
self.repos.normalize_path=lambda path: path
self.repos.rev_older_than=lambda rev1, rev2: rev1 < rev2
self.config.min_rev = 123
self.config.update()
retval = list(collect_changes(self.config))
self.assertEqual(1, len(retval))
self.assertEqual(123, retval[0][1])
def test_skip_until_maxrev(self):
self.repos.get_node=lambda path, rev=None: Mock(
get_entries=lambda: [Mock(), Mock()],
get_history=lambda: [('somepath', 123, 'edit'),
('somepath', 121, 'edit'),
('somepath', 120, 'edit')])
self.repos.normalize_path=lambda path: path
self.repos.rev_older_than=lambda rev1, rev2: rev1 < rev2
self.config.max_rev=121
self.config.update()
retval = list(collect_changes(self.config))
self.assertEqual(2, len(retval))
self.assertEqual(121, retval[0][1])
self.assertEqual(120, retval[1][1])
def test_skip_empty_dir(self):
def _mock_get_node(path, rev=None):
if rev and rev == 121:
return Mock(
get_entries=lambda: []
)
else:
return Mock(
get_entries=lambda: [Mock(), Mock()],
get_history=lambda: [('somepath', 123, 'edit'),
('somepath', 121, 'edit'),
('somepath', 120, 'edit')]
)
self.repos.get_node=_mock_get_node
self.repos.normalize_path=lambda path: path
self.repos.rev_older_than=lambda rev1, rev2: rev1 < rev2
retval = list(collect_changes(self.config))
self.assertEqual(2, len(retval))
self.assertEqual(123, retval[0][1])
self.assertEqual(120, retval[1][1])
def test_process_build_step_success_with_log(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
logs = list(BuildLog.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(logs))
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
logs[0].generator)
self.assertEqual(2, len(logs[0].messages))
self.assertEqual((u'info', u'Doing stuff'), logs[0].messages[0])
self.assertEqual((u'error', u'Ouch that hurt'), logs[0].messages[1])
def test_insert_no_name(self):
config = BuildConfig(self.env)
self.assertRaises(AssertionError, config.insert)
def _render_config(self, req, config_name):
db = self.env.get_db_cnx()
config = BuildConfig.fetch(self.env, config_name, db=db)
if not config:
raise HTTPNotFound("Build configuration '%s' does not exist." \
% config_name)
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
rev = config.max_rev or repos.youngest_rev
try:
_has_permission(req.perm,
repos,
config.path,
rev=rev,
raise_error=True)
except NoSuchNode:
raise TracError("Permission checking against repository path %s "
"at revision %s failed." % (config.path, rev))
data = {'title': 'Build Configuration "%s"' \
% config.label or config.name,
'page_mode': 'view_config'}
add_link(req, 'up', req.href.build(), 'Build Status')
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
pending_builds = list(
Build.select(self.env, config=config.name, status=Build.PENDING))
inprogress_builds = list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))
data['config'] = {
'name': config.name,
'label': config.label,
'path': config.path,
'min_rev': config.min_rev,
'min_rev_href': req.href.changeset(config.min_rev),
'max_rev': config.max_rev,
'max_rev_href': req.href.changeset(config.max_rev),
'active': config.active,
'description': description,
'browser_href': req.href.browser(config.path),
'builds_pending': len(pending_builds),
'builds_inprogress': len(inprogress_builds)
}
context = Context.from_request(req, config.resource)
data['context'] = context
data['config']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
platforms = list(
TargetPlatform.select(self.env, config=config_name, db=db))
data['config']['platforms'] = [{
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
} for platform in platforms]
has_reports = False
for report in Report.select(self.env, config=config.name, db=db):
has_reports = True
break
if has_reports:
chart_generators = []
report_categories = list(
self._report_categories_for_config(config))
for generator in ReportChartController(self.env).generators:
for category in generator.get_supported_categories():
if category in report_categories:
chart_generators.append({
'href':
req.href.build(config.name, 'chart/' + category),
'category':
category,
'style':
self.config.get('bitten', 'chart_style'),
})
data['config']['charts'] = chart_generators
page = max(1, int(req.args.get('page', 1)))
more = False
data['page_number'] = page
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
builds_per_page = 12 * len(platforms)
idx = 0
builds = {}
revisions = []
for platform, rev, build in collect_changes(repos, config):
if idx >= page * builds_per_page:
more = True
break
elif idx >= (page - 1) * builds_per_page:
if rev not in builds:
revisions.append(rev)
builds.setdefault(rev, {})
builds[rev].setdefault('href', req.href.changeset(rev))
builds[rev].setdefault('display_rev', repos.normalize_rev(rev))
if build and build.status != Build.PENDING:
build_data = _get_build_data(self.env, req, build)
build_data['steps'] = []
for step in BuildStep.select(self.env,
build=build.id,
db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds[rev][platform.id] = build_data
idx += 1
data['config']['builds'] = builds
data['config']['revisions'] = revisions
if page > 1:
if page == 2:
prev_href = req.href.build(config.name)
else:
prev_href = req.href.build(config.name, page=page - 1)
add_link(req, 'prev', prev_href, 'Previous Page')
if more:
next_href = req.href.build(config.name, page=page + 1)
add_link(req, 'next', next_href, 'Next Page')
if arity(prevnext_nav) == 4: # Trac 0.12 compat, see #450
prevnext_nav(req, 'Previous Page', 'Next Page')
else:
prevnext_nav(req, 'Page')
return data
def test_process_attach_build(self):
body, content_type = encode_multipart_formdata({
'description':
'baz baz',
'file': ('baz.txt', 'hello baz'),
'__FORM_TOKEN':
'123456'
})
args = {}
for k, v in dict(
cgi.FieldStorage(fp=StringIO(body),
environ={
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': content_type
})).items():
if v.filename:
args[k] = v
else:
args[k] = v.value
args.update({'collection': 'attach', 'member': 'build'})
self.assertTrue('file' in args)
outheaders = {}
outbody = StringIO()
req = Mock(args=args,
form_token='123456',
authname='hal',
remote_addr='127.0.0.1',
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write)
config = BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe='')
config.insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.insert()
module = BuildMaster(self.env)
self.assertRaises(RequestDone, module._process_attachment, req, config,
build)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('18', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Attachment created', outbody.getvalue())
build_atts = list(Attachment.select(self.env, 'build', 'test/1'))
self.assertEquals(1, len(build_atts))
self.assertEquals('hal', build_atts[0].author)
self.assertEquals('baz baz', build_atts[0].description)
self.assertEquals('baz.txt', build_atts[0].filename)
self.assertEquals('hello baz', build_atts[0].open().read())
def test_delete_non_existing(self):
config = BuildConfig(self.env, 'test')
self.assertRaises(AssertionError, config.delete)
def process_request(self, req):
req.perm.require('BUILD_VIEW')
db = self.env.get_db_cnx()
build_id = int(req.args.get('id'))
build = Build.fetch(self.env, build_id, db=db)
if not build:
raise HTTPNotFound("Build '%s' does not exist." \
% build_id)
if req.method == 'POST':
if req.args.get('action') == 'invalidate':
self._do_invalidate(req, build, db)
req.redirect(req.href.build(build.config, build.id))
add_link(req, 'up', req.href.build(build.config),
'Build Configuration')
data = {
'title': 'Build %s - %s' % (build_id, _status_title[build.status]),
'page_mode': 'view_build',
'build': {}
}
config = BuildConfig.fetch(self.env, build.config, db=db)
data['build']['config'] = {
'name': config.label or config.name,
'href': req.href.build(config.name)
}
context = Context.from_request(req, build.resource)
data['context'] = context
data['build']['attachments'] = AttachmentModule(
self.env).attachment_data(context)
formatters = []
for formatter in self.log_formatters:
formatters.append(formatter.get_formatter(req, build))
summarizers = {} # keyed by report type
for summarizer in self.report_summarizers:
categories = summarizer.get_supported_categories()
summarizers.update(dict([(cat, summarizer) for cat in categories]))
data['build'].update(_get_build_data(self.env, req, build))
steps = []
for step in BuildStep.select(self.env, build=build.id, db=db):
steps.append({
'name':
step.name,
'description':
step.description,
'duration':
pretty_timedelta(step.started, step.stopped
or int(time.time())),
'status':
_step_status_label[step.status],
'cls':
_step_status_label[step.status].replace(' ', '-'),
'errors':
step.errors,
'log':
self._render_log(req, build, formatters, step),
'reports':
self._render_reports(req, config, build, summarizers, step)
})
data['build']['steps'] = steps
data['build']['can_delete'] = ('BUILD_DELETE' in req.perm \
and build.status != build.PENDING)
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
_has_permission(req.perm,
repos,
config.path,
rev=build.rev,
raise_error=True)
chgset = repos.get_changeset(build.rev)
data['build']['chgset_author'] = chgset.author
data['build']['display_rev'] = repos.normalize_rev(build.rev)
add_script(req, 'common/js/folding.js')
add_script(req, 'bitten/tabset.js')
add_script(req, 'bitten/jquery.flot.js')
add_stylesheet(req, 'bitten/bitten.css')
return 'bitten_build.html', data, None
def render_admin_panel(self, req, cat, page, path_info):
data = {}
# Analyze url
try:
config_name, platform_id = path_info.split('/', 1)
except:
config_name = path_info
platform_id = None
if config_name: # Existing build config
warnings = []
if platform_id or (
# Editing or creating one of the config's target platforms
req.method == 'POST' and 'new' in req.args):
if platform_id: # Editing target platform
platform_id = int(platform_id)
platform = TargetPlatform.fetch(self.env, platform_id)
if req.method == 'POST':
if 'cancel' in req.args or \
self._update_platform(req, platform):
req.redirect(
req.abs_href.admin(cat, page, config_name))
else: # creating target platform
platform = self._create_platform(req, config_name)
req.redirect(
req.abs_href.admin(cat, page, config_name,
platform.id))
# Set up template variables
data['platform'] = {
'id':
platform.id,
'name':
platform.name,
'exists':
platform.exists,
'rules': [{
'property': propname,
'pattern': pattern
} for propname, pattern in platform.rules] or [('', '')]
}
else: # Editing existing build config itself
config = BuildConfig.fetch(self.env, config_name)
platforms = list(
TargetPlatform.select(self.env, config=config.name))
if req.method == 'POST':
if 'remove' in req.args: # Remove selected platforms
self._remove_platforms(req)
add_notice(req, "Target Platform(s) Removed.")
req.redirect(req.abs_href.admin(
cat, page, config.name))
elif 'save' in req.args: # Save this build config
warnings = self._update_config(req, config)
if not warnings:
add_notice(req, "Configuration Saved.")
req.redirect(req.abs_href.admin(
cat, page, config.name))
for warning in warnings:
add_warning(req, warning)
# FIXME: Deprecation notice for old namespace.
# Remove notice code when migration to new namespace is complete
if 'http://bitten.cmlenz.net/tools/' in config.recipe:
add_notice(
req, "Recipe uses a deprecated namespace. "
"Replace 'http://bitten.cmlenz.net/tools/' with "
"'http://bitten.edgewall.org/tools/'.")
# Add a notice if configuration is not active
if not warnings and not config.active and config.recipe:
add_notice(
req, "Configuration is not active. Activate "
"from main 'Configurations' listing to enable it.")
# Prepare template variables
data['config'] = {
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'branch':
config.branch,
'description':
config.description,
'recipe':
config.recipe,
'platforms': [{
'name':
platform.name,
'id':
platform.id,
'href':
req.href.admin('bitten', 'configs', config.name,
platform.id),
'rules': [{
'property': propname,
'pattern': pattern
} for propname, pattern in platform.rules]
} for platform in platforms]
}
else: # At the top level build config list
if req.method == 'POST':
if 'add' in req.args: # Add build config
config = self._create_config(req)
req.redirect(req.abs_href.admin(cat, page, config.name))
elif 'remove' in req.args: # Remove selected build configs
self._remove_configs(req)
elif 'apply' in req.args: # Update active state of configs
self._activate_configs(req)
req.redirect(req.abs_href.admin(cat, page))
# Prepare template variables
configs = []
for config in BuildConfig.select(self.env, include_inactive=True):
configs.append({
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'branch':
config.branch,
'href':
req.href.admin('bitten', 'configs', config.name),
'recipe':
config.recipe and True or False
})
data['configs'] = configs
add_stylesheet(req, 'bitten/admin.css')
add_script(req, 'common/js/suggest.js')
return 'bitten_admin_configs.html', data
def _render_inprogress(self, req):
data = {'title': 'In Progress Builds', 'page_mode': 'view-inprogress'}
db = self.env.get_db_cnx()
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
configs = []
for config in BuildConfig.select(self.env, include_inactive=False):
rev = config.max_rev or repos.youngest_rev
try:
if not _has_permission(req.perm, repos, config.path, rev=rev):
continue
except NoSuchNode:
add_warning(req, "Configuration '%s' points to non-existing "
"path '/%s' at revision '%s'. Configuration skipped." \
% (config.name, config.path, rev))
continue
self.log.debug(config.name)
if not config.active:
continue
in_progress_builds = Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
db=db)
current_builds = 0
builds = []
# sort correctly by revision.
for build in sorted(
in_progress_builds,
cmp=lambda x, y: int(y.rev_time) - int(x.rev_time)):
rev = build.rev
build_data = _get_build_data(self.env, req, build)
build_data['rev'] = rev
build_data['rev_href'] = req.href.changeset(rev)
platform = TargetPlatform.fetch(self.env, build.platform)
build_data['platform'] = platform.name
build_data['steps'] = []
for step in BuildStep.select(self.env, build=build.id, db=db):
build_data['steps'].append({
'name': step.name,
'description': step.description,
'duration': to_datetime(step.stopped or int(time.time()), utc) - \
to_datetime(step.started, utc),
'status': _step_status_label[step.status],
'cls': _step_status_label[step.status].replace(' ', '-'),
'errors': step.errors,
'href': build_data['href'] + '#step_' + step.name
})
builds.append(build_data)
current_builds += 1
if current_builds == 0:
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
configs.append({
'name': config.name,
'label': config.label or config.name,
'active': config.active,
'path': config.path,
'description': description,
'href': req.href.build(config.name),
'builds': builds
})
data['configs'] = sorted(configs, key=lambda x: x['label'].lower())
return data
def _render_overview(self, req):
data = {'title': 'Build Status'}
show_all = False
if req.args.get('show') == 'all':
show_all = True
data['show_all'] = show_all
repos = self.env.get_repository(authname=req.authname)
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
configs = []
for config in BuildConfig.select(self.env, include_inactive=show_all):
rev = config.max_rev or repos.youngest_rev
try:
if not _has_permission(req.perm, repos, config.path, rev=rev):
continue
except NoSuchNode:
add_warning(req, "Configuration '%s' points to non-existing "
"path '/%s' at revision '%s'. Configuration skipped." \
% (config.name, config.path, rev))
continue
description = config.description
if description:
description = wiki_to_html(description, self.env, req)
platforms_data = []
for platform in TargetPlatform.select(self.env,
config=config.name):
pd = {
'name':
platform.name,
'id':
platform.id,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING,
platform=platform.id))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS,
platform=platform.id)))
}
platforms_data.append(pd)
config_data = {
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'path':
config.path,
'description':
description,
'builds_pending':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.PENDING))),
'builds_inprogress':
len(
list(
Build.select(self.env,
config=config.name,
status=Build.IN_PROGRESS))),
'href':
req.href.build(config.name),
'builds': [],
'platforms':
platforms_data
}
configs.append(config_data)
if not config.active:
continue
prev_rev = None
for platform, rev, build in collect_changes(repos, config):
if rev != prev_rev:
if prev_rev is None:
chgset = repos.get_changeset(rev)
config_data['youngest_rev'] = {
'id':
rev,
'href':
req.href.changeset(rev),
'display_rev':
repos.normalize_rev(rev),
'author':
chgset.author or 'anonymous',
'date':
format_datetime(chgset.date),
'message':
wiki_to_oneliner(shorten_line(chgset.message),
self.env,
req=req)
}
else:
break
prev_rev = rev
if build:
build_data = _get_build_data(self.env, req, build)
build_data['platform'] = platform.name
config_data['builds'].append(build_data)
else:
config_data['builds'].append({
'platform': platform.name,
'status': 'pending'
})
data['configs'] = sorted(configs, key=lambda x: x['label'].lower())
data['page_mode'] = 'overview'
in_progress_builds = Build.select(self.env, status=Build.IN_PROGRESS)
pending_builds = Build.select(self.env, status=Build.PENDING)
data['builds_pending'] = len(list(pending_builds))
data['builds_inprogress'] = len(list(in_progress_builds))
add_link(req, 'views', req.href.build(view='inprogress'),
'In Progress Builds')
add_ctxtnav(req, 'In Progress Builds',
req.href.build(view='inprogress'))
return data
def test_fetch_none(self):
config = BuildConfig.fetch(self.env, name='test')
self.assertEqual(None, config)
def test_select_none(self):
configs = BuildConfig.select(self.env)
self.assertRaises(StopIteration, configs.next)
def populate(self):
"""Add a build for the next change on each build configuration to the
queue.
The next change is the latest repository check-in for which there isn't
a corresponding build on each target platform. Repeatedly calling this
method will eventually result in the entire change history of the build
configuration being in the build queue.
"""
repos = self.env.get_repository()
assert repos, 'No "(default)" Repository: Add a repository or alias ' \
'named "(default)" to Trac.'
db = self.env.get_db_cnx()
builds = []
for config in BuildConfig.select(self.env, db=db):
platforms = []
for platform, rev, build in collect_changes(repos, config, db):
if not self.build_all and platform.id in platforms:
# We've seen this platform already, so these are older
# builds that should only be built if built_all=True
self.log.debug(
'Ignoring older revisions for configuration '
'%r on %r', config.name, platform.name)
break
platforms.append(platform.id)
if build is None:
self.log.info(
'Enqueuing build of configuration "%s" at '
'revision [%s] on %s', config.name, rev, platform.name)
rev_time = to_timestamp(repos.get_changeset(rev).date)
age = int(time.time()) - rev_time
if self.stabilize_wait and age < self.stabilize_wait:
self.log.info('Delaying build of revision %s until %s '
'seconds pass. Current age is: %s '
'seconds' %
(rev, self.stabilize_wait, age))
continue
build = Build(self.env,
config=config.name,
platform=platform.id,
rev=str(rev),
rev_time=rev_time)
builds.append(build)
for build in builds:
try:
build.insert(db=db)
db.commit()
except Exception, e:
# really only want to catch IntegrityErrors raised when
# a second slave attempts to add builds with the same
# (config, platform, rev) as an existing build.
self.log.info(
'Failed to insert build of configuration "%s" '
'at revision [%s] on platform [%s]: %s', build.config,
build.rev, build.platform, e)
db.rollback()
def test_process_build_step_success_with_report(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<report category="test"
generator="http://bitten.edgewall.org/tools/python#unittest">
<test fixture="my.Fixture" file="my/test/file.py">
<stdout>Doing my thing</stdout>
</test>
</report>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
reports = list(Report.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(reports))
self.assertEqual('test', reports[0].category)
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
reports[0].generator)
self.assertEqual(1, len(reports[0].items))
self.assertEqual(
{
'fixture': 'my.Fixture',
'file': 'my/test/file.py',
'stdout': 'Doing my thing',
'type': 'test',
}, reports[0].items[0])
def test_process_build_step_invalidated_build(self):
recipe = """<build>
<step id="foo">
</step>
<step id="foo2">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.IN_PROGRESS, build.status)
assert not build.stopped
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(2, len(steps))
# invalidate the build.
build = Build.fetch(self.env, build.id)
build.slave = None
build.status = Build.PENDING
build.slave_info = {}
for step in list(BuildStep.select(self.env, build=build.id)):
step.delete()
build.update()
# have this slave submit more data.
inbody = StringIO("""<result step="foo2" status="success"
time="2007-04-01T15:45:00.0000"
duration="4">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">This is a step after invalidation</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEquals(409, outheaders['Status'])
self.assertEquals('Token mismatch (wrong slave): slave=123, build=',
outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.PENDING, build.status)
class CollectChangesTestCase(unittest.TestCase):
"""
Unit tests for the `bitten.queue.collect_changes` function.
"""
def setUp(self):
self.env = EnvironmentStub()
self.env.path = tempfile.mkdtemp()
db = self.env.get_db_cnx()
cursor = db.cursor()
connector, _ = DatabaseManager(self.env)._get_connector()
for table in schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
self.config = BuildConfig(self.env, name='test', path='somepath')
self.config.insert(db=db)
self.platform = TargetPlatform(self.env, config='test', name='Foo')
self.platform.insert(db=db)
db.commit()
def tearDown(self):
shutil.rmtree(self.env.path)
def test_stop_on_copy(self):
self.env.get_repository = lambda authname=None: Mock(
get_node=lambda path, rev=None: Mock(get_history=lambda: [(
'otherpath', 123, 'copy')]),
normalize_path=lambda path: path)
retval = list(collect_changes(self.env.get_repository(), self.config))
self.assertEqual(0, len(retval))
def test_stop_on_minrev(self):
self.env.get_repository = lambda authname=None: Mock(
get_node=lambda path, rev=None: Mock(
get_entries=lambda: [Mock(), Mock()],
get_history=lambda: [('somepath', 123, 'edit'),
('somepath', 121, 'edit'),
('somepath', 120, 'edit')]),
normalize_path=lambda path: path,
rev_older_than=lambda rev1, rev2: rev1 < rev2)
self.config.min_rev = 123
self.config.update()
retval = list(collect_changes(self.env.get_repository(), self.config))
self.assertEqual(1, len(retval))
self.assertEqual(123, retval[0][1])
def test_skip_until_maxrev(self):
self.env.get_repository = lambda authname=None: Mock(
get_node=lambda path, rev=None: Mock(
get_entries=lambda: [Mock(), Mock()],
get_history=lambda: [('somepath', 123, 'edit'),
('somepath', 121, 'edit'),
('somepath', 120, 'edit')]),
normalize_path=lambda path: path,
rev_older_than=lambda rev1, rev2: rev1 < rev2)
self.config.max_rev = 121
self.config.update()
retval = list(collect_changes(self.env.get_repository(), self.config))
self.assertEqual(2, len(retval))
self.assertEqual(121, retval[0][1])
self.assertEqual(120, retval[1][1])
def test_skip_empty_dir(self):
def _mock_get_node(path, rev=None):
if rev and rev == 121:
return Mock(get_entries=lambda: [])
else:
return Mock(get_entries=lambda: [Mock(), Mock()],
get_history=lambda: [('somepath', 123, 'edit'),
('somepath', 121, 'edit'),
('somepath', 120, 'edit')])
self.env.get_repository = lambda authname=None: Mock(
get_node=_mock_get_node,
normalize_path=lambda path: path,
rev_older_than=lambda rev1, rev2: rev1 < rev2)
retval = list(collect_changes(self.env.get_repository(), self.config))
self.assertEqual(2, len(retval))
self.assertEqual(123, retval[0][1])
self.assertEqual(120, retval[1][1])
def test_process_build_step_failure_continue(self):
recipe = """<build>
<step id="foo" onerror="continue">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="failure"
time="2007-04-01T15:30:00.0000"
duration="3.45">
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.FAILURE, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.FAILURE, steps[0].status)
def get_change(self):
config_path = BuildConfig.fetch(self.env, name=self.build.config).path
reposname, repos, _path = get_repos(self.env, config_path, None)
return reposname, repos, repos.get_changeset(self.build.rev)
