def populate(self): """Add a build for the next change on each build configuration to the queue. The next change is the latest repository check-in for which there isn't a corresponding build on each target platform. Repeatedly calling this method will eventually result in the entire change history of the build configuration being in the build queue. """ db = self.env.get_db_cnx() builds = [] for config in BuildConfig.select(self.env, db=db): platforms = [] for platform, rev, build in collect_changes(config, db=db): if not self.build_all and platform.id in platforms: # We've seen this platform already, so these are older # builds that should only be built if built_all=True self.log.debug('Ignoring older revisions for configuration ' '%r on %r', config.name, platform.name) break platforms.append(platform.id) if build is None: self.log.info('Enqueuing build of configuration "%s" at ' 'revision [%s] on %s', config.name, rev, platform.name) _repos_name, repos, _repos_path = get_repos( self.env, config.path, None) rev_time = to_timestamp(repos.get_changeset(rev).date) age = int(time.time()) - rev_time if self.stabilize_wait and age < self.stabilize_wait: self.log.info('Delaying build of revision %s until %s ' 'seconds pass. Current age is: %s ' 'seconds' % (rev, self.stabilize_wait, age)) continue build = Build(self.env, config=config.name, platform=platform.id, rev=str(rev), rev_time=rev_time) builds.append(build) for build in builds: try: build.insert(db=db) db.commit() except Exception, e: # really only want to catch IntegrityErrors raised when # a second slave attempts to add builds with the same # (config, platform, rev) as an existing build. self.log.info('Failed to insert build of configuration "%s" ' 'at revision [%s] on platform [%s]: %s', build.config, build.rev, build.platform, e) db.rollback()
def get_annotation_data(self, context): add_stylesheet(context.req, 'bitten/bitten_coverage.css') resource = context.resource # attempt to use the version passed in with the request, # otherwise fall back to the latest version of this file. version = context.req.args.get('rev') or resource.version # get the last change revision for the file so that we can # pick coverage data as latest(version >= file_revision) created = context.req.args.get('created') or resource.version full_path = get_resource_path(resource) _name, repos, _path = get_repos(self.env, full_path, None) version_time = to_timestamp(repos.get_changeset(version).date) if version != created: created_time = to_timestamp(repos.get_changeset(created).date) else: created_time = version_time self.log.debug("Looking for coverage report for %s@%s [%s:%s]..." % ( full_path, str(resource.version), created, version)) db = self.env.get_db_cnx() cursor = db.cursor() cursor.execute(""" SELECT b.id, b.rev, i2.value FROM bitten_config AS c INNER JOIN bitten_build AS b ON c.name=b.config INNER JOIN bitten_report AS r ON b.id=r.build INNER JOIN bitten_report_item AS i1 ON r.id=i1.report INNER JOIN bitten_report_item AS i2 ON (i1.item=i2.item AND i1.report=i2.report) WHERE i2.name='line_hits' AND b.rev_time>=%s AND b.rev_time<=%s AND i1.name='file' AND """ + db.concat('c.path', "'/'", 'i1.value') + """=%s ORDER BY b.rev_time DESC LIMIT 1""" , (created_time, version_time, full_path)) row = cursor.fetchone() if row: build_id, build_rev, line_hits = row coverage = line_hits.split() self.log.debug("Coverage annotate for %s@%s using build %d: %s", resource.id, build_rev, build_id, coverage) return coverage add_warning(context.req, "No coverage annotation found for " "/%s for revision range [%s:%s]." % ( resource.id.lstrip('/'), version, created)) return []
def max_rev_time(self, env): """Returns the time of the maximum revision being built for this configuration. Returns utcmax if not specified. """ _name, repos, _path = get_repos(env, self.path, None) max_time = utcmax if self.max_rev: max_time = repos.get_changeset(self.max_rev).date if isinstance(max_time, datetime): # Trac>=0.11 max_time = to_timestamp(max_time) return max_time
def get_formatter(self, req, build): """Return the log message formatter function.""" config = BuildConfig.fetch(self.env, name=build.config) repos_name, repos, repos_path = get_repos(self.env, config.path, req.authname) href = req.href.browser cache = {} def _replace(m): filepath = posixpath.normpath(m.group('path').replace('\\', '/')) if not cache.get(filepath) is True: parts = filepath.split('/') path = '' for part in parts: path = posixpath.join(path, part) if path not in cache: try: full_path = posixpath.join(config.path, path) full_path = posixpath.normpath(full_path) if full_path.startswith(config.path + "/") \ or full_path == config.path: repos.get_node(full_path, build.rev) cache[path] = True else: cache[path] = False except TracError: cache[path] = False if cache[path] is False: return m.group(0) link = href(config.path, filepath) if m.group('line'): link += '#L' + m.group('line')[1:] return Markup(tag.a(m.group(0), href=link)) def _formatter(step, type, level, message): buf = [] offset = 0 for mo in self._fileref_re.finditer(message): start, end = mo.span() if start > offset: buf.append(message[offset:start]) buf.append(_replace(mo)) offset = end if offset < len(message): buf.append(message[offset:]) return Markup("").join(buf) return _formatter
def get_build_for_slave(self, name, properties): """Check whether one of the pending builds can be built by the build slave. :param name: the name of the slave :type name: `basestring` :param properties: the slave configuration :type properties: `dict` :return: the allocated build, or `None` if no build was found :rtype: `Build` """ self.log.debug('Checking for pending builds...') db = self.env.get_db_cnx() self.reset_orphaned_builds() # Iterate through pending builds by descending revision timestamp, to # avoid the first configuration/platform getting all the builds platforms = [p.id for p in self.match_slave(name, properties)] builds_to_delete = [] build_found = False for build in Build.select(self.env, status=Build.PENDING, db=db): config_path = BuildConfig.fetch(self.env, name=build.config).path _name, repos, _path = get_repos(self.env, config_path, None) if self.should_delete_build(build, repos): self.log.info('Scheduling build %d for deletion', build.id) builds_to_delete.append(build) elif build.platform in platforms: build_found = True break if not build_found: self.log.debug('No pending builds.') build = None # delete any obsolete builds for build_to_delete in builds_to_delete: build_to_delete.delete(db=db) if build: build.slave = name build.slave_info.update(properties) build.status = Build.IN_PROGRESS build.update(db=db) if build or builds_to_delete: db.commit() return build
def _process_build_initiation(self, req, config, build): self.log.info('Build slave %r initiated build %d', build.slave, build.id) build.started = int(time.time()) build.last_activity = build.started build.update() for listener in BuildSystem(self.env).listeners: listener.build_started(build) repos_name, repos, repos_path = get_repos(self.env, config.path, req.authname) xml = xmlio.parse(config.recipe) xml.attr['path'] = config.path xml.attr['revision'] = build.rev xml.attr['config'] = config.name xml.attr['build'] = str(build.id) target_platform = TargetPlatform.fetch(self.env, build.platform) xml.attr['platform'] = target_platform.name xml.attr['name'] = build.slave xml.attr['form_token'] = req.form_token # For posting attachments xml.attr['reponame'] = repos_name != '(default)' and repos_name or '' xml.attr['repopath'] = repos_path.strip('/') body = str(xml) self.log.info('Build slave %r initiated build %d', build.slave, build.id) # create the first step, mark it as in-progress. recipe = Recipe(xmlio.parse(config.recipe)) stepname = recipe.__iter__().next().id step = self._start_new_step(build, stepname) step.insert() self._send_response(req, 200, body, headers={ 'Content-Type': 'application/x-bitten+xml', 'Content-Length': str(len(body)), 'Content-Disposition': 'attachment; filename=recipe_%s_r%s.xml' % (config.name, build.rev) })
def get_timeline_events(self, req, start, stop, filters): if 'build' not in filters: return # Attachments (will be rendered by attachment module) for event in AttachmentModule(self.env).get_timeline_events( req, Resource('build'), start, stop): yield event start = to_timestamp(start) stop = to_timestamp(stop) add_stylesheet(req, 'bitten/bitten.css') db = self.env.get_db_cnx() cursor = db.cursor() cursor.execute("SELECT b.id,b.config,c.label,c.path, b.rev,p.name," "b.stopped,b.status FROM bitten_build AS b" " INNER JOIN bitten_config AS c ON (c.name=b.config) " " INNER JOIN bitten_platform AS p ON (p.id=b.platform) " "WHERE b.stopped>=%s AND b.stopped<=%s " "AND b.status IN (%s, %s) ORDER BY b.stopped", (start, stop, Build.SUCCESS, Build.FAILURE)) event_kinds = {Build.SUCCESS: 'successbuild', Build.FAILURE: 'failedbuild'} for id_, config, label, path, rev, platform, stopped, status in cursor: config_object = BuildConfig.fetch(self.env, config, db=db) repos_name, repos, repos_path = get_repos(self.env, config_object.path, req.authname) if not _has_permission(req.perm, repos, repos_path, rev=rev): continue errors = [] if status == Build.FAILURE: for step in BuildStep.select(self.env, build=id_, status=BuildStep.FAILURE, db=db): errors += [(step.name, error) for error in step.errors] yield (event_kinds[status], to_datetime(stopped, utc), None, (id_, config, label, display_rev(repos, rev), platform, status, errors))
def _update_config(self, req, config): warnings = [] req.perm.assert_permission('BUILD_MODIFY') name = req.args.get('name') if not name: warnings.append('Missing required field "name".') if name and not re.match(r'^[\w.-]+$', name): warnings.append('The field "name" may only contain letters, ' 'digits, periods, or dashes.') path = req.args.get('path', '') repos_name, repos, repos_path = get_repos(self.env, path, req.authname) max_rev = req.args.get('max_rev') or None min_rev = req.args.get('min_rev') or None try: node = repos.get_node(repos_path, max_rev) assert node.isdir, '%s is not a directory' % node.path except (AssertionError, TracError), e: warnings.append('Invalid Repository Path "%s".' % path)
def collect_changes(config, authname=None, db=None): """Collect all changes for a build configuration that either have already been built, or still need to be built. This function is a generator that yields ``(platform, rev, build)`` tuples, where ``platform`` is a `TargetPlatform` object, ``rev`` is the identifier of the changeset, and ``build`` is a `Build` object or `None`. :param config: the build configuration :param authname: the logged in user :param db: a database connection (optional) """ env = config.env repos_name, repos, repos_path = get_repos(env, config.path, authname) if not db: db = env.get_db_cnx() try: node = repos.get_node(repos_path) except Exception, e: env.log.warn('Error accessing path %r for configuration %r', repos_path, config.name, exc_info=True) return
def get_change(self): config_path = BuildConfig.fetch(self.env, name=self.build.config).path reposname, repos, _path = get_repos(self.env, config_path, None) return reposname, repos, repos.get_changeset(self.build.rev)
def _render_inprogress(self, req): data = {'title': 'In Progress Builds', 'page_mode': 'view-inprogress'} db = self.env.get_db_cnx() configs = [] for config in BuildConfig.select(self.env, include_inactive=False): repos_name, repos, repos_path = get_repos(self.env, config.path, req.authname) rev = config.max_rev or repos.youngest_rev try: if not _has_permission(req.perm, repos, repos_path, rev=rev): continue except NoSuchNode: add_warning(req, "Configuration '%s' points to non-existing " "path '/%s' at revision '%s'. Configuration skipped." \ % (config.name, config.path, rev)) continue self.log.debug(config.name) if not config.active: continue in_progress_builds = Build.select(self.env, config=config.name, status=Build.IN_PROGRESS, db=db) current_builds = 0 builds = [] # sort correctly by revision. for build in sorted(in_progress_builds, cmp=lambda x, y: int(y.rev_time) - int(x.rev_time)): rev = build.rev build_data = _get_build_data(self.env, req, build, repos_name) build_data['rev'] = rev build_data['rev_href'] = build_data['chgset_href'] platform = TargetPlatform.fetch(self.env, build.platform) build_data['platform'] = platform.name build_data['steps'] = [] for step in BuildStep.select(self.env, build=build.id, db=db): build_data['steps'].append({ 'name': step.name, 'description': step.description, 'duration': to_datetime(step.stopped or int(time.time()), utc) - \ to_datetime(step.started, utc), 'status': _step_status_label[step.status], 'cls': _step_status_label[step.status].replace(' ', '-'), 'errors': step.errors, 'href': build_data['href'] + '#step_' + step.name }) builds.append(build_data) current_builds += 1 if current_builds == 0: continue description = config.description if description: description = wiki_to_html(description, self.env, req) configs.append({ 'name': config.name, 'label': config.label or config.name, 'active': config.active, 'path': config.path, 'description': description, 'href': req.href.build(config.name), 'builds': builds }) data['configs'] = sorted(configs, key=lambda x:x['label'].lower()) return data
def _render_config(self, req, config_name): db = self.env.get_db_cnx() config = BuildConfig.fetch(self.env, config_name, db=db) if not config: raise HTTPNotFound("Build configuration '%s' does not exist." \ % config_name) repos_name, repos, repos_path = get_repos(self.env, config.path, req.authname) rev = config.max_rev or repos.youngest_rev try: _has_permission(req.perm, repos, repos_path, rev=rev, raise_error=True) except NoSuchNode: raise TracError("Permission checking against repository path %s " "at revision %s failed." % (config.path, rev)) data = {'title': 'Build Configuration "%s"' \ % config.label or config.name, 'page_mode': 'view_config'} add_link(req, 'up', req.href.build(), 'Build Status') description = config.description if description: description = wiki_to_html(description, self.env, req) pending_builds = list(Build.select(self.env, config=config.name, status=Build.PENDING)) inprogress_builds = list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS)) min_chgset_url = '' if config.min_rev: min_chgset_resource = get_chgset_resource(self.env, repos_name, config.min_rev) min_chgset_url = get_resource_url(self.env, min_chgset_resource, req.href), max_chgset_url = '' if config.max_rev: max_chgset_resource = get_chgset_resource(self.env, repos_name, config.max_rev) max_chgset_url = get_resource_url(self.env, max_chgset_resource, req.href), data['config'] = { 'name': config.name, 'label': config.label, 'path': config.path, 'min_rev': config.min_rev, 'min_rev_href': min_chgset_url, 'max_rev': config.max_rev, 'max_rev_href': max_chgset_url, 'active': config.active, 'description': description, 'browser_href': req.href.browser(config.path), 'builds_pending' : len(pending_builds), 'builds_inprogress' : len(inprogress_builds) } context = Context.from_request(req, config.resource) data['context'] = context data['config']['attachments'] = AttachmentModule(self.env).attachment_data(context) platforms = list(TargetPlatform.select(self.env, config=config_name, db=db)) data['config']['platforms'] = [ { 'name': platform.name, 'id': platform.id, 'builds_pending': len(list(Build.select(self.env, config=config.name, status=Build.PENDING, platform=platform.id))), 'builds_inprogress': len(list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS, platform=platform.id))) } for platform in platforms ] has_reports = False for report in Report.select(self.env, config=config.name, db=db): has_reports = True break if has_reports: chart_generators = [] report_categories = list(self._report_categories_for_config(config)) for generator in ReportChartController(self.env).generators: for category in generator.get_supported_categories(): if category in report_categories: chart_generators.append({ 'href': req.href.build(config.name, 'chart/' + category), 'category': category, 'style': self.config.get('bitten', 'chart_style'), }) data['config']['charts'] = chart_generators page = max(1, int(req.args.get('page', 1))) more = False data['page_number'] = page builds_per_page = 12 * len(platforms) idx = 0 builds = {} revisions = [] build_order = [] for platform, rev, build in collect_changes(config,authname=req.authname): if idx >= page * builds_per_page: more = True break elif idx >= (page - 1) * builds_per_page: if rev not in builds: revisions.append(rev) builds.setdefault(rev, {}) chgset_resource = get_chgset_resource(self.env, repos_name, rev) builds[rev].setdefault('href', get_resource_url(self.env, chgset_resource, req.href)) build_order.append((rev, repos.get_changeset(rev).date)) builds[rev].setdefault('display_rev', display_rev(repos, rev)) if build and build.status != Build.PENDING: build_data = _get_build_data(self.env, req, build) build_data['steps'] = [] for step in BuildStep.select(self.env, build=build.id, db=db): build_data['steps'].append({ 'name': step.name, 'description': step.description, 'duration': to_datetime(step.stopped or int(time.time()), utc) - \ to_datetime(step.started, utc), 'status': _step_status_label[step.status], 'cls': _step_status_label[step.status].replace(' ', '-'), 'errors': step.errors, 'href': build_data['href'] + '#step_' + step.name }) builds[rev][platform.id] = build_data idx += 1 data['config']['build_order'] = [r[0] for r in sorted(build_order, key=lambda x: x[1], reverse=True)] data['config']['builds'] = builds data['config']['revisions'] = revisions if page > 1: if page == 2: prev_href = req.href.build(config.name) else: prev_href = req.href.build(config.name, page=page - 1) add_link(req, 'prev', prev_href, 'Previous Page') if more: next_href = req.href.build(config.name, page=page + 1) add_link(req, 'next', next_href, 'Next Page') if arity(prevnext_nav) == 4: # Trac 0.12 compat, see #450 prevnext_nav(req, 'Previous Page', 'Next Page') else: prevnext_nav (req, 'Page') return data
def process_request(self, req): req.perm.require('BUILD_VIEW') db = self.env.get_db_cnx() build_id = int(req.args.get('id')) build = Build.fetch(self.env, build_id, db=db) if not build: raise HTTPNotFound("Build '%s' does not exist." \ % build_id) if req.method == 'POST': if req.args.get('action') == 'invalidate': self._do_invalidate(req, build, db) req.redirect(req.href.build(build.config, build.id)) add_link(req, 'up', req.href.build(build.config), 'Build Configuration') data = {'title': 'Build %s - %s' % (build_id, _status_title[build.status]), 'page_mode': 'view_build', 'build': {}} config = BuildConfig.fetch(self.env, build.config, db=db) data['build']['config'] = { 'name': config.label or config.name, 'href': req.href.build(config.name) } context = Context.from_request(req, build.resource) data['context'] = context data['build']['attachments'] = AttachmentModule(self.env).attachment_data(context) formatters = [] for formatter in self.log_formatters: formatters.append(formatter.get_formatter(req, build)) summarizers = {} # keyed by report type for summarizer in self.report_summarizers: categories = summarizer.get_supported_categories() summarizers.update(dict([(cat, summarizer) for cat in categories])) repos_name, repos, repos_path = get_repos(self.env, config.path, req.authname) _has_permission(req.perm, repos, repos_path, rev=build.rev, raise_error=True) data['build'].update(_get_build_data(self.env, req, build, repos_name)) steps = [] for step in BuildStep.select(self.env, build=build.id, db=db): steps.append({ 'name': step.name, 'description': step.description, 'duration': pretty_timedelta(step.started, step.stopped or int(time.time())), 'status': _step_status_label[step.status], 'cls': _step_status_label[step.status].replace(' ', '-'), 'errors': step.errors, 'log': self._render_log(req, build, formatters, step), 'reports': self._render_reports(req, config, build, summarizers, step) }) data['build']['steps'] = steps data['build']['can_delete'] = ('BUILD_DELETE' in req.perm \ and build.status != build.PENDING) chgset = repos.get_changeset(build.rev) data['build']['chgset_author'] = chgset.author data['build']['display_rev'] = display_rev(repos, build.rev) add_script(req, 'common/js/folding.js') add_script(req, 'bitten/tabset.js') add_script(req, 'bitten/jquery.flot.js') add_stylesheet(req, 'bitten/bitten.css') return 'bitten_build.html', data, None
def _render_overview(self, req): data = {'title': 'Build Status'} show_all = False if req.args.get('show') == 'all': show_all = True data['show_all'] = show_all configs = [] for config in BuildConfig.select(self.env, include_inactive=show_all): repos_name, repos, repos_path = get_repos(self.env, config.path, req.authname) rev = config.max_rev or repos.youngest_rev try: if not _has_permission(req.perm, repos, repos_path, rev=rev): continue except NoSuchNode: add_warning(req, "Configuration '%s' points to non-existing " "path '/%s' at revision '%s'. Configuration skipped." \ % (config.name, config.path, rev)) continue description = config.description if description: description = wiki_to_html(description, self.env, req) platforms_data = [] for platform in TargetPlatform.select(self.env, config=config.name): pd = { 'name': platform.name, 'id': platform.id, 'builds_pending': len(list(Build.select(self.env, config=config.name, status=Build.PENDING, platform=platform.id))), 'builds_inprogress': len(list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS, platform=platform.id))) } platforms_data.append(pd) config_data = { 'name': config.name, 'label': config.label or config.name, 'active': config.active, 'path': config.path, 'description': description, 'builds_pending' : len(list(Build.select(self.env, config=config.name, status=Build.PENDING))), 'builds_inprogress' : len(list(Build.select(self.env, config=config.name, status=Build.IN_PROGRESS))), 'href': req.href.build(config.name), 'builds': [], 'platforms': platforms_data } configs.append(config_data) if not config.active: continue prev_rev = None for platform, rev, build in collect_changes(config, req.authname): if rev != prev_rev: if prev_rev is None: chgset = repos.get_changeset(rev) chgset_resource = get_chgset_resource(self.env, repos_name, rev) config_data['youngest_rev'] = { 'id': rev, 'href': get_resource_url(self.env, chgset_resource, req.href), 'display_rev': display_rev(repos, rev), 'author': chgset.author or 'anonymous', 'date': format_datetime(chgset.date), 'message': wiki_to_oneliner( shorten_line(chgset.message), self.env, req=req) } else: break prev_rev = rev if build: build_data = _get_build_data(self.env, req, build, repos_name) build_data['platform'] = platform.name config_data['builds'].append(build_data) else: config_data['builds'].append({ 'platform': platform.name, 'status': 'pending' }) data['configs'] = sorted(configs, key=lambda x:x['label'].lower()) data['page_mode'] = 'overview' in_progress_builds = Build.select(self.env, status=Build.IN_PROGRESS) pending_builds = Build.select(self.env, status=Build.PENDING) data['builds_pending'] = len(list(pending_builds)) data['builds_inprogress'] = len(list(in_progress_builds)) add_link(req, 'views', req.href.build(view='inprogress'), 'In Progress Builds') add_ctxtnav(req, 'In Progress Builds', req.href.build(view='inprogress')) return data