def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None, reason='', external_idstring=None, properties=None, builderNames=None, **kw): if sourcestamps is None: sourcestamps = [] # combine properties if properties: properties.updateFromProperties(self.properties) else: properties = self.properties # make a fresh copy that we actually can modify safely properties = Properties.fromDict(properties.asDict()) # make extra info available from properties.render() properties.master = self.master properties.sourcestamps = [] properties.changes = [] for ss in sourcestamps: if isinstance(ss, int): # fetch actual sourcestamp and changes from data API properties.sourcestamps.append( (yield self.master.data.get(('sourcestamps', ss)))) properties.changes.extend( (yield self.master.data.get(('sourcestamps', ss, 'changes')))) else: # sourcestamp with no change, see addBuildsetForChanges properties.sourcestamps.append(ss) for c in properties.changes: properties.updateFromProperties(Properties.fromDict(c['properties'])) # apply the default builderNames if not builderNames: builderNames = self.builderNames # dynamically get the builder list to schedule builderNames = yield properties.render(builderNames) # Get the builder ids # Note that there is a data.updates.findBuilderId(name) # but that would merely only optimize the single builder case, while # probably the multiple builder case will be severely impacted by the # several db requests needed. builderids = [] for bldr in (yield self.master.data.get(('builders', ))): if bldr['name'] in builderNames: builderids.append(bldr['builderid']) # translate properties object into a dict as required by the # addBuildset method properties_dict = yield properties.render(properties.asDict()) bsid, brids = yield self.master.data.updates.addBuildset( scheduler=self.name, sourcestamps=sourcestamps, reason=reason, waited_for=waited_for, properties=properties_dict, builderids=builderids, external_idstring=external_idstring, **kw) return (bsid, brids)
def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None, reason='', external_idstring=None, properties=None, builderNames=None, **kw): if sourcestamps is None: sourcestamps = [] # combine properties if properties: properties.updateFromProperties(self.properties) else: properties = self.properties # make a fresh copy that we actually can modify safely properties = Properties.fromDict(properties.asDict()) # make extra info available from properties.render() properties.master = self.master properties.sourcestamps = [] properties.changes = [] for ss in sourcestamps: if isinstance(ss, int): # fetch actual sourcestamp and changes from data API properties.sourcestamps.append( (yield self.master.data.get(('sourcestamps', ss)))) properties.changes.extend( (yield self.master.data.get(('sourcestamps', ss, 'changes')))) else: # sourcestamp with no change, see addBuildsetForChanges properties.sourcestamps.append(ss) for c in properties.changes: properties.updateFromProperties(Properties.fromDict(c['properties'])) # apply the default builderNames if not builderNames: builderNames = self.builderNames # dynamically get the builder list to schedule builderNames = yield properties.render(builderNames) # Get the builder ids # Note that there is a data.updates.findBuilderId(name) # but that would merely only optimize the single builder case, while # probably the multiple builder case will be severely impacted by the # several db requests needed. builderids = list() for bldr in (yield self.master.data.get(('builders', ))): if bldr['name'] in builderNames: builderids.append(bldr['builderid']) # translate properties object into a dict as required by the # addBuildset method properties_dict = properties.asDict() bsid, brids = yield self.master.data.updates.addBuildset( scheduler=self.name, sourcestamps=sourcestamps, reason=reason, waited_for=waited_for, properties=properties_dict, builderids=builderids, external_idstring=external_idstring, **kw) return (bsid, brids)
async def render_success(self, build, master): # extract logs named as `result` results = {} for step, log_lines in self.extract_logs(build, logname='result'): if step['results'] == SUCCESS: results[step['stepid']] = (line for _, line in log_lines) # render the crossbow repo, becuase it might be passed as a Property props = Properties.fromDict(build['properties']) props.master = master crossbow_repo = await props.render(self.crossbow_repo) if not isinstance(crossbow_repo, str): raise ValueError('crossbow_repo argument must be a string') try: # decode yaml objects and render the results as a github links # pointing to the pushed crossbow branches messages = [ self._render_message(yaml_lines, crossbow_repo=crossbow_repo) for yaml_lines in results.values() ] except Exception as e: log.error(e) raise context = '\n\n'.join(messages) return dict(status='has been succeeded', context=context)
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) else: state = 'pending' description = yield props.render(self.startDescription) context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps or not sourcestamps[0]: return project = sourcestamps[0]['project'] if project: repoOwner, repoName = project.split('/') else: repo = sourcestamps[0]['repository'].split('/')[-2:] repoOwner = repo[0] repoName = '.'.join(repo[1].split('.')[:-1]) for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: yield self.createStatus( repo_user=repoOwner.encode('utf-8'), repo_name=repoName.encode('utf-8'), sha=sha.encode('utf-8'), state=state.encode('utf-8'), target_url=build['url'].encode('utf-8'), context=context.encode('utf-8'), description=description.encode('utf-8')) if self.verbose: log.msg('Status "{state}" sent for ' '{repoOwner}/{repoName} at {sha}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha)) except Exception as e: log.err( e, 'Fail to send status "{state}" for ' '{repoOwner}/{repoName} at {sha}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha))
def start_instance(self, build): if self.instance is not None: raise ValueError('instance active') # pylint: disable=invalid-name masterFQDN = self.masterFQDN # noqa: N806 # pylint: disable=invalid-name masterPort = '9989' # noqa: N806 if self.registration is not None: # pylint: disable=invalid-name masterPort = str(self.registration.getPBPort()) # noqa: N806 if ":" in masterFQDN: masterFQDN, masterPort = masterFQDN.split(':') master_properties = Properties.fromDict({ 'masterHash': (self.masterhash, self.properties_source), 'masterFQDN': (masterFQDN, self.properties_source), 'masterPort': (masterPort, self.properties_source), 'workerName': (self.name, self.properties_source), 'workerPass': (self.password, self.properties_source) }) build.properties.updateFromProperties(master_properties) namespace = yield build.render(self.namespace) job = yield build.render(self.job) res = yield threads.deferToThread( self._thd_start_instance, namespace, job ) defer.returnValue(res)
def sendMessage(self, body, subject=None, type=None, builderName=None, results=None, builds=None, users=None, patches=None, logs=None, worker=None): pr_urls = set() for build in builds: props = Properties.fromDict(build['properties']) pr_urls.add(props.getProperty("pullrequesturl")) for pr_url in pr_urls: try: res = yield self.sendComment(pr_url=pr_url, text=body) if res.code not in (HTTP_CREATED, ): content = yield res.content() log.msg("{code}: Unable to send a comment: " "{content}".format(code=res.code, content=content)) elif self.verbose: log.msg('Comment sent to {url}'.format(url=pr_url)) except Exception as e: log.err(e, 'Failed to send a comment to "{}"'.format(pr_url))
def send(self, build): props = Properties.fromDict(build['properties']) results = build['results'] if build['complete']: status = STASH_SUCCESSFUL if results == SUCCESS else STASH_FAILED description = self.endDescription else: status = STASH_INPROGRESS description = self.startDescription for sourcestamp in build['buildset']['sourcestamps']: sha = sourcestamp['revision'] key = yield props.render(self.key) payload = { 'state': status, 'url': build['url'], 'key': key, } if description: payload['description'] = yield props.render(description) if self.statusName: payload['name'] = yield props.render(self.statusName) response = yield self._http.post('/rest/build-status/1.0/commits/' + sha, json=payload) if response.code == 204: if self.verbose: log.info('Status "{status}" sent for {sha}.', status=status, sha=sha) else: content = yield response.content() log.error("{code}: Unable to send Stash status: {content}", code=response.code, content=content)
def test_setBuildProperties(self): self.master.db.insertTestData([ fakedb.Buildset(id=28), fakedb.BuildRequest(id=5, buildsetid=28), fakedb.Master(id=3), fakedb.Worker(id=42, name="Friday"), fakedb.Build(id=1234, buildrequestid=5, masterid=3, workerid=42), ]) self.master.db.builds.setBuildProperty = mock.Mock(wraps=self.master.db.builds.setBuildProperty) props = processProperties.fromDict(dict(a=(1, 't'), b=(['abc', 9], 't'))) yield self.rtype.setBuildProperties(1234, props) self.master.db.builds.setBuildProperty.assert_has_calls([ mock.call(1234, u'a', 1, u't'), mock.call(1234, u'b', ['abc', 9], u't')]) self.master.mq.assertProductions([ (('builds', '1234', 'properties', 'update'), {u'a': (1, u't'), u'b': (['abc', 9], u't')}), ]) # sync without changes: no db write self.master.db.builds.setBuildProperty.reset_mock() self.master.mq.clearProductions() yield self.rtype.setBuildProperties(1234, props) self.master.db.builds.setBuildProperty.assert_not_called() self.master.mq.assertProductions([]) # sync with one changes: one db write props.setProperty('b', 2, 'step') self.master.db.builds.setBuildProperty.reset_mock() yield self.rtype.setBuildProperties(1234, props) self.master.db.builds.setBuildProperty.assert_called_with(1234, u'b', 2, u'step') self.master.mq.assertProductions([ (('builds', '1234', 'properties', 'update'), {u'b': (2, u'step')}) ])
def send(self, build): props = Properties.fromDict(build['properties']) results = build['results'] if build['complete']: status = STASH_SUCCESSFUL if results == SUCCESS else STASH_FAILED description = self.endDescription else: status = STASH_INPROGRESS description = self.startDescription for sourcestamp in build['buildset']['sourcestamps']: sha = sourcestamp['revision'] key = yield props.render(self.key) payload = { 'state': status, 'url': build['url'], 'key': key, } if description: payload['description'] = yield props.render(description) if self.statusName: payload['name'] = yield props.render(self.statusName) response = yield self._http.post( '/rest/build-status/1.0/commits/' + sha, json=payload) if response.code == 204: if self.verbose: log.info('Status "{status}" sent for {sha}.', status=status, sha=sha) else: content = yield response.content() log.error("{code}: Unable to send Stash status: {content}", code=response.code, content=content)
def sendMessage(self, reports): body = merge_reports_prop(reports, 'body') builds = merge_reports_prop(reports, 'builds') pr_urls = set() for build in builds: props = Properties.fromDict(build['properties']) pr_urls.add(props.getProperty("pullrequesturl")) for pr_url in pr_urls: if pr_url is None: continue try: res = yield self.sendComment( pr_url=pr_url, text=body ) if res.code not in (HTTP_CREATED,): content = yield res.content() log.msg("{code}: Unable to send a comment: " "{content}".format(code=res.code, content=content)) elif self.verbose: log.msg('Comment sent to {url}'.format(url=pr_url)) except Exception as e: log.err(e, 'Failed to send a comment to "{}"'.format(pr_url))
def gotChange(self, change, important): if '#nobuild' in change.comments: print 'ignoring this change because it has #nobuild in the commit message' return changeid = change.number log.msg('got change for change {}'.format(changeid)) to_build = transitive_closure(self.dependency_tree, self.builds_from_change(change)) yield self.setState('to_build-' + str(changeid), dependency_tree.serialize(to_build)) already_built = set() yield self.setState('already_built-' + str(changeid), []) builds_to_build_now = builds_that_can_be_built(to_build, already_built) if len(builds_to_build_now) == 0: return log.msg('initially building builds {} in change {}'.format( builds_to_build_now, changeid)) for build in builds_to_build_now: yield self.addBuildsetForChanges(reason='because of change', changeids=[changeid], builderNames=[build], properties=Properties.fromDict({ 'changeid': (changeid, 'Scheduler'), 'build': (build, 'Scheduler') }))
def gotChange(self, change, important): if not self.treeStableTimer: # if there's no treeStableTimer, we can completely ignore # unimportant changes if not important: return defer.succeed(None) properties = change.properties if(type(properties) == dict): properties = Properties.fromDict(properties) # otherwise, we'll build it right away return self.addBuildsetForChanges(reason=self.reason, properties=properties, changeids=[change.number]) timer_name = self.getTimerNameForChange(change) # if we have a treeStableTimer # - for an important change, start the timer # - for an unimportant change, reset the timer if it is running if important or self._stable_timers[timer_name]: if self._stable_timers[timer_name]: self._stable_timers[timer_name].cancel() def fire_timer(): d = self.stableTimerFired(timer_name) d.addErrback(log.err, "while firing stable timer") self._stable_timers[timer_name] = self._reactor.callLater( self.treeStableTimer, fire_timer) # record the change's importance return self.master.db.schedulers.classifyChanges( self.objectid, {change.number: important})
def sendMessage(self, reports): report = reports[0] build = reports[0]['builds'][0] props = Properties.fromDict(build['properties']) props.master = self.master description = report.get('body', None) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failed', SKIPPED: 'success', EXCEPTION: 'failed', RETRY: 'pending', CANCELLED: 'cancelled' }.get(build['results'], 'failed') else: state = 'running' context = yield props.render(self.context) sourcestamps = build['buildset']['sourcestamps'] # FIXME: probably only want to report status for the last commit in the changeset for sourcestamp in sourcestamps: sha = sourcestamp['revision'] if 'source_project_id' in props: proj_id = props['source_project_id'] else: proj_id = yield self.getProjectId(sourcestamp) if proj_id is None: continue try: if 'source_branch' in props: branch = props['source_branch'] else: branch = sourcestamp['branch'] target_url = build['url'] res = yield self.createStatus(project_id=proj_id, branch=branch, sha=sha, state=state, target_url=target_url, context=context, description=description) if res.code not in (200, 201, 204): message = yield res.json() message = message.get('message', 'unspecified error') log.msg(f'Could not send status "{state}" for ' f'{sourcestamp["repository"]} at {sha}: {message}') elif self.verbose: log.msg(f'Status "{state}" sent for ' f'{sourcestamp["repository"]} at {sha}.') except Exception as e: log.err(e, (f'Failed to send status "{state}" for ' f'{sourcestamp["repository"]} at {sha}'))
def send(self, build): props = Properties.fromDict(build['properties']) props.master = self.master results = build['results'] if build['complete']: state = SUCCESSFUL if results == SUCCESS else FAILED description = self.endDescription else: state = INPROGRESS description = self.startDescription key = yield props.render(self.key) description = yield props.render(description) if description else None context = yield props.render(self.context) if self.context else None sourcestamps = build['buildset']['sourcestamps'] for sourcestamp in sourcestamps: try: sha = unicode2NativeString(sourcestamp['revision']) if sha is None: log.msg("Unable to get the commit hash") continue key = unicode2NativeString(key) state = unicode2NativeString(state) url = unicode2NativeString(build['url']) key = unicode2NativeString(key) description = unicode2NativeString(description) context = unicode2NativeString(context) res = yield self.createStatus( sha=sha, state=state, url=url, key=key, description=description, context=context ) if res.code not in (HTTP_PROCESSED,): content = yield res.content() log.msg("{code}: Unable to send Bitbucket Server status: " "{content}".format(code=res.code, content=content)) elif self.verbose: log.msg('Status "{state}" sent for {sha}.'.format( state=state, sha=sha)) except Exception as e: log.err( e, 'Failed to send status "{state}" for ' '{repo} at {sha}'.format( state=state, repo=sourcestamp['repository'], sha=sha ))
async def send(self, build): # License note: # It is a reimplementation based on the parent GitHubStatusPush # from the original buildbot implementation. # # We must propagate the build down to the renderer callbacks # (endDescription, startDescription), otherwise there is no way to # retrieve the build and its logs. # # Only `buildername` and `builnumber` properties are set, but # data.get(('builders', buildername, 'builds', buildnumber)) raises # for non-alphanumerical builder name: # Invalid path: builders/Ursabot Python 3.7/builds/2 # So the official[quiet twisted] example wouldn't work: # http://docs.buildbot.net/2.3.0/full.html#githubcommentpush cls = self.__class__.__name__ build_number = build['number'] builder_name = build['builder']['name'] sourcestamps = build['buildset'].get('sourcestamps', []) properties = Properties.fromDict(build['properties']) for sourcestamp in sourcestamps: project = sourcestamp.get('project') repository = sourcestamp.get('repository') if self.verbose: log.info(f'Triggering {cls}.report() for project {project}, ' f'repository {repository}, builder {builder_name}, ' f'build number {build_number}') try: response = await self.report(build, sourcestamp, properties) except Exception as e: log.error(e) raise e # report() can return None to skip reporting if response is None: continue if not self.isStatus2XX(response.code): content = await response.content() e = Exception( f'Failed to execute http API call in {cls}.report() for ' f'repository {repository}, builder {builder_name}, build ' f'number {build_number} with error code {response.code} ' f'and response "{content}"') log.error(e) raise e if self.verbose: log.info(f'Successful report {cls}.report() for repository ' f'{repository}, builder {builder_name}, build number ' f'{build_number}')
def test_gerrit_changes(self): yield self.createGerritStatus() # from chdict: chdict = TestGerritChangeSource.expected_change props = Properties.fromDict( {k: (v, 'change') for k, v in chdict['properties'].items()}) changes = self.sp.getGerritChanges(props) self.assertEqual(changes, [{'change_id': '4321', 'revision_id': '12'}])
def test_gerrit_changes(self): yield self.createGerritStatus() # from chdict: chdict = TestGerritChangeSource.expected_change props = Properties.fromDict({ k: (v, 'change') for k, v in chdict['properties'].items()}) changes = self.sp.getGerritChanges(props) self.assertEqual(changes, [ {'change_id': '4321', 'revision_id': '12'} ])
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) else: state = 'pending' description = yield props.render(self.startDescription) context = yield props.render(self.context) sourcestamps = build['buildset']['sourcestamps'] project = sourcestamps[0]['project'] if project: repoOwner, repoName = project.split('/') else: repo = sourcestamps[0]['repository'].split('/')[-2:] repoOwner = repo[0] repoName = '.'.join(repo[1].split('.')[:-1]) for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: yield self.createStatus( repo_user=repoOwner.encode('utf-8'), repo_name=repoName.encode('utf-8'), sha=sha.encode('utf-8'), state=state.encode('utf-8'), target_url=build['url'].encode('utf-8'), context=context.encode('utf-8'), description=description.encode('utf-8') ) if self.verbose: log.msg( 'Status "{state}" sent for ' '{repoOwner}/{repoName} at {sha}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha)) except Exception as e: log.err( e, 'Fail to send status "{state}" for ' '{repoOwner}/{repoName} at {sha}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha))
def sendMessage(self, reports): # Only use OAuth if basic auth has not been specified if not self.auth: request = yield self.oauthhttp.post("", data=_GET_TOKEN_DATA) if request.code != 200: content = yield request.content() log.msg( f"{request.code}: unable to authenticate to Bitbucket {content}" ) return token = (yield request.json())['access_token'] self._http.updateHeaders({'Authorization': f'Bearer {token}'}) build = reports[0]['builds'][0] if build['complete']: status = BITBUCKET_SUCCESSFUL if build[ 'results'] == SUCCESS else BITBUCKET_FAILED else: status = BITBUCKET_INPROGRESS props = Properties.fromDict(build['properties']) props.master = self.master body = { 'state': status, 'key': (yield props.render(self.status_key)), 'name': (yield props.render(self.status_name)), 'description': reports[0]['subject'], 'url': build['url'] } for sourcestamp in build['buildset']['sourcestamps']: if not sourcestamp['repository']: log.msg(f"Empty repository URL for Bitbucket status {body}") continue owner, repo = self.get_owner_and_repo(sourcestamp['repository']) endpoint = (owner, repo, 'commit', sourcestamp['revision'], 'statuses', 'build') bitbucket_uri = f"/{'/'.join(endpoint)}" if self.debug: log.msg(f"Bitbucket status {bitbucket_uri} {body}") response = yield self._http.post(bitbucket_uri, json=body) if response.code not in (200, 201): content = yield response.content() log.msg( f"{response.code}: unable to upload Bitbucket status {content}" )
def sendMessage(self, reports): report = reports[0] build = reports[0]['builds'][0] props = Properties.fromDict(build['properties']) props.master = self.master description = report.get('body', None) results = build['results'] if build['complete']: state = SUCCESSFUL if results == SUCCESS else FAILED else: state = INPROGRESS key = yield props.render(self.key) context = yield props.render(self.context) if self.context else None sourcestamps = build['buildset']['sourcestamps'] for sourcestamp in sourcestamps: try: sha = sourcestamp['revision'] if sha is None: log.msg("Unable to get the commit hash") continue url = build['url'] res = yield self.createStatus(sha=sha, state=state, url=url, key=key, description=description, context=context) if res.code not in (HTTP_PROCESSED, ): content = yield res.content() log.msg("{code}: Unable to send Bitbucket Server status: " "{content}".format(code=res.code, content=content)) elif self.verbose: log.msg('Status "{state}" sent for {sha}.'.format( state=state, sha=sha)) except Exception as e: log.err( e, 'Failed to send status "{state}" for ' '{repo} at {sha}'.format(state=state, repo=sourcestamp['repository'], sha=sha))
def test_setBuildProperties(self): self.master.db.insertTestData([ fakedb.Buildset(id=28), fakedb.BuildRequest(id=5, buildsetid=28), fakedb.Master(id=3), fakedb.Buildslave(id=42, name="Friday"), fakedb.Build(id=1234, buildrequestid=5, masterid=3, buildslaveid=42), ]) self.master.db.builds.setBuildProperty = mock.Mock( wraps=self.master.db.builds.setBuildProperty) props = processProperties.fromDict( dict(a=(1, 't'), b=(['abc', 9], 't'))) yield self.rtype.setBuildProperties(1234, props) self.master.db.builds.setBuildProperty.assert_has_calls([ mock.call(1234, u'a', 1, u't'), mock.call(1234, u'b', ['abc', 9], u't') ]) self.master.mq.assertProductions([ (('builds', '1234', 'properties', 'update'), { u'a': (1, u't'), u'b': (['abc', 9], u't') }), ]) # sync without changes: no db write self.master.db.builds.setBuildProperty.reset_mock() self.master.mq.clearProductions() yield self.rtype.setBuildProperties(1234, props) self.master.db.builds.setBuildProperty.assert_not_called() self.master.mq.assertProductions([]) # sync with one changes: one db write props.setProperty('b', 2, 'step') self.master.db.builds.setBuildProperty.reset_mock() yield self.rtype.setBuildProperties(1234, props) self.master.db.builds.setBuildProperty.assert_called_with( 1234, u'b', 2, u'step') self.master.mq.assertProductions([(('builds', '1234', 'properties', 'update'), { u'b': (2, u'step') })])
def sendMessage(self, reports): report = reports[0] build = reports[0]['builds'][0] props = Properties.fromDict(build['properties']) props.master = self.master comment = report.get('body', None) if build['complete']: value = self.RESULTS_TABLE.get(build['results'], self.DEFAULT_RESULT) duration = self.formatDuration(build['complete_at'] - build['started_at']) else: value = 0 duration = 'pending' name = yield props.render(self._verification_name) reporter = yield props.render(self._reporter) category = yield props.render(self._category) abstain = yield props.render(self._abstain) # TODO: find reliable way to find out whether its a rebuild rerun = None changes = yield self.getGerritChanges(props) for change in changes: try: yield self.createStatus(change['change_id'], change['revision_id'], name, value, abstain=abstain, rerun=rerun, comment=comment, url=build['url'], reporter=reporter, category=category, duration=duration) except Exception: log.failure('Failed to send status!', failure=failure.Failure())
def sendMessage(self, body, subject=None, type=None, builderName=None, results=None, builds=None, users=None, patches=None, logs=None, worker=None): pr_urls = set() for build in builds: props = Properties.fromDict(build['properties']) pr_urls.add(props.getProperty("pullrequesturl")) for pr_url in pr_urls: try: res = yield self.sendComment( pr_url=pr_url, text=body ) if res.code not in (HTTP_CREATED,): content = yield res.content() log.msg("{code}: Unable to send a comment: " "{content}".format(code=res.code, content=content)) elif self.verbose: log.msg('Comment sent to {url}'.format(url=pr_url)) except Exception as e: log.err(e, 'Failed to send a comment to "{}"'.format(pr_url))
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: value = self.RESULTS_TABLE.get(build['results'], self.DEFAULT_RESULT) comment = yield props.render(self._endDescription) duration = self.formatDuration(build['complete_at'] - build[ 'started_at']) else: value = 0 comment = yield props.render(self._startDescription) duration = 'pending' name = yield props.render(self._verification_name) reporter = yield props.render(self._reporter) category = yield props.render(self._category) abstain = yield props.render(self._abstain) # TODO: find reliable way to find out whether its a rebuild rerun = None changes = yield self.getGerritChanges(props) for change in changes: try: yield self.createStatus( change['change_id'], change['revision_id'], name, value, abstain=abstain, rerun=rerun, comment=comment, url=build['url'], reporter=reporter, category=category, duration=duration) except Exception: log.failure( 'Failed to send status!', failure=failure.Failure())
def test_gerrit_changes(self): yield self.createGerritStatus() # from chdict: change_props = { 'event.change.owner.email': '*****@*****.**', 'event.change.subject': 'fix 1234', 'event.change.project': 'pr', 'event.change.owner.name': 'Dustin', 'event.change.number': '4321', 'event.change.url': 'http://buildbot.net', 'event.change.branch': 'br', 'event.type': 'patchset-created', 'event.patchSet.revision': 'abcdef', 'event.patchSet.number': '12', 'event.source': 'GerritChangeSource' } props = Properties.fromDict({k: (v, 'change') for k, v in change_props.items()}) changes = self.sp.getGerritChanges(props) self.assertEqual(changes, [ {'change_id': '4321', 'revision_id': '12'} ])
def sendMessage(self, reports): report = reports[0] build = reports[0]['builds'][0] props = Properties.fromDict(build['properties']) props.master = self.master description = report.get('body', None) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') else: state = 'pending' context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps: return issue = self._extract_issue(props) for sourcestamp in sourcestamps: repo_owner, repo_name = self._extract_github_info(sourcestamp) if not repo_owner or not repo_name: log.msg( 'Skipped status update because required repo information is missing.' ) continue sha = sourcestamp['revision'] response = None # If the scheduler specifies multiple codebases, don't bother updating # the ones for which there is no revision if not sha: log.msg( f"Skipped status update for codebase {sourcestamp['codebase']}, " f"context '{context}', issue {issue}.") continue try: if self.verbose: log.msg( f"Updating github status: repo_owner={repo_owner}, repo_name={repo_name}" ) response = yield self.createStatus(repo_user=repo_owner, repo_name=repo_name, sha=sha, state=state, target_url=build['url'], context=context, issue=issue, description=description) if not response: # the implementation of createStatus refused to post update due to missing data continue if not self.is_status_2xx(response.code): raise Exception() if self.verbose: log.msg( f'Updated status with "{state}" for {repo_owner}/{repo_name} ' f'at {sha}, context "{context}", issue {issue}.') except Exception as e: if response: content = yield response.content() code = response.code else: content = code = "n/a" log.err(e, ( f'Failed to update "{state}" for {repo_owner}/{repo_name} ' f'at {sha}, context "{context}", issue {issue}. ' f'http {code}, {content}'))
def send(self, build): props = Properties.fromDict(build["properties"]) props.master = self.master if build["complete"]: state = { SUCCESS: "success", WARNINGS: "success", FAILURE: "failure", SKIPPED: "success", EXCEPTION: "error", RETRY: "pending", CANCELLED: "error", }.get(build["results"], "error") else: return if state != "failure": return yield getDetailsForBuild(self.master, build, wantLogs=True, wantSteps=True) tracebacks = list() try: test_log = build["steps"][TESTS_STEP]["logs"][0]["content"]["content"] test_log = "\n".join([line.lstrip("eo") for line in test_log.splitlines()]) tracebacks = TRACEBACK_REGEX.findall(test_log) except IndexError: pass if not tracebacks: tracebacks = list(self._construct_tracebacks_from_stderr(build)) context = yield props.render(self.context) sourcestamps = build["buildset"].get("sourcestamps") if not sourcestamps or not sourcestamps[0]: return changes = yield self.master.data.get(("builds", build["buildid"], "changes")) if len(changes) > 1: return change, = changes m = re.search(r"\((?:GH-|#)(\d+)\)", change["comments"]) if m is None: return issue = m.groups()[-1] project = sourcestamps[0]["project"] if "/" in project: repoOwner, repoName = project.split("/") else: giturl = giturlparse(sourcestamps[0]["repository"]) repoOwner = giturl.owner repoName = giturl.repo if self.verbose: log.msg( "Updating github status: repoOwner={repoOwner}, repoName={repoName}".format( repoOwner=repoOwner, repoName=repoName ) ) try: repo_user = unicode2NativeString(repoOwner) repo_name = unicode2NativeString(repoName) sha = unicode2NativeString(change["revision"]) target_url = unicode2NativeString(build["url"]) context = unicode2NativeString(context) yield self.createStatus( build=build, repo_user=repo_user, repo_name=repo_name, sha=sha, state=state, target_url=target_url, context=context, issue=issue, tracebacks=tracebacks, ) if self.verbose: log.msg( "Issued a Pull Request comment for {repoOwner}/{repoName} " 'at {sha}, context "{context}", issue {issue}.'.format( repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue, context=context, ) ) except Exception as e: log.err( e, "Failed to issue a Pull Request comment for {repoOwner}/{repoName} " 'at {sha}, context "{context}", issue {issue}.'.format( repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue, context=context, ), )
def _send_impl(self, reports): report = reports[0] build = report['builds'][0] props = Properties.fromDict(build['properties']) props.master = self.master description = report.get('body', None) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success' if self.warningAsSuccess else 'warning', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'failure') else: state = 'pending' if 'pr_id' in props: context = yield props.render(self.context_pr) else: context = yield props.render(self.context) sourcestamps = build['buildset']['sourcestamps'] for sourcestamp in sourcestamps: sha = sourcestamp['revision'] if sha is None: # No special revision for this, so ignore it continue if 'repository_name' in props: repository_name = props['repository_name'] else: match = re.match(self.ssh_url_match, sourcestamp['repository']) if match is not None: repository_name = match.group("repo_name") else: log.msg( "Could not send status, " "build has no repository_name property for Gitea.") continue if 'owner' in props: repository_owner = props['owner'] else: match = re.match(self.ssh_url_match, sourcestamp['repository']) if match is not None: repository_owner = match.group("owner") else: log.msg( "Could not send status, " "build has no owner property for Gitea.") continue try: target_url = build['url'] res = yield self.createStatus( project_owner=repository_owner, repo_name=repository_name, sha=sha, state=state, target_url=target_url, context=context, description=description ) if res.code not in (200, 201, 204): message = yield res.json() message = message.get('message', 'unspecified error') log.msg( 'Could not send status "{state}" for ' '{repo} at {sha}: {code} : {message}'.format( state=state, repo=sourcestamp['repository'], sha=sha, code=res.code, message=message)) elif self.verbose: log.msg( 'Status "{state}" sent for ' '{repo} at {sha}.'.format( state=state, repo=sourcestamp['repository'], sha=sha)) except Exception as e: log.err( e, 'Failed to send status "{state}" for ' '{repo} at {sha}'.format( state=state, repo=sourcestamp['repository'], sha=sha ))
def send(self, build): props = Properties.fromDict(build['properties']) props.master = self.master if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) elif self.startDescription: state = 'pending' description = yield props.render(self.startDescription) else: return context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps: return for sourcestamp in sourcestamps: project = sourcestamp['project'] issue = None if 'branch' in props: m = re.search(r"refs/pull/([0-9]*)/merge", props['branch']) if m: issue = m.group(1) repo_owner = None repo_name = None if "/" in project: repo_owner, repo_name = project.split('/') else: giturl = giturlparse(sourcestamp['repository']) if giturl: repo_owner = giturl.owner repo_name = giturl.repo sha = sourcestamp['revision'] response = None # If the scheduler specifies multiple codebases, don't bother updating # the ones for which there is no revision if not sha: log.msg('Skipped status update for codebase {codebase}, ' 'context "{context}", issue {issue}.'.format( codebase=sourcestamp['codebase'], issue=issue, context=context)) continue try: if self.verbose: log.msg( "Updating github status: repo_owner={}, repo_name={}". format(repo_owner, repo_name)) response = yield self.createStatus(repo_user=repo_owner, repo_name=repo_name, sha=sha, state=state, target_url=build['url'], context=context, issue=issue, description=description) if not response: # the implementation of createStatus refused to post update due to missing data continue if not self.isStatus2XX(response.code): raise Exception() if self.verbose: log.msg( 'Updated status with "{state}" for {repo_owner}/{repo_name} ' 'at {sha}, context "{context}", issue {issue}.'.format( state=state, repo_owner=repo_owner, repo_name=repo_name, sha=sha, issue=issue, context=context)) except Exception as e: if response: content = yield response.content() code = response.code else: content = code = "n/a" log.err( e, 'Failed to update "{state}" for {repo_owner}/{repo_name} ' 'at {sha}, context "{context}", issue {issue}. ' 'http {code}, {content}'.format(state=state, repo_owner=repo_owner, repo_name=repo_name, sha=sha, issue=issue, context=context, code=code, content=content))
def send(self, build): props = Properties.fromDict(build['properties']) props.master = self.master duration = None test_results = None if build['complete']: state = SUCCESSFUL if build['results'] == SUCCESS else FAILED description = yield props.render(self.end_description) if self.duration: duration = yield props.render(self.duration) else: td = build['complete_at'] - build['started_at'] duration = int(td.seconds * 1000) if self.test_results: test_results = yield props.render(self.test_results) else: state = INPROGRESS description = yield props.render(self.start_description) duration = None parent_name = (build['parentbuilder'] or {}).get('name') if self.parent_name: parent = yield props.render(self.parent_name) elif parent_name: parent = parent_name else: parent = build['builder']['name'] if self.status_name: status_name = yield props.render(self.status_name) else: status_name = "{} #{}".format(props.getProperty("buildername"), props.getProperty("buildnumber")) if parent_name: status_name = "{} #{} \u00BB {}".format( parent_name, build['parentbuild']['number'], status_name) if self.status_suffix: status_name = status_name + (yield props.render( self.status_suffix)) key = yield props.render(self.key) build_number = yield props.render(self.build_number) url = build['url'] sourcestamps = build['buildset']['sourcestamps'] for sourcestamp in sourcestamps: try: ssid = sourcestamp.get('ssid') sha = sourcestamp.get('revision') branch = sourcestamp.get('branch') repo = sourcestamp.get('repository') if not sha: log.msg("Unable to get the commit hash for SSID: " "{}".format(ssid)) continue ref = None if self.ref is None: if branch is not None: if branch.startswith("refs/"): ref = branch else: ref = "refs/heads/{}".format(branch) else: ref = yield props.render(self.ref) if not ref: log.msg("WARNING: Unable to resolve ref for SSID: {}. " "Build status will not be visible on Builds or " "PullRequest pages only for commits".format(ssid)) r = re.search(r"^.*?/([^/]+)/([^/]+?)(?:\.git)?$", repo or "") if r: proj_key = r.group(1) repo_slug = r.group(2) else: log.msg("Unable to parse repository info from '{}' for " "SSID: {}".format(repo, ssid)) continue res = yield self.createStatus(proj_key=proj_key, repo_slug=repo_slug, sha=sha, state=state, url=url, key=key, parent=parent, build_number=build_number, ref=ref, description=description, name=status_name, duration=duration, test_results=test_results) if res.code not in (HTTP_PROCESSED, ): content = yield res.content() log.msg("{}: Unable to send Bitbucket Server status for " "{}/{} {}: {}".format(res.code, proj_key, repo_slug, sha, content)) elif self.verbose: log.msg('Status "{}" sent for {}/{} {}'.format( state, proj_key, repo_slug, sha)) except Exception as e: log.err( e, 'Failed to send status "{}" for {}/{} {}'.format( state, proj_key, repo_slug, sha))
def createEmail(self, msgdict, builderName, title, results, builds=None, patches=None, logs=None): text = msgdict['body'] type = msgdict['type'] subject = msgdict['subject'] assert '\n' not in subject, \ "Subject cannot contain newlines" assert type in ('plain', 'html'), \ "'{}' message type must be 'plain' or 'html'.".format(type) if patches or logs: m = MIMEMultipart() txt = MIMEText(text, type, ENCODING) m.attach(txt) else: m = Message() m.set_payload(text, ENCODING) m.set_type("text/{}".format(type)) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = self.fromaddr # m['To'] is added later if patches: for (i, patch) in enumerate(patches): a = self.patch_to_attachment(patch, i) m.attach(a) if logs: for log in logs: # Use distinct filenames for the e-mail summary name = "{}.{}".format(log['stepname'], log['name']) if len(builds) > 1: filename = "{}.{}".format(log['buildername'], name) else: filename = name text = log['content']['content'] a = MIMEText(text.encode(ENCODING), _charset=ENCODING) # convert to base64 to conform with RFC 5322 2.1.1 del a['Content-Transfer-Encoding'] encoders.encode_base64(a) a.add_header('Content-Disposition', "attachment", filename=filename) m.attach(a) # @todo: is there a better way to do this? # Add any extra headers that were requested, doing WithProperties # interpolation if only one build was given if self.extraHeaders: extraHeaders = self.extraHeaders if builds is not None and len(builds) == 1: props = Properties.fromDict(builds[0]['properties']) props.master = self.master extraHeaders = yield props.render(extraHeaders) for k, v in extraHeaders.items(): if k in m: twlog.msg("Warning: Got header " + k + " in self.extraHeaders " "but it already exists in the Message - " "not adding it.") m[k] = v return m
def send(self, build): props = Properties.fromDict(build['properties']) props.master = self.master if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) elif self.startDescription: state = 'pending' description = yield props.render(self.startDescription) else: return context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps or not sourcestamps[0]: return project = sourcestamps[0]['project'] branch = props['branch'] m = re.search(r"refs/pull/([0-9]*)/merge", branch) if m: issue = m.group(1) else: issue = None if "/" in project: repoOwner, repoName = project.split('/') else: giturl = giturlparse(sourcestamps[0]['repository']) repoOwner = giturl.owner repoName = giturl.repo if self.verbose: log.msg("Updating github status: repoOwner={repoOwner}, repoName={repoName}".format( repoOwner=repoOwner, repoName=repoName)) for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: repo_user = repoOwner repo_name = repoName target_url = build['url'] response = yield self.createStatus( repo_user=repo_user, repo_name=repo_name, sha=sha, state=state, target_url=target_url, context=context, issue=issue, description=description ) if not self.isStatus2XX(response.code): raise Exception() if self.verbose: log.msg( 'Updated status with "{state}" for {repoOwner}/{repoName} ' 'at {sha}, context "{context}", issue {issue}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue, context=context)) except Exception as e: content = yield response.content() log.err( e, 'Failed to update "{state}" for {repoOwner}/{repoName} ' 'at {sha}, context "{context}", issue {issue}. ' 'http {code}, {content}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue, context=context, code=response.code, content=content))
def createEmail(self, msgdict, builderName, title, results, builds=None, patches=None, logs=None): text = msgdict['body'].encode(ENCODING) type = msgdict['type'] if 'subject' in msgdict: subject = msgdict['subject'].encode(ENCODING) else: subject = self.subject % { 'result': Results[results], 'projectName': title, 'title': title, 'builder': builderName } assert '\n' not in subject, \ "Subject cannot contain newlines" assert type in ('plain', 'html'), \ "'%s' message type must be 'plain' or 'html'." % type if patches or logs: m = MIMEMultipart() txt = MIMEText(text, type, ENCODING) m.attach(txt) else: m = Message() m.set_payload(text, ENCODING) m.set_type("text/%s" % type) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = self.fromaddr # m['To'] is added later if patches: for (i, patch) in enumerate(patches): a = self.patch_to_attachment(patch, i) m.attach(a) if logs: for log in logs: name = "%s.%s" % (log['stepname'], log['name']) if (self._shouldAttachLog(log['name']) or self._shouldAttachLog(name)): # Use distinct filenames for the e-mail summary if self.buildSetSummary: filename = "%s.%s" % (log['buildername'], name) else: filename = name text = log['content']['content'] a = MIMEText(text.encode(ENCODING), _charset=ENCODING) a.add_header('Content-Disposition', "attachment", filename=filename) m.attach(a) # @todo: is there a better way to do this? # Add any extra headers that were requested, doing WithProperties # interpolation if only one build was given if self.extraHeaders: extraHeaders = self.extraHeaders if len(builds) == 1: props = Properties.fromDict(builds[0]['properties']) extraHeaders = yield props.render(extraHeaders) for k, v in iteritems(extraHeaders): if k in m: twlog.msg("Warning: Got header " + k + " in self.extraHeaders " "but it already exists in the Message - " "not adding it.") m[k] = v defer.returnValue(m)
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) elif self.startDescription: state = 'pending' description = yield props.render(self.startDescription) else: return context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps or not sourcestamps[0]: return project = sourcestamps[0]['project'] branch = props['branch'] m = re.search(r"refs/pull/([0-9]*)/merge", branch) if m: issue = m.group(1) else: issue = None if "/" in project: repoOwner, repoName = project.split('/') else: giturl = giturlparse(sourcestamps[0]['repository']) repoOwner = giturl.owner repoName = giturl.repo if self.verbose: log.msg( "Updating github status: repoOwner={repoOwner}, repoName={repoName}" .format(repoOwner=repoOwner, repoName=repoName)) for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: repo_user = unicode2NativeString(repoOwner) repo_name = unicode2NativeString(repoName) sha = unicode2NativeString(sha) state = unicode2NativeString(state) target_url = unicode2NativeString(build['url']) context = unicode2NativeString(context) issue = unicode2NativeString(issue) description = unicode2NativeString(description) yield self.createStatus(repo_user=repo_user, repo_name=repo_name, sha=sha, state=state, target_url=target_url, context=context, issue=issue, description=description) if self.verbose: log.msg('Updated status with "{state}" for ' '{repoOwner}/{repoName} at {sha}, issue {issue}.'. format(state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue)) except Exception as e: log.err( e, 'Failed to update "{state}" for ' '{repoOwner}/{repoName} at {sha}, issue {issue}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue))
def send(self, build): props = Properties.fromDict(build['properties']) props.master = self.master if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) elif self.startDescription: state = 'pending' description = yield props.render(self.startDescription) else: return context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps or not sourcestamps[0]: return branch = props['branch'] m = re.search(r"refs/pull/([0-9]*)/merge", branch) if m: issue = m.group(1) else: # We only want to comment pull requests, so we exit here return for sourcestamp in sourcestamps: if branch == sourcestamp['branch']: project = sourcestamp['project'] repository = sourcestamp['repository'] sha = sourcestamp['revision'] break if project is None: log.err('Failed to determine the project of PR "{branch}"'.format( branch=branch)) return if "/" in project: repoOwner, repoName = project.split('/') else: giturl = giturlparse(repository) repoOwner = giturl.owner repoName = giturl.repo if self.verbose: log.msg("Updating github status: repoOwner={repoOwner}, repoName={repoName}".format( repoOwner=repoOwner, repoName=repoName)) try: repo_user = unicode2NativeString(repoOwner) repo_name = unicode2NativeString(repoName) sha = unicode2NativeString(sha) state = unicode2NativeString(state) target_url = unicode2NativeString(build['url']) context = unicode2NativeString(context) issue = unicode2NativeString(issue) description = unicode2NativeString(description) yield self.createStatus( repo_user=repo_user, repo_name=repo_name, sha=sha, state=state, target_url=target_url, context=context, issue=issue, description=description ) if self.verbose: log.msg( 'Updated status with "{state}" for {repoOwner}/{repoName} ' 'at {sha}, context "{context}", issue {issue}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue, context=context)) except Exception as e: log.err( e, 'Failed to update "{state}" for {repoOwner}/{repoName} ' 'at {sha}, context "{context}", issue {issue}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue, context=context))
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failed', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) else: state = 'pending' description = yield props.render(self.startDescription) context = yield props.render(self.context) sourcestamps = build['buildset']['sourcestamps'] project = sourcestamps[0]['project'] # default to master if not found branch = sourcestamps[0].get('branch', 'master') if project: repoOwner, repoName = project.split('/') else: repo = sourcestamps[0]['repository'].split('/')[-2:] repoOwner = repo[0] repoName = '.'.join(repo[1].split('.')[:-1]) m = re.match(".*:(.*)", repoOwner) if m is not None: repoOwner = m.group(1) # retrieve project id via cache self.project_ids project_full_name = "%s%%2F%s" % (repoOwner, repoName) project_full_name = unicode2NativeString(project_full_name) if project_full_name not in self.project_ids: proj = yield self._http.get('/api/v3/projects/%s' % (project_full_name)) proj = yield proj.json() self.project_ids[project_full_name] = proj['id'] proj_id = self.project_ids[project_full_name] for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: branch = unicode2NativeString(branch) sha = unicode2NativeString(sha) state = unicode2NativeString(state) target_url = unicode2NativeString(build['url']) context = unicode2NativeString(context) description = unicode2NativeString(description) res = yield self.createStatus( project_id=proj_id, branch=branch, sha=sha, state=state, target_url=target_url, context=context, description=description ) if res.code not in (200, 201, 204): message = yield res.json() message = message.get('message', 'unspecified error') log.msg( 'Could not send status "{state}" for ' '{repoOwner}/{repoName} at {sha}: {message}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, message=message)) elif self.verbose: log.msg( 'Status "{state}" sent for ' '{repoOwner}/{repoName} at {sha}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha)) except Exception as e: log.err( e, 'Fail to send status "{state}" for ' '{repoOwner}/{repoName} at {sha}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha))
def createEmail(self, msgdict, builderName, title, results, builds=None, patches=None, logs=None): text = msgdict['body'] type = msgdict['type'] if msgdict.get('subject') is not None: subject = msgdict['subject'] else: subject = self.subject % {'result': Results[results], 'projectName': title, 'title': title, 'builder': builderName} assert '\n' not in subject, \ "Subject cannot contain newlines" assert type in ('plain', 'html'), \ "'{}' message type must be 'plain' or 'html'.".format(type) if patches or logs: m = MIMEMultipart() txt = MIMEText(text, type, ENCODING) m.attach(txt) else: m = Message() m.set_payload(text, ENCODING) m.set_type("text/{}".format(type)) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = self.fromaddr # m['To'] is added later if patches: for (i, patch) in enumerate(patches): a = self.patch_to_attachment(patch, i) m.attach(a) if logs: for log in logs: name = "{}.{}".format(log['stepname'], log['name']) if (self._shouldAttachLog(log['name']) or self._shouldAttachLog(name)): # Use distinct filenames for the e-mail summary if self.buildSetSummary: filename = "{}.{}".format(log['buildername'], name) else: filename = name text = log['content']['content'] a = MIMEText(text.encode(ENCODING), _charset=ENCODING) a.add_header('Content-Disposition', "attachment", filename=filename) m.attach(a) # @todo: is there a better way to do this? # Add any extra headers that were requested, doing WithProperties # interpolation if only one build was given if self.extraHeaders: extraHeaders = self.extraHeaders if builds is not None and len(builds) == 1: props = Properties.fromDict(builds[0]['properties']) props.master = self.master extraHeaders = yield props.render(extraHeaders) for k, v in extraHeaders.items(): if k in m: twlog.msg("Warning: Got header " + k + " in self.extraHeaders " "but it already exists in the Message - " "not adding it.") m[k] = v return m
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) else: state = 'pending' description = yield props.render(self.startDescription) context = yield props.render(self.context) sourcestamps = build['buildset']['sourcestamps'] project = sourcestamps[0]['project'] if project: repoOwner, repoName = project.split('/') else: repo = sourcestamps[0]['repository'].split('/')[-2:] repoOwner = repo[0] repoName = '.'.join(repo[1].split('.')[:-1]) m = re.match(".*:(.*)", repoOwner) if m is not None: repoOwner = m.group(1) # retrieve project id proj = yield self.session.get('%s/api/v3/projects/%s%%2F%s' % ( self.baseURL, repoOwner.encode('utf-8'), repoName.encode('utf-8'))) proj = proj.json() for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: res = yield self.createStatus( project_id=proj['id'], sha=sha.encode('utf-8'), state=state.encode('utf-8'), target_url=build['url'].encode('utf-8'), context=context.encode('utf-8'), description=description.encode('utf-8') ) if res.status_code not in (200, 201, 204): message = res.json().get('message', 'unspecified error') log.msg( 'Could not send status "{state}" for ' '{repoOwner}/{repoName} at {sha}: {message}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, message=message)) elif self.verbose: log.msg( 'Status "{state}" sent for ' '{repoOwner}/{repoName} at {sha}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha)) except Exception as e: log.err( e, 'Fail to send status "{state}" for ' '{repoOwner}/{repoName} at {sha}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha))
def send(self, build): props = Properties.fromDict(build["properties"]) if build["complete"]: state = { SUCCESS: "success", WARNINGS: "success", FAILURE: "failed", SKIPPED: "success", EXCEPTION: "error", RETRY: "pending", CANCELLED: "error", }.get(build["results"], "error") description = yield props.render(self.endDescription) else: state = "pending" description = yield props.render(self.startDescription) context = yield props.render(self.context) sourcestamps = build["buildset"]["sourcestamps"] project = sourcestamps[0]["project"] if project: repoOwner, repoName = project.split("/") else: repo = sourcestamps[0]["repository"].split("/")[-2:] repoOwner = repo[0] repoName = ".".join(repo[1].split(".")[:-1]) m = re.match(".*:(.*)", repoOwner) if m is not None: repoOwner = m.group(1) # retrieve project id proj = yield self.session.get( "%s/api/v3/projects/%s%%2F%s" % (self.baseURL, repoOwner.encode("utf-8"), repoName.encode("utf-8")) ) proj = proj.json() for sourcestamp in sourcestamps: sha = sourcestamp["revision"] try: res = yield self.createStatus( project_id=proj["id"], sha=sha.encode("utf-8"), state=state.encode("utf-8"), target_url=build["url"].encode("utf-8"), context=context.encode("utf-8"), description=description.encode("utf-8"), ) if res.status_code not in (200, 201, 204): message = res.json().get("message", "unspecified error") log.msg( 'Could not send status "{state}" for ' "{repoOwner}/{repoName} at {sha}: {message}".format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, message=message ) ) elif self.verbose: log.msg( 'Status "{state}" sent for ' "{repoOwner}/{repoName} at {sha}.".format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha ) ) except Exception as e: log.err( e, 'Fail to send status "{state}" for ' "{repoOwner}/{repoName} at {sha}".format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha ), )
def _buildset_complete_cb(self, key, msg): bsid = msg['bsid'] changeid, build_name, is_complete, result = yield self._get_build_info( bsid) if changeid is None: # Not a build that was created by DependencyTreeScheduler return if not is_complete: log.msg( 'strange - got a noncomlete build for {}'.format(build_name)) return already_built = set( (yield self.getState('already_built-' + str(changeid), []))) if build_name in already_built: log.msg( 'strange - got the same build twice {} for change {}'.format( bsid, changeid)) return to_build = dependency_tree.deserialize( (yield self.getState('to_build-' + str(changeid)))) already_built.add(build_name) if result not in (SUCCESS, WARNINGS): builds_to_skip_due_to_error = transitive_closure( to_build, set([build_name])).keys() log.msg('change #{}, build {} failed. Killing future builds of {}'. format(changeid, build_name, builds_to_skip_due_to_error)) already_built.update(set(builds_to_skip_due_to_error)) yield self.setState('already_built-' + str(changeid), list(already_built)) return yield self.setState('already_built-' + str(changeid), list(already_built)) yet_to_be_built = to_build.copy() for build in already_built: yet_to_be_built.pop(build, None) builds_to_build_now = builds_that_can_be_built(yet_to_be_built, already_built) log.msg( 'change #{}, build {} completed. need to build {}, but will build only {}' .format(changeid, build_name, yet_to_be_built.keys(), builds_to_build_now)) if len(builds_to_build_now) > 0: for build in builds_to_build_now: log.msg( 'change #{}, build {} completed. Creating build for {}'. format(changeid, build_name, build)) yield self.addBuildsetForChanges( reason='because {} completed'.format(build_name), changeids=[changeid], builderNames=[build], properties=Properties.fromDict({ 'changeid': (changeid, 'Scheduler'), 'build': (build, 'Scheduler') })) already_built.add(build) yield self.setState('already_built-' + str(changeid), list(already_built)) else: log.msg('Woohoo! Finished change {}'.format(changeid))
def sendMessage(self, reports): report = reports[0] build = reports[0]['builds'][0] props = Properties.fromDict(build['properties']) props.master = self.master description = report.get('body', None) duration = None test_results = None if build['complete']: state = SUCCESSFUL if build['results'] == SUCCESS else FAILED if self.duration: duration = yield props.render(self.duration) else: td = build['complete_at'] - build['started_at'] duration = int(td.seconds * 1000) if self.test_results: test_results = yield props.render(self.test_results) else: state = INPROGRESS duration = None parent_name = (build['parentbuilder'] or {}).get('name') if self.parent_name: parent = yield props.render(self.parent_name) elif parent_name: parent = parent_name else: parent = build['builder']['name'] if self.status_name: status_name = yield props.render(self.status_name) else: status_name = f'{props.getProperty("buildername")} #{props.getProperty("buildnumber")}' if parent_name: status_name = \ f"{parent_name} #{build['parentbuild']['number']} \u00BB {status_name}" if self.status_suffix: status_name = status_name + (yield props.render( self.status_suffix)) key = yield props.render(self.key) build_number = yield props.render(self.build_number) url = build['url'] sourcestamps = build['buildset']['sourcestamps'] for sourcestamp in sourcestamps: try: ssid = sourcestamp.get('ssid') sha = sourcestamp.get('revision') branch = sourcestamp.get('branch') repo = sourcestamp.get('repository') if not sha: log.msg(f"Unable to get the commit hash for SSID: {ssid}") continue ref = None if self.ref is None: if branch is not None: if branch.startswith("refs/"): ref = branch else: ref = f"refs/heads/{branch}" else: ref = yield props.render(self.ref) if not ref: log.msg( f"WARNING: Unable to resolve ref for SSID: {ssid}. " "Build status will not be visible on Builds or " "PullRequest pages only for commits") r = re.search(r"^.*?/([^/]+)/([^/]+?)(?:\.git)?$", repo or "") if r: proj_key = r.group(1) repo_slug = r.group(2) else: log.msg( f"Unable to parse repository info from '{repo}' for SSID: {ssid}" ) continue res = yield self.createStatus(proj_key=proj_key, repo_slug=repo_slug, sha=sha, state=state, url=url, key=key, parent=parent, build_number=build_number, ref=ref, description=description, name=status_name, duration=duration, test_results=test_results) if res.code not in (HTTP_PROCESSED, ): content = yield res.content() log.msg( f"{res.code}: Unable to send Bitbucket Server status for " f"{proj_key}/{repo_slug} {sha}: {content}") elif self.verbose: log.msg( f'Status "{state}" sent for {proj_key}/{repo_slug} {sha}' ) except Exception as e: log.err( e, f'Failed to send status "{state}" for {proj_key}/{repo_slug} {sha}' )
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failed', SKIPPED: 'success', EXCEPTION: 'failed', RETRY: 'pending', CANCELLED: 'cancelled' }.get(build['results'], 'failed') description = yield props.render(self.endDescription) else: state = 'running' description = yield props.render(self.startDescription) context = yield props.render(self.context) sourcestamps = build['buildset']['sourcestamps'] for sourcestamp in sourcestamps: sha = sourcestamp['revision'] proj_id = yield self.getProjectId(sourcestamp) if proj_id is None: continue try: branch = unicode2NativeString(sourcestamp['branch']) sha = unicode2NativeString(sha) state = unicode2NativeString(state) target_url = unicode2NativeString(build['url']) context = unicode2NativeString(context) description = unicode2NativeString(description) res = yield self.createStatus( project_id=proj_id, branch=branch, sha=sha, state=state, target_url=target_url, context=context, description=description ) if res.code not in (200, 201, 204): message = yield res.json() message = message.get('message', 'unspecified error') log.msg( 'Could not send status "{state}" for ' '{repo} at {sha}: {message}'.format( state=state, repo=sourcestamp['repository'], sha=sha, message=message)) elif self.verbose: log.msg( 'Status "{state}" sent for ' '{repo} at {sha}.'.format( state=state, repo=sourcestamp['repository'], sha=sha)) except Exception as e: log.err( e, 'Failed to send status "{state}" for ' '{repo} at {sha}'.format( state=state, repo=sourcestamp['repository'], sha=sha ))
def render_message_subject(self, context): props = Properties.fromDict(context['build']['properties']) props.master = context['master'] body = yield props.render(self.subject) return body
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) elif self.startDescription: state = 'pending' description = yield props.render(self.startDescription) else: return context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps or not sourcestamps[0]: return project = sourcestamps[0]['project'] branch = props['branch'] m = re.search(r"refs/pull/([0-9]*)/merge", branch) if m: issue = m.group(1) else: issue = None if project: repoOwner, repoName = project.split('/') else: repo = sourcestamps[0]['repository'].split('/')[-2:] repoOwner = repo[0] repoName = '.'.join(repo[1].split('.')[:-1]) for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: repo_user = unicode2NativeString(repoOwner) repo_name = unicode2NativeString(repoName) sha = unicode2NativeString(sha) state = unicode2NativeString(state) target_url = unicode2NativeString(build['url']) context = unicode2NativeString(context) issue = unicode2NativeString(issue) description = unicode2NativeString(description) yield self.createStatus( repo_user=repo_user, repo_name=repo_name, sha=sha, state=state, target_url=target_url, context=context, issue=issue, description=description ) if self.verbose: log.msg( 'Updated status with "{state}" for' '{repoOwner}/{repoName} at {sha}, issue {issue}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue)) except Exception as e: log.err( e, 'Failed to update "{state}" for ' '{repoOwner}/{repoName} at {sha}, issue {issue}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue))
def createEmail(self, msgdict, builderName, title, results, builds=None, patches=None, logs=None): text = msgdict["body"].encode(ENCODING) type = msgdict["type"] if "subject" in msgdict: subject = msgdict["subject"].encode(ENCODING) else: subject = self.subject % { "result": Results[results], "projectName": title, "title": title, "builder": builderName, } assert "\n" not in subject, "Subject cannot contain newlines" assert type in ("plain", "html"), "'%s' message type must be 'plain' or 'html'." % type if patches or logs: m = MIMEMultipart() txt = MIMEText(text, type, ENCODING) m.attach(txt) else: m = Message() m.set_payload(text, ENCODING) m.set_type("text/%s" % type) m["Date"] = formatdate(localtime=True) m["Subject"] = subject m["From"] = self.fromaddr # m['To'] is added later if patches: for (i, patch) in enumerate(patches): a = self.patch_to_attachment(patch, i) m.attach(a) if logs: for log in logs: name = "%s.%s" % (log["stepname"], log["name"]) if self._shouldAttachLog(log["name"]) or self._shouldAttachLog(name): # Use distinct filenames for the e-mail summary if self.buildSetSummary: filename = "%s.%s" % (log["buildername"], name) else: filename = name text = log["content"]["content"] a = MIMEText(text.encode(ENCODING), _charset=ENCODING) a.add_header("Content-Disposition", "attachment", filename=filename) m.attach(a) # @todo: is there a better way to do this? # Add any extra headers that were requested, doing WithProperties # interpolation if only one build was given if self.extraHeaders: extraHeaders = self.extraHeaders if len(builds) == 1: props = Properties.fromDict(builds[0]["properties"]) extraHeaders = yield props.render(extraHeaders) for k, v in iteritems(extraHeaders): if k in m: twlog.msg( "Warning: Got header " + k + " in self.extraHeaders " "but it already exists in the Message - " "not adding it." ) m[k] = v defer.returnValue(m)