def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None,
reason='', external_idstring=None, properties=None,
builderNames=None, **kw):
if sourcestamps is None:
sourcestamps = []
# combine properties
if properties:
properties.updateFromProperties(self.properties)
else:
properties = self.properties
# make a fresh copy that we actually can modify safely
properties = Properties.fromDict(properties.asDict())
# make extra info available from properties.render()
properties.master = self.master
properties.sourcestamps = []
properties.changes = []
for ss in sourcestamps:
if isinstance(ss, int):
# fetch actual sourcestamp and changes from data API
properties.sourcestamps.append(
(yield self.master.data.get(('sourcestamps', ss))))
properties.changes.extend(
(yield self.master.data.get(('sourcestamps', ss, 'changes'))))
else:
# sourcestamp with no change, see addBuildsetForChanges
properties.sourcestamps.append(ss)
for c in properties.changes:
properties.updateFromProperties(Properties.fromDict(c['properties']))
# apply the default builderNames
if not builderNames:
builderNames = self.builderNames
# dynamically get the builder list to schedule
builderNames = yield properties.render(builderNames)
# Get the builder ids
# Note that there is a data.updates.findBuilderId(name)
# but that would merely only optimize the single builder case, while
# probably the multiple builder case will be severely impacted by the
# several db requests needed.
builderids = []
for bldr in (yield self.master.data.get(('builders', ))):
if bldr['name'] in builderNames:
builderids.append(bldr['builderid'])
# translate properties object into a dict as required by the
# addBuildset method
properties_dict = yield properties.render(properties.asDict())
bsid, brids = yield self.master.data.updates.addBuildset(
scheduler=self.name, sourcestamps=sourcestamps, reason=reason,
waited_for=waited_for, properties=properties_dict, builderids=builderids,
external_idstring=external_idstring, **kw)
return (bsid, brids)
def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None,
reason='', external_idstring=None, properties=None,
builderNames=None, **kw):
if sourcestamps is None:
sourcestamps = []
# combine properties
if properties:
properties.updateFromProperties(self.properties)
else:
properties = self.properties
# make a fresh copy that we actually can modify safely
properties = Properties.fromDict(properties.asDict())
# make extra info available from properties.render()
properties.master = self.master
properties.sourcestamps = []
properties.changes = []
for ss in sourcestamps:
if isinstance(ss, int):
# fetch actual sourcestamp and changes from data API
properties.sourcestamps.append(
(yield self.master.data.get(('sourcestamps', ss))))
properties.changes.extend(
(yield self.master.data.get(('sourcestamps', ss, 'changes'))))
else:
# sourcestamp with no change, see addBuildsetForChanges
properties.sourcestamps.append(ss)
for c in properties.changes:
properties.updateFromProperties(Properties.fromDict(c['properties']))
# apply the default builderNames
if not builderNames:
builderNames = self.builderNames
# dynamically get the builder list to schedule
builderNames = yield properties.render(builderNames)
# Get the builder ids
# Note that there is a data.updates.findBuilderId(name)
# but that would merely only optimize the single builder case, while
# probably the multiple builder case will be severely impacted by the
# several db requests needed.
builderids = list()
for bldr in (yield self.master.data.get(('builders', ))):
if bldr['name'] in builderNames:
builderids.append(bldr['builderid'])
# translate properties object into a dict as required by the
# addBuildset method
properties_dict = properties.asDict()
bsid, brids = yield self.master.data.updates.addBuildset(
scheduler=self.name, sourcestamps=sourcestamps, reason=reason,
waited_for=waited_for, properties=properties_dict, builderids=builderids,
external_idstring=external_idstring, **kw)
return (bsid, brids)
async def render_success(self, build, master):
# extract logs named as `result`
results = {}
for step, log_lines in self.extract_logs(build, logname='result'):
if step['results'] == SUCCESS:
results[step['stepid']] = (line for _, line in log_lines)
# render the crossbow repo, becuase it might be passed as a Property
props = Properties.fromDict(build['properties'])
props.master = master
crossbow_repo = await props.render(self.crossbow_repo)
if not isinstance(crossbow_repo, str):
raise ValueError('crossbow_repo argument must be a string')
try:
# decode yaml objects and render the results as a github links
# pointing to the pushed crossbow branches
messages = [
self._render_message(yaml_lines, crossbow_repo=crossbow_repo)
for yaml_lines in results.values()
]
except Exception as e:
log.error(e)
raise
context = '\n\n'.join(messages)
return dict(status='has been succeeded', context=context)
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
else:
state = 'pending'
description = yield props.render(self.startDescription)
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps or not sourcestamps[0]:
return
project = sourcestamps[0]['project']
if project:
repoOwner, repoName = project.split('/')
else:
repo = sourcestamps[0]['repository'].split('/')[-2:]
repoOwner = repo[0]
repoName = '.'.join(repo[1].split('.')[:-1])
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
yield self.createStatus(
repo_user=repoOwner.encode('utf-8'),
repo_name=repoName.encode('utf-8'),
sha=sha.encode('utf-8'),
state=state.encode('utf-8'),
target_url=build['url'].encode('utf-8'),
context=context.encode('utf-8'),
description=description.encode('utf-8'))
if self.verbose:
log.msg('Status "{state}" sent for '
'{repoOwner}/{repoName} at {sha}.'.format(
state=state,
repoOwner=repoOwner,
repoName=repoName,
sha=sha))
except Exception as e:
log.err(
e, 'Fail to send status "{state}" for '
'{repoOwner}/{repoName} at {sha}'.format(
state=state,
repoOwner=repoOwner,
repoName=repoName,
sha=sha))
def start_instance(self, build):
if self.instance is not None:
raise ValueError('instance active')
# pylint: disable=invalid-name
masterFQDN = self.masterFQDN # noqa: N806
# pylint: disable=invalid-name
masterPort = '9989' # noqa: N806
if self.registration is not None:
# pylint: disable=invalid-name
masterPort = str(self.registration.getPBPort()) # noqa: N806
if ":" in masterFQDN:
masterFQDN, masterPort = masterFQDN.split(':')
master_properties = Properties.fromDict({
'masterHash': (self.masterhash, self.properties_source),
'masterFQDN': (masterFQDN, self.properties_source),
'masterPort': (masterPort, self.properties_source),
'workerName': (self.name, self.properties_source),
'workerPass': (self.password, self.properties_source)
})
build.properties.updateFromProperties(master_properties)
namespace = yield build.render(self.namespace)
job = yield build.render(self.job)
res = yield threads.deferToThread(
self._thd_start_instance,
namespace,
job
)
defer.returnValue(res)
def sendMessage(self,
body,
subject=None,
type=None,
builderName=None,
results=None,
builds=None,
users=None,
patches=None,
logs=None,
worker=None):
pr_urls = set()
for build in builds:
props = Properties.fromDict(build['properties'])
pr_urls.add(props.getProperty("pullrequesturl"))
for pr_url in pr_urls:
try:
res = yield self.sendComment(pr_url=pr_url, text=body)
if res.code not in (HTTP_CREATED, ):
content = yield res.content()
log.msg("{code}: Unable to send a comment: "
"{content}".format(code=res.code, content=content))
elif self.verbose:
log.msg('Comment sent to {url}'.format(url=pr_url))
except Exception as e:
log.err(e, 'Failed to send a comment to "{}"'.format(pr_url))
def send(self, build):
props = Properties.fromDict(build['properties'])
results = build['results']
if build['complete']:
status = STASH_SUCCESSFUL if results == SUCCESS else STASH_FAILED
description = self.endDescription
else:
status = STASH_INPROGRESS
description = self.startDescription
for sourcestamp in build['buildset']['sourcestamps']:
sha = sourcestamp['revision']
key = yield props.render(self.key)
payload = {
'state': status,
'url': build['url'],
'key': key,
}
if description:
payload['description'] = yield props.render(description)
if self.statusName:
payload['name'] = yield props.render(self.statusName)
response = yield self._http.post('/rest/build-status/1.0/commits/' + sha,
json=payload)
if response.code == 204:
if self.verbose:
log.info('Status "{status}" sent for {sha}.',
status=status, sha=sha)
else:
content = yield response.content()
log.error("{code}: Unable to send Stash status: {content}",
code=response.code, content=content)
def test_setBuildProperties(self):
self.master.db.insertTestData([
fakedb.Buildset(id=28),
fakedb.BuildRequest(id=5, buildsetid=28),
fakedb.Master(id=3),
fakedb.Worker(id=42, name="Friday"),
fakedb.Build(id=1234, buildrequestid=5, masterid=3, workerid=42),
])
self.master.db.builds.setBuildProperty = mock.Mock(wraps=self.master.db.builds.setBuildProperty)
props = processProperties.fromDict(dict(a=(1, 't'), b=(['abc', 9], 't')))
yield self.rtype.setBuildProperties(1234, props)
self.master.db.builds.setBuildProperty.assert_has_calls([
mock.call(1234, u'a', 1, u't'),
mock.call(1234, u'b', ['abc', 9], u't')])
self.master.mq.assertProductions([
(('builds', '1234', 'properties', 'update'), {u'a': (1, u't'), u'b': (['abc', 9], u't')}),
])
# sync without changes: no db write
self.master.db.builds.setBuildProperty.reset_mock()
self.master.mq.clearProductions()
yield self.rtype.setBuildProperties(1234, props)
self.master.db.builds.setBuildProperty.assert_not_called()
self.master.mq.assertProductions([])
# sync with one changes: one db write
props.setProperty('b', 2, 'step')
self.master.db.builds.setBuildProperty.reset_mock()
yield self.rtype.setBuildProperties(1234, props)
self.master.db.builds.setBuildProperty.assert_called_with(1234, u'b', 2, u'step')
self.master.mq.assertProductions([
(('builds', '1234', 'properties', 'update'), {u'b': (2, u'step')})
])
def send(self, build):
props = Properties.fromDict(build['properties'])
results = build['results']
if build['complete']:
status = STASH_SUCCESSFUL if results == SUCCESS else STASH_FAILED
description = self.endDescription
else:
status = STASH_INPROGRESS
description = self.startDescription
for sourcestamp in build['buildset']['sourcestamps']:
sha = sourcestamp['revision']
key = yield props.render(self.key)
payload = {
'state': status,
'url': build['url'],
'key': key,
}
if description:
payload['description'] = yield props.render(description)
if self.statusName:
payload['name'] = yield props.render(self.statusName)
response = yield self._http.post(
'/rest/build-status/1.0/commits/' + sha, json=payload)
if response.code == 204:
if self.verbose:
log.info('Status "{status}" sent for {sha}.',
status=status,
sha=sha)
else:
content = yield response.content()
log.error("{code}: Unable to send Stash status: {content}",
code=response.code,
content=content)
def sendMessage(self, reports):
body = merge_reports_prop(reports, 'body')
builds = merge_reports_prop(reports, 'builds')
pr_urls = set()
for build in builds:
props = Properties.fromDict(build['properties'])
pr_urls.add(props.getProperty("pullrequesturl"))
for pr_url in pr_urls:
if pr_url is None:
continue
try:
res = yield self.sendComment(
pr_url=pr_url,
text=body
)
if res.code not in (HTTP_CREATED,):
content = yield res.content()
log.msg("{code}: Unable to send a comment: "
"{content}".format(code=res.code, content=content))
elif self.verbose:
log.msg('Comment sent to {url}'.format(url=pr_url))
except Exception as e:
log.err(e, 'Failed to send a comment to "{}"'.format(pr_url))
def gotChange(self, change, important):
if '#nobuild' in change.comments:
print 'ignoring this change because it has #nobuild in the commit message'
return
changeid = change.number
log.msg('got change for change {}'.format(changeid))
to_build = transitive_closure(self.dependency_tree,
self.builds_from_change(change))
yield self.setState('to_build-' + str(changeid),
dependency_tree.serialize(to_build))
already_built = set()
yield self.setState('already_built-' + str(changeid), [])
builds_to_build_now = builds_that_can_be_built(to_build, already_built)
if len(builds_to_build_now) == 0:
return
log.msg('initially building builds {} in change {}'.format(
builds_to_build_now, changeid))
for build in builds_to_build_now:
yield self.addBuildsetForChanges(reason='because of change',
changeids=[changeid],
builderNames=[build],
properties=Properties.fromDict({
'changeid':
(changeid, 'Scheduler'),
'build': (build, 'Scheduler')
}))
def gotChange(self, change, important):
if not self.treeStableTimer:
# if there's no treeStableTimer, we can completely ignore
# unimportant changes
if not important:
return defer.succeed(None)
properties = change.properties
if(type(properties) == dict):
properties = Properties.fromDict(properties)
# otherwise, we'll build it right away
return self.addBuildsetForChanges(reason=self.reason,
properties=properties,
changeids=[change.number])
timer_name = self.getTimerNameForChange(change)
# if we have a treeStableTimer
# - for an important change, start the timer
# - for an unimportant change, reset the timer if it is running
if important or self._stable_timers[timer_name]:
if self._stable_timers[timer_name]:
self._stable_timers[timer_name].cancel()
def fire_timer():
d = self.stableTimerFired(timer_name)
d.addErrback(log.err, "while firing stable timer")
self._stable_timers[timer_name] = self._reactor.callLater(
self.treeStableTimer, fire_timer)
# record the change's importance
return self.master.db.schedulers.classifyChanges(
self.objectid, {change.number: important})
def sendMessage(self, reports):
report = reports[0]
build = reports[0]['builds'][0]
props = Properties.fromDict(build['properties'])
props.master = self.master
description = report.get('body', None)
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failed',
SKIPPED: 'success',
EXCEPTION: 'failed',
RETRY: 'pending',
CANCELLED: 'cancelled'
}.get(build['results'], 'failed')
else:
state = 'running'
context = yield props.render(self.context)
sourcestamps = build['buildset']['sourcestamps']
# FIXME: probably only want to report status for the last commit in the changeset
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
if 'source_project_id' in props:
proj_id = props['source_project_id']
else:
proj_id = yield self.getProjectId(sourcestamp)
if proj_id is None:
continue
try:
if 'source_branch' in props:
branch = props['source_branch']
else:
branch = sourcestamp['branch']
target_url = build['url']
res = yield self.createStatus(project_id=proj_id,
branch=branch,
sha=sha,
state=state,
target_url=target_url,
context=context,
description=description)
if res.code not in (200, 201, 204):
message = yield res.json()
message = message.get('message', 'unspecified error')
log.msg(f'Could not send status "{state}" for '
f'{sourcestamp["repository"]} at {sha}: {message}')
elif self.verbose:
log.msg(f'Status "{state}" sent for '
f'{sourcestamp["repository"]} at {sha}.')
except Exception as e:
log.err(e, (f'Failed to send status "{state}" for '
f'{sourcestamp["repository"]} at {sha}'))
def send(self, build):
props = Properties.fromDict(build['properties'])
props.master = self.master
results = build['results']
if build['complete']:
state = SUCCESSFUL if results == SUCCESS else FAILED
description = self.endDescription
else:
state = INPROGRESS
description = self.startDescription
key = yield props.render(self.key)
description = yield props.render(description) if description else None
context = yield props.render(self.context) if self.context else None
sourcestamps = build['buildset']['sourcestamps']
for sourcestamp in sourcestamps:
try:
sha = unicode2NativeString(sourcestamp['revision'])
if sha is None:
log.msg("Unable to get the commit hash")
continue
key = unicode2NativeString(key)
state = unicode2NativeString(state)
url = unicode2NativeString(build['url'])
key = unicode2NativeString(key)
description = unicode2NativeString(description)
context = unicode2NativeString(context)
res = yield self.createStatus(
sha=sha,
state=state,
url=url,
key=key,
description=description,
context=context
)
if res.code not in (HTTP_PROCESSED,):
content = yield res.content()
log.msg("{code}: Unable to send Bitbucket Server status: "
"{content}".format(code=res.code, content=content))
elif self.verbose:
log.msg('Status "{state}" sent for {sha}.'.format(
state=state, sha=sha))
except Exception as e:
log.err(
e,
'Failed to send status "{state}" for '
'{repo} at {sha}'.format(
state=state,
repo=sourcestamp['repository'], sha=sha
))
async def send(self, build):
# License note:
# It is a reimplementation based on the parent GitHubStatusPush
# from the original buildbot implementation.
#
# We must propagate the build down to the renderer callbacks
# (endDescription, startDescription), otherwise there is no way to
# retrieve the build and its logs.
#
# Only `buildername` and `builnumber` properties are set, but
# data.get(('builders', buildername, 'builds', buildnumber)) raises
# for non-alphanumerical builder name:
# Invalid path: builders/Ursabot Python 3.7/builds/2
# So the official[quiet twisted] example wouldn't work:
# http://docs.buildbot.net/2.3.0/full.html#githubcommentpush
cls = self.__class__.__name__
build_number = build['number']
builder_name = build['builder']['name']
sourcestamps = build['buildset'].get('sourcestamps', [])
properties = Properties.fromDict(build['properties'])
for sourcestamp in sourcestamps:
project = sourcestamp.get('project')
repository = sourcestamp.get('repository')
if self.verbose:
log.info(f'Triggering {cls}.report() for project {project}, '
f'repository {repository}, builder {builder_name}, '
f'build number {build_number}')
try:
response = await self.report(build, sourcestamp, properties)
except Exception as e:
log.error(e)
raise e
# report() can return None to skip reporting
if response is None:
continue
if not self.isStatus2XX(response.code):
content = await response.content()
e = Exception(
f'Failed to execute http API call in {cls}.report() for '
f'repository {repository}, builder {builder_name}, build '
f'number {build_number} with error code {response.code} '
f'and response "{content}"')
log.error(e)
raise e
if self.verbose:
log.info(f'Successful report {cls}.report() for repository '
f'{repository}, builder {builder_name}, build number '
f'{build_number}')
def test_gerrit_changes(self):
yield self.createGerritStatus()
# from chdict:
chdict = TestGerritChangeSource.expected_change
props = Properties.fromDict(
{k: (v, 'change')
for k, v in chdict['properties'].items()})
changes = self.sp.getGerritChanges(props)
self.assertEqual(changes, [{'change_id': '4321', 'revision_id': '12'}])
def test_gerrit_changes(self):
yield self.createGerritStatus()
# from chdict:
chdict = TestGerritChangeSource.expected_change
props = Properties.fromDict({
k: (v, 'change') for k, v in chdict['properties'].items()})
changes = self.sp.getGerritChanges(props)
self.assertEqual(changes, [
{'change_id': '4321', 'revision_id': '12'}
])
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
else:
state = 'pending'
description = yield props.render(self.startDescription)
context = yield props.render(self.context)
sourcestamps = build['buildset']['sourcestamps']
project = sourcestamps[0]['project']
if project:
repoOwner, repoName = project.split('/')
else:
repo = sourcestamps[0]['repository'].split('/')[-2:]
repoOwner = repo[0]
repoName = '.'.join(repo[1].split('.')[:-1])
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
yield self.createStatus(
repo_user=repoOwner.encode('utf-8'),
repo_name=repoName.encode('utf-8'),
sha=sha.encode('utf-8'),
state=state.encode('utf-8'),
target_url=build['url'].encode('utf-8'),
context=context.encode('utf-8'),
description=description.encode('utf-8')
)
if self.verbose:
log.msg(
'Status "{state}" sent for '
'{repoOwner}/{repoName} at {sha}.'.format(
state=state, repoOwner=repoOwner, repoName=repoName, sha=sha))
except Exception as e:
log.err(
e,
'Fail to send status "{state}" for '
'{repoOwner}/{repoName} at {sha}'.format(
state=state, repoOwner=repoOwner, repoName=repoName, sha=sha))
def sendMessage(self, reports):
# Only use OAuth if basic auth has not been specified
if not self.auth:
request = yield self.oauthhttp.post("", data=_GET_TOKEN_DATA)
if request.code != 200:
content = yield request.content()
log.msg(
f"{request.code}: unable to authenticate to Bitbucket {content}"
)
return
token = (yield request.json())['access_token']
self._http.updateHeaders({'Authorization': f'Bearer {token}'})
build = reports[0]['builds'][0]
if build['complete']:
status = BITBUCKET_SUCCESSFUL if build[
'results'] == SUCCESS else BITBUCKET_FAILED
else:
status = BITBUCKET_INPROGRESS
props = Properties.fromDict(build['properties'])
props.master = self.master
body = {
'state': status,
'key': (yield props.render(self.status_key)),
'name': (yield props.render(self.status_name)),
'description': reports[0]['subject'],
'url': build['url']
}
for sourcestamp in build['buildset']['sourcestamps']:
if not sourcestamp['repository']:
log.msg(f"Empty repository URL for Bitbucket status {body}")
continue
owner, repo = self.get_owner_and_repo(sourcestamp['repository'])
endpoint = (owner, repo, 'commit', sourcestamp['revision'],
'statuses', 'build')
bitbucket_uri = f"/{'/'.join(endpoint)}"
if self.debug:
log.msg(f"Bitbucket status {bitbucket_uri} {body}")
response = yield self._http.post(bitbucket_uri, json=body)
if response.code not in (200, 201):
content = yield response.content()
log.msg(
f"{response.code}: unable to upload Bitbucket status {content}"
)
def sendMessage(self, reports):
report = reports[0]
build = reports[0]['builds'][0]
props = Properties.fromDict(build['properties'])
props.master = self.master
description = report.get('body', None)
results = build['results']
if build['complete']:
state = SUCCESSFUL if results == SUCCESS else FAILED
else:
state = INPROGRESS
key = yield props.render(self.key)
context = yield props.render(self.context) if self.context else None
sourcestamps = build['buildset']['sourcestamps']
for sourcestamp in sourcestamps:
try:
sha = sourcestamp['revision']
if sha is None:
log.msg("Unable to get the commit hash")
continue
url = build['url']
res = yield self.createStatus(sha=sha,
state=state,
url=url,
key=key,
description=description,
context=context)
if res.code not in (HTTP_PROCESSED, ):
content = yield res.content()
log.msg("{code}: Unable to send Bitbucket Server status: "
"{content}".format(code=res.code, content=content))
elif self.verbose:
log.msg('Status "{state}" sent for {sha}.'.format(
state=state, sha=sha))
except Exception as e:
log.err(
e, 'Failed to send status "{state}" for '
'{repo} at {sha}'.format(state=state,
repo=sourcestamp['repository'],
sha=sha))
def test_setBuildProperties(self):
self.master.db.insertTestData([
fakedb.Buildset(id=28),
fakedb.BuildRequest(id=5, buildsetid=28),
fakedb.Master(id=3),
fakedb.Buildslave(id=42, name="Friday"),
fakedb.Build(id=1234,
buildrequestid=5,
masterid=3,
buildslaveid=42),
])
self.master.db.builds.setBuildProperty = mock.Mock(
wraps=self.master.db.builds.setBuildProperty)
props = processProperties.fromDict(
dict(a=(1, 't'), b=(['abc', 9], 't')))
yield self.rtype.setBuildProperties(1234, props)
self.master.db.builds.setBuildProperty.assert_has_calls([
mock.call(1234, u'a', 1, u't'),
mock.call(1234, u'b', ['abc', 9], u't')
])
self.master.mq.assertProductions([
(('builds', '1234', 'properties', 'update'), {
u'a': (1, u't'),
u'b': (['abc', 9], u't')
}),
])
# sync without changes: no db write
self.master.db.builds.setBuildProperty.reset_mock()
self.master.mq.clearProductions()
yield self.rtype.setBuildProperties(1234, props)
self.master.db.builds.setBuildProperty.assert_not_called()
self.master.mq.assertProductions([])
# sync with one changes: one db write
props.setProperty('b', 2, 'step')
self.master.db.builds.setBuildProperty.reset_mock()
yield self.rtype.setBuildProperties(1234, props)
self.master.db.builds.setBuildProperty.assert_called_with(
1234, u'b', 2, u'step')
self.master.mq.assertProductions([(('builds', '1234', 'properties',
'update'), {
u'b': (2, u'step')
})])
def sendMessage(self, reports):
report = reports[0]
build = reports[0]['builds'][0]
props = Properties.fromDict(build['properties'])
props.master = self.master
comment = report.get('body', None)
if build['complete']:
value = self.RESULTS_TABLE.get(build['results'],
self.DEFAULT_RESULT)
duration = self.formatDuration(build['complete_at'] -
build['started_at'])
else:
value = 0
duration = 'pending'
name = yield props.render(self._verification_name)
reporter = yield props.render(self._reporter)
category = yield props.render(self._category)
abstain = yield props.render(self._abstain)
# TODO: find reliable way to find out whether its a rebuild
rerun = None
changes = yield self.getGerritChanges(props)
for change in changes:
try:
yield self.createStatus(change['change_id'],
change['revision_id'],
name,
value,
abstain=abstain,
rerun=rerun,
comment=comment,
url=build['url'],
reporter=reporter,
category=category,
duration=duration)
except Exception:
log.failure('Failed to send status!',
failure=failure.Failure())
def sendMessage(self, body, subject=None, type=None, builderName=None,
results=None, builds=None, users=None, patches=None,
logs=None, worker=None):
pr_urls = set()
for build in builds:
props = Properties.fromDict(build['properties'])
pr_urls.add(props.getProperty("pullrequesturl"))
for pr_url in pr_urls:
try:
res = yield self.sendComment(
pr_url=pr_url,
text=body
)
if res.code not in (HTTP_CREATED,):
content = yield res.content()
log.msg("{code}: Unable to send a comment: "
"{content}".format(code=res.code, content=content))
elif self.verbose:
log.msg('Comment sent to {url}'.format(url=pr_url))
except Exception as e:
log.err(e, 'Failed to send a comment to "{}"'.format(pr_url))
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
value = self.RESULTS_TABLE.get(build['results'],
self.DEFAULT_RESULT)
comment = yield props.render(self._endDescription)
duration = self.formatDuration(build['complete_at'] - build[
'started_at'])
else:
value = 0
comment = yield props.render(self._startDescription)
duration = 'pending'
name = yield props.render(self._verification_name)
reporter = yield props.render(self._reporter)
category = yield props.render(self._category)
abstain = yield props.render(self._abstain)
# TODO: find reliable way to find out whether its a rebuild
rerun = None
changes = yield self.getGerritChanges(props)
for change in changes:
try:
yield self.createStatus(
change['change_id'],
change['revision_id'],
name,
value,
abstain=abstain,
rerun=rerun,
comment=comment,
url=build['url'],
reporter=reporter,
category=category,
duration=duration)
except Exception:
log.failure(
'Failed to send status!', failure=failure.Failure())
def test_gerrit_changes(self):
yield self.createGerritStatus()
# from chdict:
change_props = {
'event.change.owner.email': '*****@*****.**',
'event.change.subject': 'fix 1234',
'event.change.project': 'pr',
'event.change.owner.name': 'Dustin',
'event.change.number': '4321',
'event.change.url': 'http://buildbot.net',
'event.change.branch': 'br',
'event.type': 'patchset-created',
'event.patchSet.revision': 'abcdef',
'event.patchSet.number': '12',
'event.source': 'GerritChangeSource'
}
props = Properties.fromDict({k: (v, 'change') for k, v in change_props.items()})
changes = self.sp.getGerritChanges(props)
self.assertEqual(changes, [
{'change_id': '4321', 'revision_id': '12'}
])
def sendMessage(self, reports):
report = reports[0]
build = reports[0]['builds'][0]
props = Properties.fromDict(build['properties'])
props.master = self.master
description = report.get('body', None)
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
else:
state = 'pending'
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps:
return
issue = self._extract_issue(props)
for sourcestamp in sourcestamps:
repo_owner, repo_name = self._extract_github_info(sourcestamp)
if not repo_owner or not repo_name:
log.msg(
'Skipped status update because required repo information is missing.'
)
continue
sha = sourcestamp['revision']
response = None
# If the scheduler specifies multiple codebases, don't bother updating
# the ones for which there is no revision
if not sha:
log.msg(
f"Skipped status update for codebase {sourcestamp['codebase']}, "
f"context '{context}', issue {issue}.")
continue
try:
if self.verbose:
log.msg(
f"Updating github status: repo_owner={repo_owner}, repo_name={repo_name}"
)
response = yield self.createStatus(repo_user=repo_owner,
repo_name=repo_name,
sha=sha,
state=state,
target_url=build['url'],
context=context,
issue=issue,
description=description)
if not response:
# the implementation of createStatus refused to post update due to missing data
continue
if not self.is_status_2xx(response.code):
raise Exception()
if self.verbose:
log.msg(
f'Updated status with "{state}" for {repo_owner}/{repo_name} '
f'at {sha}, context "{context}", issue {issue}.')
except Exception as e:
if response:
content = yield response.content()
code = response.code
else:
content = code = "n/a"
log.err(e, (
f'Failed to update "{state}" for {repo_owner}/{repo_name} '
f'at {sha}, context "{context}", issue {issue}. '
f'http {code}, {content}'))
def send(self, build):
props = Properties.fromDict(build["properties"])
props.master = self.master
if build["complete"]:
state = {
SUCCESS: "success",
WARNINGS: "success",
FAILURE: "failure",
SKIPPED: "success",
EXCEPTION: "error",
RETRY: "pending",
CANCELLED: "error",
}.get(build["results"], "error")
else:
return
if state != "failure":
return
yield getDetailsForBuild(self.master, build, wantLogs=True, wantSteps=True)
tracebacks = list()
try:
test_log = build["steps"][TESTS_STEP]["logs"][0]["content"]["content"]
test_log = "\n".join([line.lstrip("eo") for line in test_log.splitlines()])
tracebacks = TRACEBACK_REGEX.findall(test_log)
except IndexError:
pass
if not tracebacks:
tracebacks = list(self._construct_tracebacks_from_stderr(build))
context = yield props.render(self.context)
sourcestamps = build["buildset"].get("sourcestamps")
if not sourcestamps or not sourcestamps[0]:
return
changes = yield self.master.data.get(("builds", build["buildid"], "changes"))
if len(changes) > 1:
return
change, = changes
m = re.search(r"\((?:GH-|#)(\d+)\)", change["comments"])
if m is None:
return
issue = m.groups()[-1]
project = sourcestamps[0]["project"]
if "/" in project:
repoOwner, repoName = project.split("/")
else:
giturl = giturlparse(sourcestamps[0]["repository"])
repoOwner = giturl.owner
repoName = giturl.repo
if self.verbose:
log.msg(
"Updating github status: repoOwner={repoOwner}, repoName={repoName}".format(
repoOwner=repoOwner, repoName=repoName
)
)
try:
repo_user = unicode2NativeString(repoOwner)
repo_name = unicode2NativeString(repoName)
sha = unicode2NativeString(change["revision"])
target_url = unicode2NativeString(build["url"])
context = unicode2NativeString(context)
yield self.createStatus(
build=build,
repo_user=repo_user,
repo_name=repo_name,
sha=sha,
state=state,
target_url=target_url,
context=context,
issue=issue,
tracebacks=tracebacks,
)
if self.verbose:
log.msg(
"Issued a Pull Request comment for {repoOwner}/{repoName} "
'at {sha}, context "{context}", issue {issue}.'.format(
repoOwner=repoOwner,
repoName=repoName,
sha=sha,
issue=issue,
context=context,
)
)
except Exception as e:
log.err(
e,
"Failed to issue a Pull Request comment for {repoOwner}/{repoName} "
'at {sha}, context "{context}", issue {issue}.'.format(
repoOwner=repoOwner,
repoName=repoName,
sha=sha,
issue=issue,
context=context,
),
)
def _send_impl(self, reports):
report = reports[0]
build = report['builds'][0]
props = Properties.fromDict(build['properties'])
props.master = self.master
description = report.get('body', None)
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success' if self.warningAsSuccess else 'warning',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'failure')
else:
state = 'pending'
if 'pr_id' in props:
context = yield props.render(self.context_pr)
else:
context = yield props.render(self.context)
sourcestamps = build['buildset']['sourcestamps']
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
if sha is None:
# No special revision for this, so ignore it
continue
if 'repository_name' in props:
repository_name = props['repository_name']
else:
match = re.match(self.ssh_url_match, sourcestamp['repository'])
if match is not None:
repository_name = match.group("repo_name")
else:
log.msg(
"Could not send status, "
"build has no repository_name property for Gitea.")
continue
if 'owner' in props:
repository_owner = props['owner']
else:
match = re.match(self.ssh_url_match, sourcestamp['repository'])
if match is not None:
repository_owner = match.group("owner")
else:
log.msg(
"Could not send status, "
"build has no owner property for Gitea.")
continue
try:
target_url = build['url']
res = yield self.createStatus(
project_owner=repository_owner,
repo_name=repository_name,
sha=sha,
state=state,
target_url=target_url,
context=context,
description=description
)
if res.code not in (200, 201, 204):
message = yield res.json()
message = message.get('message', 'unspecified error')
log.msg(
'Could not send status "{state}" for '
'{repo} at {sha}: {code} : {message}'.format(
state=state,
repo=sourcestamp['repository'], sha=sha,
code=res.code,
message=message))
elif self.verbose:
log.msg(
'Status "{state}" sent for '
'{repo} at {sha}.'.format(
state=state,
repo=sourcestamp['repository'], sha=sha))
except Exception as e:
log.err(
e,
'Failed to send status "{state}" for '
'{repo} at {sha}'.format(
state=state,
repo=sourcestamp['repository'], sha=sha
))
def send(self, build):
props = Properties.fromDict(build['properties'])
props.master = self.master
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
elif self.startDescription:
state = 'pending'
description = yield props.render(self.startDescription)
else:
return
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps:
return
for sourcestamp in sourcestamps:
project = sourcestamp['project']
issue = None
if 'branch' in props:
m = re.search(r"refs/pull/([0-9]*)/merge", props['branch'])
if m:
issue = m.group(1)
repo_owner = None
repo_name = None
if "/" in project:
repo_owner, repo_name = project.split('/')
else:
giturl = giturlparse(sourcestamp['repository'])
if giturl:
repo_owner = giturl.owner
repo_name = giturl.repo
sha = sourcestamp['revision']
response = None
# If the scheduler specifies multiple codebases, don't bother updating
# the ones for which there is no revision
if not sha:
log.msg('Skipped status update for codebase {codebase}, '
'context "{context}", issue {issue}.'.format(
codebase=sourcestamp['codebase'],
issue=issue,
context=context))
continue
try:
if self.verbose:
log.msg(
"Updating github status: repo_owner={}, repo_name={}".
format(repo_owner, repo_name))
response = yield self.createStatus(repo_user=repo_owner,
repo_name=repo_name,
sha=sha,
state=state,
target_url=build['url'],
context=context,
issue=issue,
description=description)
if not response:
# the implementation of createStatus refused to post update due to missing data
continue
if not self.isStatus2XX(response.code):
raise Exception()
if self.verbose:
log.msg(
'Updated status with "{state}" for {repo_owner}/{repo_name} '
'at {sha}, context "{context}", issue {issue}.'.format(
state=state,
repo_owner=repo_owner,
repo_name=repo_name,
sha=sha,
issue=issue,
context=context))
except Exception as e:
if response:
content = yield response.content()
code = response.code
else:
content = code = "n/a"
log.err(
e,
'Failed to update "{state}" for {repo_owner}/{repo_name} '
'at {sha}, context "{context}", issue {issue}. '
'http {code}, {content}'.format(state=state,
repo_owner=repo_owner,
repo_name=repo_name,
sha=sha,
issue=issue,
context=context,
code=code,
content=content))
def send(self, build):
props = Properties.fromDict(build['properties'])
props.master = self.master
duration = None
test_results = None
if build['complete']:
state = SUCCESSFUL if build['results'] == SUCCESS else FAILED
description = yield props.render(self.end_description)
if self.duration:
duration = yield props.render(self.duration)
else:
td = build['complete_at'] - build['started_at']
duration = int(td.seconds * 1000)
if self.test_results:
test_results = yield props.render(self.test_results)
else:
state = INPROGRESS
description = yield props.render(self.start_description)
duration = None
parent_name = (build['parentbuilder'] or {}).get('name')
if self.parent_name:
parent = yield props.render(self.parent_name)
elif parent_name:
parent = parent_name
else:
parent = build['builder']['name']
if self.status_name:
status_name = yield props.render(self.status_name)
else:
status_name = "{} #{}".format(props.getProperty("buildername"),
props.getProperty("buildnumber"))
if parent_name:
status_name = "{} #{} \u00BB {}".format(
parent_name, build['parentbuild']['number'], status_name)
if self.status_suffix:
status_name = status_name + (yield props.render(
self.status_suffix))
key = yield props.render(self.key)
build_number = yield props.render(self.build_number)
url = build['url']
sourcestamps = build['buildset']['sourcestamps']
for sourcestamp in sourcestamps:
try:
ssid = sourcestamp.get('ssid')
sha = sourcestamp.get('revision')
branch = sourcestamp.get('branch')
repo = sourcestamp.get('repository')
if not sha:
log.msg("Unable to get the commit hash for SSID: "
"{}".format(ssid))
continue
ref = None
if self.ref is None:
if branch is not None:
if branch.startswith("refs/"):
ref = branch
else:
ref = "refs/heads/{}".format(branch)
else:
ref = yield props.render(self.ref)
if not ref:
log.msg("WARNING: Unable to resolve ref for SSID: {}. "
"Build status will not be visible on Builds or "
"PullRequest pages only for commits".format(ssid))
r = re.search(r"^.*?/([^/]+)/([^/]+?)(?:\.git)?$", repo or "")
if r:
proj_key = r.group(1)
repo_slug = r.group(2)
else:
log.msg("Unable to parse repository info from '{}' for "
"SSID: {}".format(repo, ssid))
continue
res = yield self.createStatus(proj_key=proj_key,
repo_slug=repo_slug,
sha=sha,
state=state,
url=url,
key=key,
parent=parent,
build_number=build_number,
ref=ref,
description=description,
name=status_name,
duration=duration,
test_results=test_results)
if res.code not in (HTTP_PROCESSED, ):
content = yield res.content()
log.msg("{}: Unable to send Bitbucket Server status for "
"{}/{} {}: {}".format(res.code, proj_key,
repo_slug, sha, content))
elif self.verbose:
log.msg('Status "{}" sent for {}/{} {}'.format(
state, proj_key, repo_slug, sha))
except Exception as e:
log.err(
e, 'Failed to send status "{}" for {}/{} {}'.format(
state, proj_key, repo_slug, sha))
def createEmail(self,
msgdict,
builderName,
title,
results,
builds=None,
patches=None,
logs=None):
text = msgdict['body']
type = msgdict['type']
subject = msgdict['subject']
assert '\n' not in subject, \
"Subject cannot contain newlines"
assert type in ('plain', 'html'), \
"'{}' message type must be 'plain' or 'html'.".format(type)
if patches or logs:
m = MIMEMultipart()
txt = MIMEText(text, type, ENCODING)
m.attach(txt)
else:
m = Message()
m.set_payload(text, ENCODING)
m.set_type("text/{}".format(type))
m['Date'] = formatdate(localtime=True)
m['Subject'] = subject
m['From'] = self.fromaddr
# m['To'] is added later
if patches:
for (i, patch) in enumerate(patches):
a = self.patch_to_attachment(patch, i)
m.attach(a)
if logs:
for log in logs:
# Use distinct filenames for the e-mail summary
name = "{}.{}".format(log['stepname'], log['name'])
if len(builds) > 1:
filename = "{}.{}".format(log['buildername'], name)
else:
filename = name
text = log['content']['content']
a = MIMEText(text.encode(ENCODING), _charset=ENCODING)
# convert to base64 to conform with RFC 5322 2.1.1
del a['Content-Transfer-Encoding']
encoders.encode_base64(a)
a.add_header('Content-Disposition',
"attachment",
filename=filename)
m.attach(a)
# @todo: is there a better way to do this?
# Add any extra headers that were requested, doing WithProperties
# interpolation if only one build was given
if self.extraHeaders:
extraHeaders = self.extraHeaders
if builds is not None and len(builds) == 1:
props = Properties.fromDict(builds[0]['properties'])
props.master = self.master
extraHeaders = yield props.render(extraHeaders)
for k, v in extraHeaders.items():
if k in m:
twlog.msg("Warning: Got header " + k +
" in self.extraHeaders "
"but it already exists in the Message - "
"not adding it.")
m[k] = v
return m
def send(self, build):
props = Properties.fromDict(build['properties'])
props.master = self.master
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
elif self.startDescription:
state = 'pending'
description = yield props.render(self.startDescription)
else:
return
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps or not sourcestamps[0]:
return
project = sourcestamps[0]['project']
branch = props['branch']
m = re.search(r"refs/pull/([0-9]*)/merge", branch)
if m:
issue = m.group(1)
else:
issue = None
if "/" in project:
repoOwner, repoName = project.split('/')
else:
giturl = giturlparse(sourcestamps[0]['repository'])
repoOwner = giturl.owner
repoName = giturl.repo
if self.verbose:
log.msg("Updating github status: repoOwner={repoOwner}, repoName={repoName}".format(
repoOwner=repoOwner, repoName=repoName))
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
repo_user = repoOwner
repo_name = repoName
target_url = build['url']
response = yield self.createStatus(
repo_user=repo_user,
repo_name=repo_name,
sha=sha,
state=state,
target_url=target_url,
context=context,
issue=issue,
description=description
)
if not self.isStatus2XX(response.code):
raise Exception()
if self.verbose:
log.msg(
'Updated status with "{state}" for {repoOwner}/{repoName} '
'at {sha}, context "{context}", issue {issue}.'.format(
state=state, repoOwner=repoOwner, repoName=repoName,
sha=sha, issue=issue, context=context))
except Exception as e:
content = yield response.content()
log.err(
e,
'Failed to update "{state}" for {repoOwner}/{repoName} '
'at {sha}, context "{context}", issue {issue}. '
'http {code}, {content}'.format(
state=state, repoOwner=repoOwner, repoName=repoName,
sha=sha, issue=issue, context=context,
code=response.code, content=content))
def createEmail(self,
msgdict,
builderName,
title,
results,
builds=None,
patches=None,
logs=None):
text = msgdict['body'].encode(ENCODING)
type = msgdict['type']
if 'subject' in msgdict:
subject = msgdict['subject'].encode(ENCODING)
else:
subject = self.subject % {
'result': Results[results],
'projectName': title,
'title': title,
'builder': builderName
}
assert '\n' not in subject, \
"Subject cannot contain newlines"
assert type in ('plain', 'html'), \
"'%s' message type must be 'plain' or 'html'." % type
if patches or logs:
m = MIMEMultipart()
txt = MIMEText(text, type, ENCODING)
m.attach(txt)
else:
m = Message()
m.set_payload(text, ENCODING)
m.set_type("text/%s" % type)
m['Date'] = formatdate(localtime=True)
m['Subject'] = subject
m['From'] = self.fromaddr
# m['To'] is added later
if patches:
for (i, patch) in enumerate(patches):
a = self.patch_to_attachment(patch, i)
m.attach(a)
if logs:
for log in logs:
name = "%s.%s" % (log['stepname'], log['name'])
if (self._shouldAttachLog(log['name'])
or self._shouldAttachLog(name)):
# Use distinct filenames for the e-mail summary
if self.buildSetSummary:
filename = "%s.%s" % (log['buildername'], name)
else:
filename = name
text = log['content']['content']
a = MIMEText(text.encode(ENCODING), _charset=ENCODING)
a.add_header('Content-Disposition',
"attachment",
filename=filename)
m.attach(a)
# @todo: is there a better way to do this?
# Add any extra headers that were requested, doing WithProperties
# interpolation if only one build was given
if self.extraHeaders:
extraHeaders = self.extraHeaders
if len(builds) == 1:
props = Properties.fromDict(builds[0]['properties'])
extraHeaders = yield props.render(extraHeaders)
for k, v in iteritems(extraHeaders):
if k in m:
twlog.msg("Warning: Got header " + k +
" in self.extraHeaders "
"but it already exists in the Message - "
"not adding it.")
m[k] = v
defer.returnValue(m)
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
elif self.startDescription:
state = 'pending'
description = yield props.render(self.startDescription)
else:
return
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps or not sourcestamps[0]:
return
project = sourcestamps[0]['project']
branch = props['branch']
m = re.search(r"refs/pull/([0-9]*)/merge", branch)
if m:
issue = m.group(1)
else:
issue = None
if "/" in project:
repoOwner, repoName = project.split('/')
else:
giturl = giturlparse(sourcestamps[0]['repository'])
repoOwner = giturl.owner
repoName = giturl.repo
if self.verbose:
log.msg(
"Updating github status: repoOwner={repoOwner}, repoName={repoName}"
.format(repoOwner=repoOwner, repoName=repoName))
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
repo_user = unicode2NativeString(repoOwner)
repo_name = unicode2NativeString(repoName)
sha = unicode2NativeString(sha)
state = unicode2NativeString(state)
target_url = unicode2NativeString(build['url'])
context = unicode2NativeString(context)
issue = unicode2NativeString(issue)
description = unicode2NativeString(description)
yield self.createStatus(repo_user=repo_user,
repo_name=repo_name,
sha=sha,
state=state,
target_url=target_url,
context=context,
issue=issue,
description=description)
if self.verbose:
log.msg('Updated status with "{state}" for '
'{repoOwner}/{repoName} at {sha}, issue {issue}.'.
format(state=state,
repoOwner=repoOwner,
repoName=repoName,
sha=sha,
issue=issue))
except Exception as e:
log.err(
e, 'Failed to update "{state}" for '
'{repoOwner}/{repoName} at {sha}, issue {issue}'.format(
state=state,
repoOwner=repoOwner,
repoName=repoName,
sha=sha,
issue=issue))
def send(self, build):
props = Properties.fromDict(build['properties'])
props.master = self.master
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
elif self.startDescription:
state = 'pending'
description = yield props.render(self.startDescription)
else:
return
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps or not sourcestamps[0]:
return
branch = props['branch']
m = re.search(r"refs/pull/([0-9]*)/merge", branch)
if m:
issue = m.group(1)
else:
# We only want to comment pull requests, so we exit here
return
for sourcestamp in sourcestamps:
if branch == sourcestamp['branch']:
project = sourcestamp['project']
repository = sourcestamp['repository']
sha = sourcestamp['revision']
break
if project is None:
log.err('Failed to determine the project of PR "{branch}"'.format(
branch=branch))
return
if "/" in project:
repoOwner, repoName = project.split('/')
else:
giturl = giturlparse(repository)
repoOwner = giturl.owner
repoName = giturl.repo
if self.verbose:
log.msg("Updating github status: repoOwner={repoOwner}, repoName={repoName}".format(
repoOwner=repoOwner, repoName=repoName))
try:
repo_user = unicode2NativeString(repoOwner)
repo_name = unicode2NativeString(repoName)
sha = unicode2NativeString(sha)
state = unicode2NativeString(state)
target_url = unicode2NativeString(build['url'])
context = unicode2NativeString(context)
issue = unicode2NativeString(issue)
description = unicode2NativeString(description)
yield self.createStatus(
repo_user=repo_user,
repo_name=repo_name,
sha=sha,
state=state,
target_url=target_url,
context=context,
issue=issue,
description=description
)
if self.verbose:
log.msg(
'Updated status with "{state}" for {repoOwner}/{repoName} '
'at {sha}, context "{context}", issue {issue}.'.format(
state=state, repoOwner=repoOwner, repoName=repoName,
sha=sha, issue=issue, context=context))
except Exception as e:
log.err(
e,
'Failed to update "{state}" for {repoOwner}/{repoName} '
'at {sha}, context "{context}", issue {issue}.'.format(
state=state, repoOwner=repoOwner, repoName=repoName,
sha=sha, issue=issue, context=context))
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failed',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
else:
state = 'pending'
description = yield props.render(self.startDescription)
context = yield props.render(self.context)
sourcestamps = build['buildset']['sourcestamps']
project = sourcestamps[0]['project']
# default to master if not found
branch = sourcestamps[0].get('branch', 'master')
if project:
repoOwner, repoName = project.split('/')
else:
repo = sourcestamps[0]['repository'].split('/')[-2:]
repoOwner = repo[0]
repoName = '.'.join(repo[1].split('.')[:-1])
m = re.match(".*:(.*)", repoOwner)
if m is not None:
repoOwner = m.group(1)
# retrieve project id via cache
self.project_ids
project_full_name = "%s%%2F%s" % (repoOwner, repoName)
project_full_name = unicode2NativeString(project_full_name)
if project_full_name not in self.project_ids:
proj = yield self._http.get('/api/v3/projects/%s' % (project_full_name))
proj = yield proj.json()
self.project_ids[project_full_name] = proj['id']
proj_id = self.project_ids[project_full_name]
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
branch = unicode2NativeString(branch)
sha = unicode2NativeString(sha)
state = unicode2NativeString(state)
target_url = unicode2NativeString(build['url'])
context = unicode2NativeString(context)
description = unicode2NativeString(description)
res = yield self.createStatus(
project_id=proj_id,
branch=branch,
sha=sha,
state=state,
target_url=target_url,
context=context,
description=description
)
if res.code not in (200, 201, 204):
message = yield res.json()
message = message.get('message', 'unspecified error')
log.msg(
'Could not send status "{state}" for '
'{repoOwner}/{repoName} at {sha}: {message}'.format(
state=state, repoOwner=repoOwner,
repoName=repoName, sha=sha,
message=message))
elif self.verbose:
log.msg(
'Status "{state}" sent for '
'{repoOwner}/{repoName} at {sha}.'.format(
state=state, repoOwner=repoOwner,
repoName=repoName, sha=sha))
except Exception as e:
log.err(
e,
'Fail to send status "{state}" for '
'{repoOwner}/{repoName} at {sha}'.format(
state=state, repoOwner=repoOwner,
repoName=repoName, sha=sha))
def createEmail(self, msgdict, builderName, title, results, builds=None,
patches=None, logs=None):
text = msgdict['body']
type = msgdict['type']
if msgdict.get('subject') is not None:
subject = msgdict['subject']
else:
subject = self.subject % {'result': Results[results],
'projectName': title,
'title': title,
'builder': builderName}
assert '\n' not in subject, \
"Subject cannot contain newlines"
assert type in ('plain', 'html'), \
"'{}' message type must be 'plain' or 'html'.".format(type)
if patches or logs:
m = MIMEMultipart()
txt = MIMEText(text, type, ENCODING)
m.attach(txt)
else:
m = Message()
m.set_payload(text, ENCODING)
m.set_type("text/{}".format(type))
m['Date'] = formatdate(localtime=True)
m['Subject'] = subject
m['From'] = self.fromaddr
# m['To'] is added later
if patches:
for (i, patch) in enumerate(patches):
a = self.patch_to_attachment(patch, i)
m.attach(a)
if logs:
for log in logs:
name = "{}.{}".format(log['stepname'],
log['name'])
if (self._shouldAttachLog(log['name']) or
self._shouldAttachLog(name)):
# Use distinct filenames for the e-mail summary
if self.buildSetSummary:
filename = "{}.{}".format(log['buildername'],
name)
else:
filename = name
text = log['content']['content']
a = MIMEText(text.encode(ENCODING),
_charset=ENCODING)
a.add_header('Content-Disposition', "attachment",
filename=filename)
m.attach(a)
# @todo: is there a better way to do this?
# Add any extra headers that were requested, doing WithProperties
# interpolation if only one build was given
if self.extraHeaders:
extraHeaders = self.extraHeaders
if builds is not None and len(builds) == 1:
props = Properties.fromDict(builds[0]['properties'])
props.master = self.master
extraHeaders = yield props.render(extraHeaders)
for k, v in extraHeaders.items():
if k in m:
twlog.msg("Warning: Got header " + k +
" in self.extraHeaders "
"but it already exists in the Message - "
"not adding it.")
m[k] = v
return m
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
else:
state = 'pending'
description = yield props.render(self.startDescription)
context = yield props.render(self.context)
sourcestamps = build['buildset']['sourcestamps']
project = sourcestamps[0]['project']
if project:
repoOwner, repoName = project.split('/')
else:
repo = sourcestamps[0]['repository'].split('/')[-2:]
repoOwner = repo[0]
repoName = '.'.join(repo[1].split('.')[:-1])
m = re.match(".*:(.*)", repoOwner)
if m is not None:
repoOwner = m.group(1)
# retrieve project id
proj = yield self.session.get('%s/api/v3/projects/%s%%2F%s' % (
self.baseURL,
repoOwner.encode('utf-8'),
repoName.encode('utf-8')))
proj = proj.json()
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
res = yield self.createStatus(
project_id=proj['id'],
sha=sha.encode('utf-8'),
state=state.encode('utf-8'),
target_url=build['url'].encode('utf-8'),
context=context.encode('utf-8'),
description=description.encode('utf-8')
)
if res.status_code not in (200, 201, 204):
message = res.json().get('message', 'unspecified error')
log.msg(
'Could not send status "{state}" for '
'{repoOwner}/{repoName} at {sha}: {message}'.format(
state=state, repoOwner=repoOwner,
repoName=repoName, sha=sha,
message=message))
elif self.verbose:
log.msg(
'Status "{state}" sent for '
'{repoOwner}/{repoName} at {sha}.'.format(
state=state, repoOwner=repoOwner,
repoName=repoName, sha=sha))
except Exception as e:
log.err(
e,
'Fail to send status "{state}" for '
'{repoOwner}/{repoName} at {sha}'.format(
state=state, repoOwner=repoOwner,
repoName=repoName, sha=sha))
def send(self, build):
props = Properties.fromDict(build["properties"])
if build["complete"]:
state = {
SUCCESS: "success",
WARNINGS: "success",
FAILURE: "failed",
SKIPPED: "success",
EXCEPTION: "error",
RETRY: "pending",
CANCELLED: "error",
}.get(build["results"], "error")
description = yield props.render(self.endDescription)
else:
state = "pending"
description = yield props.render(self.startDescription)
context = yield props.render(self.context)
sourcestamps = build["buildset"]["sourcestamps"]
project = sourcestamps[0]["project"]
if project:
repoOwner, repoName = project.split("/")
else:
repo = sourcestamps[0]["repository"].split("/")[-2:]
repoOwner = repo[0]
repoName = ".".join(repo[1].split(".")[:-1])
m = re.match(".*:(.*)", repoOwner)
if m is not None:
repoOwner = m.group(1)
# retrieve project id
proj = yield self.session.get(
"%s/api/v3/projects/%s%%2F%s" % (self.baseURL, repoOwner.encode("utf-8"), repoName.encode("utf-8"))
)
proj = proj.json()
for sourcestamp in sourcestamps:
sha = sourcestamp["revision"]
try:
res = yield self.createStatus(
project_id=proj["id"],
sha=sha.encode("utf-8"),
state=state.encode("utf-8"),
target_url=build["url"].encode("utf-8"),
context=context.encode("utf-8"),
description=description.encode("utf-8"),
)
if res.status_code not in (200, 201, 204):
message = res.json().get("message", "unspecified error")
log.msg(
'Could not send status "{state}" for '
"{repoOwner}/{repoName} at {sha}: {message}".format(
state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, message=message
)
)
elif self.verbose:
log.msg(
'Status "{state}" sent for '
"{repoOwner}/{repoName} at {sha}.".format(
state=state, repoOwner=repoOwner, repoName=repoName, sha=sha
)
)
except Exception as e:
log.err(
e,
'Fail to send status "{state}" for '
"{repoOwner}/{repoName} at {sha}".format(
state=state, repoOwner=repoOwner, repoName=repoName, sha=sha
),
)
def _buildset_complete_cb(self, key, msg):
bsid = msg['bsid']
changeid, build_name, is_complete, result = yield self._get_build_info(
bsid)
if changeid is None:
# Not a build that was created by DependencyTreeScheduler
return
if not is_complete:
log.msg(
'strange - got a noncomlete build for {}'.format(build_name))
return
already_built = set(
(yield self.getState('already_built-' + str(changeid), [])))
if build_name in already_built:
log.msg(
'strange - got the same build twice {} for change {}'.format(
bsid, changeid))
return
to_build = dependency_tree.deserialize(
(yield self.getState('to_build-' + str(changeid))))
already_built.add(build_name)
if result not in (SUCCESS, WARNINGS):
builds_to_skip_due_to_error = transitive_closure(
to_build, set([build_name])).keys()
log.msg('change #{}, build {} failed. Killing future builds of {}'.
format(changeid, build_name, builds_to_skip_due_to_error))
already_built.update(set(builds_to_skip_due_to_error))
yield self.setState('already_built-' + str(changeid),
list(already_built))
return
yield self.setState('already_built-' + str(changeid),
list(already_built))
yet_to_be_built = to_build.copy()
for build in already_built:
yet_to_be_built.pop(build, None)
builds_to_build_now = builds_that_can_be_built(yet_to_be_built,
already_built)
log.msg(
'change #{}, build {} completed. need to build {}, but will build only {}'
.format(changeid, build_name, yet_to_be_built.keys(),
builds_to_build_now))
if len(builds_to_build_now) > 0:
for build in builds_to_build_now:
log.msg(
'change #{}, build {} completed. Creating build for {}'.
format(changeid, build_name, build))
yield self.addBuildsetForChanges(
reason='because {} completed'.format(build_name),
changeids=[changeid],
builderNames=[build],
properties=Properties.fromDict({
'changeid': (changeid, 'Scheduler'),
'build': (build, 'Scheduler')
}))
already_built.add(build)
yield self.setState('already_built-' + str(changeid),
list(already_built))
else:
log.msg('Woohoo! Finished change {}'.format(changeid))
def sendMessage(self, reports):
report = reports[0]
build = reports[0]['builds'][0]
props = Properties.fromDict(build['properties'])
props.master = self.master
description = report.get('body', None)
duration = None
test_results = None
if build['complete']:
state = SUCCESSFUL if build['results'] == SUCCESS else FAILED
if self.duration:
duration = yield props.render(self.duration)
else:
td = build['complete_at'] - build['started_at']
duration = int(td.seconds * 1000)
if self.test_results:
test_results = yield props.render(self.test_results)
else:
state = INPROGRESS
duration = None
parent_name = (build['parentbuilder'] or {}).get('name')
if self.parent_name:
parent = yield props.render(self.parent_name)
elif parent_name:
parent = parent_name
else:
parent = build['builder']['name']
if self.status_name:
status_name = yield props.render(self.status_name)
else:
status_name = f'{props.getProperty("buildername")} #{props.getProperty("buildnumber")}'
if parent_name:
status_name = \
f"{parent_name} #{build['parentbuild']['number']} \u00BB {status_name}"
if self.status_suffix:
status_name = status_name + (yield props.render(
self.status_suffix))
key = yield props.render(self.key)
build_number = yield props.render(self.build_number)
url = build['url']
sourcestamps = build['buildset']['sourcestamps']
for sourcestamp in sourcestamps:
try:
ssid = sourcestamp.get('ssid')
sha = sourcestamp.get('revision')
branch = sourcestamp.get('branch')
repo = sourcestamp.get('repository')
if not sha:
log.msg(f"Unable to get the commit hash for SSID: {ssid}")
continue
ref = None
if self.ref is None:
if branch is not None:
if branch.startswith("refs/"):
ref = branch
else:
ref = f"refs/heads/{branch}"
else:
ref = yield props.render(self.ref)
if not ref:
log.msg(
f"WARNING: Unable to resolve ref for SSID: {ssid}. "
"Build status will not be visible on Builds or "
"PullRequest pages only for commits")
r = re.search(r"^.*?/([^/]+)/([^/]+?)(?:\.git)?$", repo or "")
if r:
proj_key = r.group(1)
repo_slug = r.group(2)
else:
log.msg(
f"Unable to parse repository info from '{repo}' for SSID: {ssid}"
)
continue
res = yield self.createStatus(proj_key=proj_key,
repo_slug=repo_slug,
sha=sha,
state=state,
url=url,
key=key,
parent=parent,
build_number=build_number,
ref=ref,
description=description,
name=status_name,
duration=duration,
test_results=test_results)
if res.code not in (HTTP_PROCESSED, ):
content = yield res.content()
log.msg(
f"{res.code}: Unable to send Bitbucket Server status for "
f"{proj_key}/{repo_slug} {sha}: {content}")
elif self.verbose:
log.msg(
f'Status "{state}" sent for {proj_key}/{repo_slug} {sha}'
)
except Exception as e:
log.err(
e,
f'Failed to send status "{state}" for {proj_key}/{repo_slug} {sha}'
)
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failed',
SKIPPED: 'success',
EXCEPTION: 'failed',
RETRY: 'pending',
CANCELLED: 'cancelled'
}.get(build['results'], 'failed')
description = yield props.render(self.endDescription)
else:
state = 'running'
description = yield props.render(self.startDescription)
context = yield props.render(self.context)
sourcestamps = build['buildset']['sourcestamps']
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
proj_id = yield self.getProjectId(sourcestamp)
if proj_id is None:
continue
try:
branch = unicode2NativeString(sourcestamp['branch'])
sha = unicode2NativeString(sha)
state = unicode2NativeString(state)
target_url = unicode2NativeString(build['url'])
context = unicode2NativeString(context)
description = unicode2NativeString(description)
res = yield self.createStatus(
project_id=proj_id,
branch=branch,
sha=sha,
state=state,
target_url=target_url,
context=context,
description=description
)
if res.code not in (200, 201, 204):
message = yield res.json()
message = message.get('message', 'unspecified error')
log.msg(
'Could not send status "{state}" for '
'{repo} at {sha}: {message}'.format(
state=state,
repo=sourcestamp['repository'], sha=sha,
message=message))
elif self.verbose:
log.msg(
'Status "{state}" sent for '
'{repo} at {sha}.'.format(
state=state, repo=sourcestamp['repository'], sha=sha))
except Exception as e:
log.err(
e,
'Failed to send status "{state}" for '
'{repo} at {sha}'.format(
state=state,
repo=sourcestamp['repository'], sha=sha
))
def render_message_subject(self, context):
props = Properties.fromDict(context['build']['properties'])
props.master = context['master']
body = yield props.render(self.subject)
return body
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
elif self.startDescription:
state = 'pending'
description = yield props.render(self.startDescription)
else:
return
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps or not sourcestamps[0]:
return
project = sourcestamps[0]['project']
branch = props['branch']
m = re.search(r"refs/pull/([0-9]*)/merge", branch)
if m:
issue = m.group(1)
else:
issue = None
if project:
repoOwner, repoName = project.split('/')
else:
repo = sourcestamps[0]['repository'].split('/')[-2:]
repoOwner = repo[0]
repoName = '.'.join(repo[1].split('.')[:-1])
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
repo_user = unicode2NativeString(repoOwner)
repo_name = unicode2NativeString(repoName)
sha = unicode2NativeString(sha)
state = unicode2NativeString(state)
target_url = unicode2NativeString(build['url'])
context = unicode2NativeString(context)
issue = unicode2NativeString(issue)
description = unicode2NativeString(description)
yield self.createStatus(
repo_user=repo_user,
repo_name=repo_name,
sha=sha,
state=state,
target_url=target_url,
context=context,
issue=issue,
description=description
)
if self.verbose:
log.msg(
'Updated status with "{state}" for'
'{repoOwner}/{repoName} at {sha}, issue {issue}.'.format(
state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue))
except Exception as e:
log.err(
e,
'Failed to update "{state}" for '
'{repoOwner}/{repoName} at {sha}, issue {issue}'.format(
state=state, repoOwner=repoOwner, repoName=repoName, sha=sha, issue=issue))
def createEmail(self, msgdict, builderName, title, results, builds=None, patches=None, logs=None):
text = msgdict["body"].encode(ENCODING)
type = msgdict["type"]
if "subject" in msgdict:
subject = msgdict["subject"].encode(ENCODING)
else:
subject = self.subject % {
"result": Results[results],
"projectName": title,
"title": title,
"builder": builderName,
}
assert "\n" not in subject, "Subject cannot contain newlines"
assert type in ("plain", "html"), "'%s' message type must be 'plain' or 'html'." % type
if patches or logs:
m = MIMEMultipart()
txt = MIMEText(text, type, ENCODING)
m.attach(txt)
else:
m = Message()
m.set_payload(text, ENCODING)
m.set_type("text/%s" % type)
m["Date"] = formatdate(localtime=True)
m["Subject"] = subject
m["From"] = self.fromaddr
# m['To'] is added later
if patches:
for (i, patch) in enumerate(patches):
a = self.patch_to_attachment(patch, i)
m.attach(a)
if logs:
for log in logs:
name = "%s.%s" % (log["stepname"], log["name"])
if self._shouldAttachLog(log["name"]) or self._shouldAttachLog(name):
# Use distinct filenames for the e-mail summary
if self.buildSetSummary:
filename = "%s.%s" % (log["buildername"], name)
else:
filename = name
text = log["content"]["content"]
a = MIMEText(text.encode(ENCODING), _charset=ENCODING)
a.add_header("Content-Disposition", "attachment", filename=filename)
m.attach(a)
# @todo: is there a better way to do this?
# Add any extra headers that were requested, doing WithProperties
# interpolation if only one build was given
if self.extraHeaders:
extraHeaders = self.extraHeaders
if len(builds) == 1:
props = Properties.fromDict(builds[0]["properties"])
extraHeaders = yield props.render(extraHeaders)
for k, v in iteritems(extraHeaders):
if k in m:
twlog.msg(
"Warning: Got header " + k + " in self.extraHeaders "
"but it already exists in the Message - "
"not adding it."
)
m[k] = v
defer.returnValue(m)
