def __init__(self,
owner,
slug,
branch=None,
pollInterval=10 * 60,
useTimestamps=True,
category=None,
project='',
pullrequest_filter=True,
encoding='utf-8',
pollAtLaunch=False):
self.owner = owner
self.slug = slug
self.branch = branch
base.PollingChangeSource.__init__(self,
name='/'.join([owner, slug]),
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch)
self.encoding = encoding
if hasattr(pullrequest_filter, '__call__'):
self.pullrequest_filter = pullrequest_filter
else:
self.pullrequest_filter = (lambda _: pullrequest_filter)
self.lastChange = time.time()
self.lastPoll = time.time()
self.useTimestamps = useTimestamps
self.category = category if callable(category) else ascii2unicode(
category)
self.project = ascii2unicode(project)
self.initLock = defer.DeferredLock()
def perspective_try(self, branch, revision, patch, repository, project,
builderNames, who="", comment="", properties=None):
log.msg("user %s requesting build on builders %s" % (self.username,
builderNames))
if properties is None:
properties = {}
# build the intersection of the request and our configured list
builderNames = self.scheduler.filterBuilderList(builderNames)
if not builderNames:
return
reason = u"'try' job"
if who:
reason += u" by user %s" % ascii2unicode(who)
if comment:
reason += u" (%s)" % ascii2unicode(comment)
sourcestamp = dict(
branch=branch, revision=revision, repository=repository,
project=project, patch_level=patch[0], patch_body=patch[1],
patch_subdir='', patch_author=who or '',
patch_comment=comment or '', codebase='',
) # note: no way to specify patch subdir - #1769
requested_props = Properties()
requested_props.update(properties, "try build")
(bsid, brids) = yield self.scheduler.addBuildsetForSourceStamps(
sourcestamps=[sourcestamp], reason=reason,
properties=requested_props, builderNames=builderNames)
# return a remotely-usable BuildSetStatus object
bss = RemoteBuildSetStatus(self.scheduler.master, bsid, brids)
defer.returnValue(bss)
def perspective_try(self, branch, revision, patch, repository, project,
builderNames, who="", comment="", properties=None):
log.msg("user %s requesting build on builders %s" % (self.username,
builderNames))
if properties is None:
properties = {}
# build the intersection of the request and our configured list
builderNames = self.scheduler.filterBuilderList(builderNames)
if not builderNames:
return
reason = u"'try' job"
if who:
reason += u" by user %s" % ascii2unicode(who)
if comment:
reason += u" (%s)" % ascii2unicode(comment)
sourcestamp = dict(
branch=branch, revision=revision, repository=repository,
project=project, patch_level=patch[0], patch_body=patch[1],
patch_subdir='', patch_author=who or '',
patch_comment=comment or '', codebase='',
) # note: no way to specify patch subdir - #1769
requested_props = Properties()
requested_props.update(properties, "try build")
(bsid, brids) = yield self.scheduler.addBuildsetForSourceStamps(
sourcestamps=[sourcestamp], reason=reason,
properties=requested_props, builderNames=builderNames)
# return a remotely-usable BuildSetStatus object
bss = RemoteBuildSetStatus(self.scheduler.master, bsid, brids)
defer.returnValue(bss)
def __init__(self, name, password, max_builds=None,
notify_on_missing=[], missing_timeout=3600,
properties={}, locks=None, keepalive_interval=3600):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this buildslave (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this slave
@type properties: dictionary
@param locks: A list of locks that must be acquired before this slave
can be used
@type locks: dictionary
"""
name = ascii2unicode(name)
service.AsyncMultiService.__init__(self)
self.slavename = ascii2unicode(name)
self.password = password
# protocol registration
self.registration = None
# these are set when the service is started
self.botmaster = None
self.manager = None
self.master = None
self.buildslaveid = None
self.slave_status = SlaveStatus(name)
self.slave_commands = None
self.slavebuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties, "BuildSlave")
self.properties.setProperty("slavename", name, "BuildSlave")
self.lastMessageReceived = 0
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error(
'notify_on_missing arg %r is not a string' % (i,))
self.missing_timeout = missing_timeout
self.missing_timer = None
# a protocol connection, if we're currently connected
self.conn = None
self._old_builder_list = None
def __init__(self, owner, slug,
branch=None,
pollInterval=10 * 60,
useTimestamps=True,
category=None,
project='',
pullrequest_filter=True,
encoding='utf-8',
pollAtLaunch=False
):
self.owner = owner
self.slug = slug
self.branch = branch
base.PollingChangeSource.__init__(
self, name='/'.join([owner, slug]), pollInterval=pollInterval, pollAtLaunch=pollAtLaunch)
self.encoding = encoding
if hasattr(pullrequest_filter, '__call__'):
self.pullrequest_filter = pullrequest_filter
else:
self.pullrequest_filter = (lambda _: pullrequest_filter)
self.lastChange = time.time()
self.lastPoll = time.time()
self.useTimestamps = useTimestamps
self.category = category if callable(
category) else ascii2unicode(category)
self.project = ascii2unicode(project)
self.initLock = defer.DeferredLock()
def _processChanges(self, page):
result = json.loads(page, encoding=self.encoding)
for pr in result["values"]:
branch = pr["source"]["branch"]["name"]
nr = int(pr["id"])
# Note that this is a short hash. The full length hash can be accessed via the
# commit api resource but we want to avoid requesting multiple pages as long as
# we are not sure that the pull request is new or updated.
revision = pr["source"]["commit"]["hash"]
# check branch
if not self.branch or branch in self.branch:
current = yield self._getCurrentRev(nr)
if not current or current != revision:
# parse pull request api page (required for the filter)
page = yield client.getPage(str(pr["links"]["self"]["href"]))
pr_json = json.loads(page, encoding=self.encoding)
# filter pull requests by user function
if not self.pullrequest_filter(pr_json):
log.msg("pull request does not match filter")
continue
# access additional information
author = pr["author"]["display_name"]
prlink = pr["links"]["html"]["href"]
# Get time updated time. Note that the timezone offset is ignored.
if self.useTimestamps:
updated = datetime.strptime(pr["updated_on"].split(".")[0], "%Y-%m-%dT%H:%M:%S")
else:
updated = epoch2datetime(reactor.seconds())
title = pr["title"]
# parse commit api page
page = yield client.getPage(str(pr["source"]["commit"]["links"]["self"]["href"]))
commit_json = json.loads(page, encoding=self.encoding)
# use the full-length hash from now on
revision = commit_json["hash"]
revlink = commit_json["links"]["html"]["href"]
# parse repo api page
page = yield client.getPage(str(pr["source"]["repository"]["links"]["self"]["href"]))
repo_json = json.loads(page, encoding=self.encoding)
repo = repo_json["links"]["html"]["href"]
# update database
yield self._setCurrentRev(nr, revision)
# emit the change
yield self.master.data.updates.addChange(
author=ascii2unicode(author),
revision=ascii2unicode(revision),
revlink=ascii2unicode(revlink),
comments=u"pull-request #%d: %s\n%s" % (nr, title, prlink),
when_timestamp=datetime2epoch(updated),
branch=self.branch,
category=self.category,
project=self.project,
repository=ascii2unicode(repo),
src=u"bitbucket",
)
def setProperty(self, name, value, source, runtime=False):
name = util.ascii2unicode(name)
json.dumps(value) # Let the exception propagate ...
source = util.ascii2unicode(source)
self.properties[name] = (value, source)
if runtime:
self.runtime.add(name)
def setProperty(self, name, value, source, runtime=False):
name = util.ascii2unicode(name)
json.dumps(value) # Let the exception propagate ...
source = util.ascii2unicode(source)
self.properties[name] = (value, source)
if runtime:
self.runtime.add(name)
def __init__(self,
repourl,
branches=None,
branch=None,
workdir=None,
pollInterval=10 * 60,
gitbin='git',
usetimestamps=True,
category=None,
project=None,
pollinterval=-2,
fetch_refspec=None,
encoding='utf-8',
name=None,
pollAtLaunch=False,
buildPushesWithNoCommits=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = repourl
base.PollingChangeSource.__init__(self,
name=name,
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch)
if project is None:
project = ''
if branch and branches:
config.error("GitPoller: can't specify both branch and branches")
elif branch:
branches = [branch]
elif not branches:
branches = ['master']
self.repourl = repourl
self.branches = branches
self.encoding = encoding
self.buildPushesWithNoCommits = buildPushesWithNoCommits
self.gitbin = gitbin
self.workdir = workdir
self.usetimestamps = usetimestamps
self.category = category if callable(category) else ascii2unicode(
category)
self.project = ascii2unicode(project)
self.changeCount = 0
self.lastRev = {}
if fetch_refspec is not None:
config.error("GitPoller: fetch_refspec is no longer supported. "
"Instead, only the given branches are downloaded.")
if self.workdir is None:
self.workdir = 'gitpoller-work'
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
# initial run, don't parse all history
if not self.lastRev:
return
if newRev in itervalues(self.lastRev):
# TODO: no new changes on this branch
# should we just use the lastRev again, but with a different branch?
pass
# get the change list
revListArgs = ([r'--format=%H', r'%s' % newRev] +
[r'^%s' % rev for rev in itervalues(self.lastRev)] +
[r'--'])
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
self.changeCount = len(revList)
self.lastRev[branch] = newRev
if self.changeCount:
log.msg('gitpoller: processing %d changes: %s from "%s" branch "%s"'
% (self.changeCount, revList, self.repourl, branch))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
], consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
# just fail on the first error; they're probably all related!
raise failures[0]
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author, revision=ascii2unicode(rev), files=files,
comments=comments, when_timestamp=timestamp,
branch=ascii2unicode(self._removeHeads(branch)),
project=self.project, repository=ascii2unicode(self.repourl),
category=self.category, src=u'git')
def getResultSummary(self):
src = ascii2unicode(self.src, 'replace')
dest = ascii2unicode(self.dest, 'replace')
copy = u"%s to %s" % (src, dest)
if self.results == SUCCESS:
rv = u'Copied ' + copy
else:
rv = u'Copying ' + copy + ' failed.'
return {u'step': rv}
def _processChanges(self, unused_output):
"""Send info about pulled changes to the master and record current.
HgPoller does the recording by moving the working dir to the head
of the branch.
We don't update the tree (unnecessary treatment and waste of space)
instead, we simply store the current rev number in a file.
Recall that hg rev numbers are local and incremental.
"""
oid, current = yield self._getCurrentRev()
# hg log on a range of revisions is never empty
# also, if a numeric revision does not exist, a node may match.
# Therefore, we have to check explicitly that branch head > current.
head = yield self._getHead()
if head is None:
return
elif current is not None and head <= current:
return
if current is None:
# we could have used current = -1 convention as well (as hg does)
revrange = '%d:%d' % (head, head)
else:
revrange = '%d:%s' % (current + 1, head)
# two passes for hg log makes parsing simpler (comments is multi-lines)
revListArgs = [
'log', '-b', self.branch, '-r', revrange,
r'--template={rev}:{node}\n'
]
results = yield utils.getProcessOutput(self.hgbin,
revListArgs,
path=self._absWorkdir(),
env=os.environ,
errortoo=False)
results = results.decode(self.encoding)
revNodeList = [rn.split(u':', 1) for rn in results.strip().split()]
log.msg('hgpoller: processing %d changes: %r in %r' %
(len(revNodeList), revNodeList, self._absWorkdir()))
for rev, node in revNodeList:
timestamp, author, files, comments = yield self._getRevDetails(
node)
yield self.master.data.updates.addChange(
author=author,
revision=text_type(node),
files=files,
comments=comments,
when_timestamp=int(timestamp) if timestamp else None,
branch=ascii2unicode(self.branch),
category=ascii2unicode(self.category),
project=ascii2unicode(self.project),
repository=ascii2unicode(self.repourl),
src=u'hg')
# writing after addChange so that a rev is never missed,
# but at once to avoid impact from later errors
yield self._setCurrentRev(rev, oid=oid)
def addChange(self, who=None, files=None, comments=None, **kwargs):
# deprecated in 0.9.0; will be removed in 1.0.0
log.msg(
"WARNING: change source is using deprecated "
"self.master.addChange method; this method will disappear in "
"Buildbot-1.0.0"
)
# handle positional arguments
kwargs["who"] = who
kwargs["files"] = files
kwargs["comments"] = comments
def handle_deprec(oldname, newname):
if oldname not in kwargs:
return
old = kwargs.pop(oldname)
if old is not None:
if kwargs.get(newname) is None:
log.msg("WARNING: change source is using deprecated " "addChange parameter '%s'" % oldname)
return old
raise TypeError("Cannot provide '%s' and '%s' to addChange" % (oldname, newname))
return kwargs.get(newname)
kwargs["author"] = handle_deprec("who", "author")
kwargs["when_timestamp"] = handle_deprec("when", "when_timestamp")
# timestamp must be an epoch timestamp now
if isinstance(kwargs.get("when_timestamp"), datetime.datetime):
kwargs["when_timestamp"] = datetime2epoch(kwargs["when_timestamp"])
# unicodify stuff
for k in (
"comments",
"author",
"revision",
"branch",
"category",
"revlink",
"repository",
"codebase",
"project",
):
if k in kwargs:
kwargs[k] = ascii2unicode(kwargs[k])
if kwargs.get("files"):
kwargs["files"] = [ascii2unicode(f) for f in kwargs["files"]]
if kwargs.get("properties"):
kwargs["properties"] = dict((ascii2unicode(k), v) for k, v in iteritems(kwargs["properties"]))
# pass the converted call on to the data API
changeid = yield self.data.updates.addChange(**kwargs)
# and turn that changeid into a change object, since that's what
# callers expected (and why this method was deprecated)
chdict = yield self.db.changes.getChange(changeid)
change = yield changes.Change.fromChdict(self, chdict)
defer.returnValue(change)
def getResultSummary(self):
src = ascii2unicode(self.src, 'replace')
dest = ascii2unicode(self.dest, 'replace')
copy = u"%s to %s" % (src, dest)
if self.results == SUCCESS:
rv = u'Copied ' + copy
else:
rv = u'Copying ' + copy + ' failed.'
return {u'step': rv}
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
lastRev = self.lastRev.get(branch)
self.lastRev[branch] = newRev
if not lastRev:
return
# get the change list
revListArgs = [r'--format=%H', '%s..%s' % (lastRev, newRev), '--']
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
self.changeCount = len(revList)
log.msg('gitpoller: processing %d changes: %s from "%s"' %
(self.changeCount, revList, self.repourl))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
],
consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
# just fail on the first error; they're probably all related!
raise failures[0]
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author,
revision=unicode(rev),
files=files,
comments=comments,
when_timestamp=timestamp,
branch=ascii2unicode(self._removeHeads(branch)),
category=self.category,
project=self.project,
repository=ascii2unicode(self.repourl),
src=u'git')
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
lastRev = self.lastRev.get(branch)
self.lastRev[branch] = newRev
if not lastRev:
return
# get the change list
revListArgs = [r'--format=%H', '%s..%s' % (lastRev, newRev), '--']
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
self.changeCount = len(revList)
log.msg('gitpoller: processing %d changes: %s from "%s"'
% (self.changeCount, revList, self.repourl))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
], consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
# just fail on the first error; they're probably all related!
raise failures[0]
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author,
revision=unicode(rev),
files=files,
comments=comments,
when_timestamp=timestamp,
branch=ascii2unicode(self._removeHeads(branch)),
category=self.category,
project=self.project,
repository=ascii2unicode(self.repourl),
src=u'git')
def __init__(self, repourl, split_file=None,
svnuser=None, svnpasswd=None,
pollInterval=10 * 60, histmax=100,
svnbin='svn', revlinktmpl='', category=None,
project='', cachepath=None, pollinterval=-2,
extra_args=None, name=None, pollAtLaunch=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = repourl
base.PollingChangeSource.__init__(self, name=name,
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch)
if repourl.endswith("/"):
repourl = repourl[:-1] # strip the trailing slash
self.repourl = repourl
self.extra_args = extra_args
self.split_file = split_file or split_file_alwaystrunk
self.svnuser = svnuser
self.svnpasswd = svnpasswd
self.revlinktmpl = revlinktmpl
# include environment variables required for ssh-agent auth
self.environ = os.environ.copy()
self.svnbin = svnbin
self.histmax = histmax
self._prefix = None
self.category = category if callable(
category) else util.ascii2unicode(category)
self.project = util.ascii2unicode(project)
self.cachepath = cachepath
if self.cachepath and os.path.exists(self.cachepath):
try:
with open(self.cachepath, "r") as f:
self.last_change = int(f.read().strip())
log.msg("SVNPoller: SVNPoller(%s) setting last_change to %s" % (
self.repourl, self.last_change))
# try writing it, too
with open(self.cachepath, "w") as f:
f.write(str(self.last_change))
except Exception:
self.cachepath = None
log.msg(("SVNPoller: SVNPoller(%s) cache file corrupt or unwriteable; " +
"skipping and not using") % self.repourl)
log.err()
def __init__(self, repourl, branches=None, branch=None,
workdir=None, pollInterval=10 * 60,
gitbin='git', usetimestamps=True,
category=None, project=None,
pollinterval=-2, fetch_refspec=None,
encoding='utf-8', name=None, pollAtLaunch=False,
buildPushesWithNoCommits=False, only_tags=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = repourl
base.PollingChangeSource.__init__(self, name=name,
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch)
if project is None:
project = ''
if only_tags and (branch or branches):
config.error("GitPoller: can't specify only_tags and branch/branches")
if branch and branches:
config.error("GitPoller: can't specify both branch and branches")
elif branch:
branches = [branch]
elif not branches:
if only_tags:
branches = lambda ref: ref.startswith('refs/tags/') # noqa: E731
else:
branches = ['master']
self.repourl = repourl
self.branches = branches
self.encoding = encoding
self.buildPushesWithNoCommits = buildPushesWithNoCommits
self.gitbin = gitbin
self.workdir = workdir
self.usetimestamps = usetimestamps
self.category = category if callable(
category) else ascii2unicode(category)
self.project = ascii2unicode(project)
self.changeCount = 0
self.lastRev = {}
if fetch_refspec is not None:
config.error("GitPoller: fetch_refspec is no longer supported. "
"Instead, only the given branches are downloaded.")
if self.workdir is None:
self.workdir = 'gitpoller-work'
def __init__(self, repourl, split_file=None,
svnuser=None, svnpasswd=None,
pollInterval=10 * 60, histmax=100,
svnbin='svn', revlinktmpl='', category=None,
project='', cachepath=None, pollinterval=-2,
extra_args=None, name=None, pollAtLaunch=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = repourl
base.PollingChangeSource.__init__(self, name=name,
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch)
if repourl.endswith("/"):
repourl = repourl[:-1] # strip the trailing slash
self.repourl = repourl
self.extra_args = extra_args
self.split_file = split_file or split_file_alwaystrunk
self.svnuser = svnuser
self.svnpasswd = svnpasswd
self.revlinktmpl = revlinktmpl
# include environment variables required for ssh-agent auth
self.environ = os.environ.copy()
self.svnbin = svnbin
self.histmax = histmax
self._prefix = None
self.category = category if callable(
category) else util.ascii2unicode(category)
self.project = util.ascii2unicode(project)
self.cachepath = cachepath
if self.cachepath and os.path.exists(self.cachepath):
try:
with open(self.cachepath, "r") as f:
self.last_change = int(f.read().strip())
log.msg("SVNPoller: SVNPoller(%s) setting last_change to %s" % (
self.repourl, self.last_change))
# try writing it, too
with open(self.cachepath, "w") as f:
f.write(str(self.last_change))
except Exception:
self.cachepath = None
log.msg(("SVNPoller: SVNPoller(%s) cache file corrupt or unwriteable; " +
"skipping and not using") % self.repourl)
log.err()
def _processChanges(self, unused_output):
"""Send info about pulled changes to the master and record current.
HgPoller does the recording by moving the working dir to the head
of the branch.
We don't update the tree (unnecessary treatment and waste of space)
instead, we simply store the current rev number in a file.
Recall that hg rev numbers are local and incremental.
"""
oid, current = yield self._getCurrentRev()
# hg log on a range of revisions is never empty
# also, if a numeric revision does not exist, a node may match.
# Therefore, we have to check explicitly that branch head > current.
head = yield self._getHead()
if head is None:
return
elif current is not None and head <= current:
return
if current is None:
# we could have used current = -1 convention as well (as hg does)
revrange = '%d:%d' % (head, head)
else:
revrange = '%d:%s' % (current + 1, head)
# two passes for hg log makes parsing simpler (comments is multi-lines)
revListArgs = ['log', '-b', self.branch, '-r', revrange,
r'--template={rev}:{node}\n']
results = yield utils.getProcessOutput(self.hgbin, revListArgs,
path=self._absWorkdir(), env=os.environ, errortoo=False)
revNodeList = [rn.split(':', 1) for rn in results.strip().split()]
log.msg('hgpoller: processing %d changes: %r in %r'
% (len(revNodeList), revNodeList, self._absWorkdir()))
for rev, node in revNodeList:
timestamp, author, files, comments = yield self._getRevDetails(
node)
yield self.master.data.updates.addChange(
author=author,
revision=text_type(node),
files=files,
comments=comments,
when_timestamp=int(timestamp) if timestamp else None,
branch=ascii2unicode(self.branch),
category=ascii2unicode(self.category),
project=ascii2unicode(self.project),
repository=ascii2unicode(self.repourl),
src=u'hg')
# writing after addChange so that a rev is never missed,
# but at once to avoid impact from later errors
yield self._setCurrentRev(rev, oid=oid)
def addChange(self, who=None, files=None, comments=None, **kwargs):
# deprecated in 0.9.0; will be removed in 1.0.0
log.msg("WARNING: change source is using deprecated "
"self.master.addChange method; this method will disappear in "
"Buildbot-1.0.0")
# handle positional arguments
kwargs['who'] = who
kwargs['files'] = files
kwargs['comments'] = comments
def handle_deprec(oldname, newname):
if oldname not in kwargs:
return
old = kwargs.pop(oldname)
if old is not None:
if kwargs.get(newname) is None:
log.msg("WARNING: change source is using deprecated "
"addChange parameter '%s'" % oldname)
return old
raise TypeError("Cannot provide '%s' and '%s' to addChange" %
(oldname, newname))
return kwargs.get(newname)
kwargs['author'] = handle_deprec("who", "author")
kwargs['when_timestamp'] = handle_deprec("when", "when_timestamp")
# timestamp must be an epoch timestamp now
if isinstance(kwargs.get('when_timestamp'), datetime.datetime):
kwargs['when_timestamp'] = datetime2epoch(kwargs['when_timestamp'])
# unicodify stuff
for k in ('comments', 'author', 'revision', 'branch', 'category',
'revlink', 'repository', 'codebase', 'project'):
if k in kwargs:
kwargs[k] = ascii2unicode(kwargs[k])
if kwargs.get('files'):
kwargs['files'] = [ascii2unicode(f) for f in kwargs['files']]
if kwargs.get('properties'):
kwargs['properties'] = dict(
(ascii2unicode(k), v)
for k, v in iteritems(kwargs['properties']))
# pass the converted call on to the data API
changeid = yield self.data.updates.addChange(**kwargs)
# and turn that changeid into a change object, since that's what
# callers expected (and why this method was deprecated)
chdict = yield self.db.changes.getChange(changeid)
change = yield changes.Change.fromChdict(self, chdict)
defer.returnValue(change)
def handleJobFile(self, filename, f):
try:
parsed_job = self.parseJob(f)
builderNames = parsed_job['builderNames']
except BadJobfile:
log.msg("%s reports a bad jobfile in %s" % (self, filename))
log.err()
return defer.succeed(None)
# Validate/fixup the builder names.
builderNames = self.filterBuilderList(builderNames)
if not builderNames:
log.msg(
"incoming Try job did not specify any allowed builder names")
return defer.succeed(None)
who = ""
if parsed_job['who']:
who = parsed_job['who']
comment = ""
if parsed_job['comment']:
comment = parsed_job['comment']
sourcestamp = dict(
branch=parsed_job['branch'],
codebase='',
revision=parsed_job['baserev'],
patch_body=parsed_job['patch_body'],
patch_level=parsed_job['patch_level'],
patch_author=who,
patch_comment=comment,
# TODO: can't set this remotely - #1769
patch_subdir='',
project=parsed_job['project'],
repository=parsed_job['repository'])
reason = u"'try' job"
if parsed_job['who']:
reason += u" by user %s" % ascii2unicode(parsed_job['who'])
properties = parsed_job['properties']
requested_props = Properties()
requested_props.update(properties, "try build")
return self.addBuildsetForSourceStamps(sourcestamps=[sourcestamp],
reason=reason,
external_idstring=ascii2unicode(
parsed_job['jobid']),
builderNames=builderNames,
properties=requested_props)
def handleJobFile(self, filename, f):
try:
parsed_job = self.parseJob(f)
builderNames = parsed_job["builderNames"]
except BadJobfile:
log.msg("%s reports a bad jobfile in %s" % (self, filename))
log.err()
return defer.succeed(None)
# Validate/fixup the builder names.
builderNames = self.filterBuilderList(builderNames)
if not builderNames:
log.msg("incoming Try job did not specify any allowed builder names")
return defer.succeed(None)
who = ""
if parsed_job["who"]:
who = parsed_job["who"]
comment = ""
if parsed_job["comment"]:
comment = parsed_job["comment"]
sourcestamp = dict(
branch=parsed_job["branch"],
codebase="",
revision=parsed_job["baserev"],
patch_body=parsed_job["patch_body"],
patch_level=parsed_job["patch_level"],
patch_author=who,
patch_comment=comment,
patch_subdir="", # TODO: can't set this remotely - #1769
project=parsed_job["project"],
repository=parsed_job["repository"],
)
reason = u"'try' job"
if parsed_job["who"]:
reason += u" by user %s" % ascii2unicode(parsed_job["who"])
properties = parsed_job["properties"]
requested_props = Properties()
requested_props.update(properties, "try build")
return self.addBuildsetForSourceStamps(
sourcestamps=[sourcestamp],
reason=reason,
external_idstring=ascii2unicode(parsed_job["jobid"]),
builderNames=builderNames,
properties=requested_props,
)
def addHTMLLog(self, name, html):
logid = yield self.master.data.updates.addLog(self.stepid,
util.ascii2unicode(name), u'h')
_log = self._newLog(name, u'h', logid)
html = bytes2NativeString(html)
yield _log.addContent(html)
yield _log.finish()
def addCompleteLog(self, name, text):
log.msg("addCompleteLog(%s)" % name)
logid = yield self.master.data.updates.newLog(self.stepid,
util.ascii2unicode(name), u't')
l = self._newLog(name, u't', logid)
yield l.addContent(text)
yield l.finish()
def addStep(self):
# create and start the step, noting that the name may be altered to
# ensure uniqueness
self.stepid, self.number, self.name = yield self.master.data.updates.addStep(
buildid=self.build.buildid,
name=util.ascii2unicode(self.name))
yield self.master.data.updates.startStep(self.stepid)
def addHTMLLog(self, name, html):
logid = yield self.master.data.updates.addLog(self.stepid,
util.ascii2unicode(name),
u'h')
l = self._newLog(name, u'h', logid)
yield l.addContent(html)
yield l.finish()
def __init__(self, repourl, branch='default',
workdir=None, pollInterval=10 * 60,
hgbin='hg', usetimestamps=True,
category=None, project='', pollinterval=-2,
encoding='utf-8', name=None, pollAtLaunch=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = "%s[%s]" % (repourl, branch)
self.repourl = repourl
self.branch = branch
base.PollingChangeSource.__init__(
self, name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch)
self.encoding = encoding
self.lastChange = time.time()
self.lastPoll = time.time()
self.hgbin = hgbin
self.workdir = workdir
self.usetimestamps = usetimestamps
self.category = category if callable(category) else ascii2unicode(category)
self.project = project
self.commitInfo = {}
self.initLock = defer.DeferredLock()
if self.workdir is None:
config.error("workdir is mandatory for now in HgPoller")
def addCompleteLog(self, name, text):
log.msg("addCompleteLog(%s)" % name)
logid = yield self.master.data.updates.addLog(self.stepid,
util.ascii2unicode(name), u't')
l = self._newLog(name, u't', logid)
yield l.addContent(text)
yield l.finish()
def __init__(self,
name,
builderNames,
reason='',
createAbsoluteSourceStamps=False,
onlyIfChanged=False,
branch=NoBranch,
change_filter=None,
fileIsImportant=None,
onlyImportant=False,
**kwargs):
base.BaseScheduler.__init__(self, name, builderNames, **kwargs)
# tracking for when to start the next build
self.lastActuated = None
# A lock to make sure that each actuation occurs without interruption.
# This lock governs actuateAt, actuateAtTimer, and actuateOk
self.actuationLock = defer.DeferredLock()
self.actuateOk = False
self.actuateAt = None
self.actuateAtTimer = None
self.reason = util.ascii2unicode(reason % {'name': name})
self.branch = branch
self.change_filter = ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter)
self.createAbsoluteSourceStamps = createAbsoluteSourceStamps
self.onlyIfChanged = onlyIfChanged
if fileIsImportant and not callable(fileIsImportant):
config.error("fileIsImportant must be a callable")
self.fileIsImportant = fileIsImportant
# If True, only important changes will be added to the buildset.
self.onlyImportant = onlyImportant
self._reactor = reactor # patched by tests
def addCompleteLog(self, name, text):
logid = yield self.master.data.updates.addLog(self.stepid,
util.ascii2unicode(name),
u't')
_log = self._newLog(name, u't', logid)
yield _log.addContent(text)
yield _log.finish()
def reconfigServiceWithBuildbotConfig(self, new_config):
if new_config.mq.get('type', 'simple') != "wamp":
return
wamp = new_config.mq
log.msg("Starting wamp with config: %r", wamp)
router_url = wamp.get('router_url', None)
# This is not a good idea to allow people to switch the router via reconfig
# how would we continue the current transactions ?
# how would we tell the workers to switch router ?
if self.app is not None and self.router_url != router_url:
raise ValueError(
"Cannot use different wamp router url when reconfiguring")
if router_url is None:
return
self.router_url = router_url
self.app = self.serviceClass(
url=self.router_url,
extra=dict(master=self.master, parent=self),
realm=ascii2unicode(wamp.get('realm', 'buildbot')),
make=make
)
wamp_debug_level = wamp.get('wamp_debug_level', 'error')
txaio.set_global_log_level(wamp_debug_level)
yield self.app.setServiceParent(self)
yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
new_config)
def addHTMLLog(self, name, html):
logid = yield self.master.data.updates.addLog(self.stepid,
util.ascii2unicode(name), u'h')
_log = self._newLog(name, u'h', logid)
html = bytes2NativeString(html)
yield _log.addContent(html)
yield _log.finish()
def addStep(self):
# create and start the step, noting that the name may be altered to
# ensure uniqueness
self.name = yield self.build.render(self.name)
self.stepid, self.number, self.name = yield self.master.data.updates.addStep(
buildid=self.build.buildid, name=util.ascii2unicode(self.name))
yield self.master.data.updates.startStep(self.stepid)
def __init__(self, name, shouldntBeSet=NotSet, treeStableTimer=None,
builderNames=None, branch=NotABranch, branches=NotABranch,
fileIsImportant=None, categories=None,
reason="The %(classname)s scheduler named '%(name)s' triggered this build",
change_filter=None, onlyImportant=False, **kwargs):
if shouldntBeSet is not self.NotSet:
config.error(
"pass arguments to schedulers using keyword arguments")
if fileIsImportant and not callable(fileIsImportant):
config.error(
"fileIsImportant must be a callable")
# initialize parent classes
base.BaseScheduler.__init__(self, name, builderNames, **kwargs)
self.treeStableTimer = treeStableTimer
if fileIsImportant is not None:
self.fileIsImportant = fileIsImportant
self.onlyImportant = onlyImportant
self.change_filter = self.getChangeFilter(branch=branch,
branches=branches, change_filter=change_filter,
categories=categories)
# the IDelayedCall used to wake up when this scheduler's
# treeStableTimer expires.
self._stable_timers = defaultdict(lambda: None)
self._stable_timers_lock = defer.DeferredLock()
self.reason = util.ascii2unicode(reason % {
'name': name, 'classname': self.__class__.__name__
})
def __init__(self, name, shouldntBeSet=NotSet, treeStableTimer=None,
builderNames=None, branch=NotABranch, branches=NotABranch,
fileIsImportant=None, categories=None,
reason="The %(classname)s scheduler named '%(name)s' triggered this build",
change_filter=None, onlyImportant=False, **kwargs):
if shouldntBeSet is not self.NotSet:
config.error(
"pass arguments to schedulers using keyword arguments")
if fileIsImportant and not callable(fileIsImportant):
config.error(
"fileIsImportant must be a callable")
# initialize parent classes
base.BaseScheduler.__init__(self, name, builderNames, **kwargs)
self.treeStableTimer = treeStableTimer
if fileIsImportant is not None:
self.fileIsImportant = fileIsImportant
self.onlyImportant = onlyImportant
self.change_filter = self.getChangeFilter(branch=branch,
branches=branches, change_filter=change_filter,
categories=categories)
# the IDelayedCall used to wake up when this scheduler's
# treeStableTimer expires.
self._stable_timers = defaultdict(lambda: None)
self._stable_timers_lock = defer.DeferredLock()
self.reason = util.ascii2unicode(reason % {
'name': name, 'classname': self.__class__.__name__
})
def addChangeFromEvent(self, properties, event):
if "change" in event and "patchSet" in event:
event_change = event["change"]
username = event_change["owner"].get("username", u"unknown")
return self.addChange({
'author':
"%s <%s>" %
(event_change["owner"].get("name", username),
event_change["owner"].get("email", u'*****@*****.**')),
'project':
util.ascii2unicode(event_change["project"]),
'repository':
u"ssh://%s@%s:%s/%s" %
(self.username, self.gerritserver, self.gerritport,
event_change["project"]),
'branch':
self.getGroupingPolicyFromEvent(event),
'revision':
event["patchSet"]["revision"],
'revlink':
event_change["url"],
'comments':
event_change["subject"],
'files': [u"unknown"],
'category':
event["type"],
'properties':
properties
})
def addHTMLLog(self, name, html):
log.msg("addHTMLLog(%s)" % name)
logid = yield self.master.data.updates.addLog(self.stepid,
util.ascii2unicode(name), u'h')
l = self._newLog(name, u'h', logid)
yield l.addContent(html)
yield l.finish()
def reconfigServiceWithBuildbotConfig(self, new_config):
if new_config.mq.get('type', 'simple') != "wamp":
return
wamp = new_config.mq
log.msg("Starting wamp with config: %r", wamp)
router_url = wamp.get('router_url', None)
# This is not a good idea to allow people to switch the router via reconfig
# how would we continue the current transactions ?
# how would we tell the workers to switch router ?
if self.app is not None and self.router_url != router_url:
raise ValueError(
"Cannot use different wamp router url when reconfiguring")
if router_url is None:
return
self.router_url = router_url
self.app = self.serviceClass(url=self.router_url,
extra=dict(master=self.master,
parent=self),
realm=ascii2unicode(
wamp.get('realm', 'buildbot')),
make=make)
wamp_debug_level = wamp.get('wamp_debug_level', 'error')
txaio.set_global_log_level(wamp_debug_level)
yield self.app.setServiceParent(self)
yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
self, new_config)
def __init__(self, name, builderNames, properties={}, reason='',
createAbsoluteSourceStamps=False, onlyIfChanged=False,
branch=NoBranch, change_filter=None, fileIsImportant=None,
onlyImportant=False, **kwargs):
base.BaseScheduler.__init__(self, name, builderNames, properties,
**kwargs)
# tracking for when to start the next build
self.lastActuated = None
# A lock to make sure that each actuation occurs without interruption.
# This lock governs actuateAt, actuateAtTimer, and actuateOk
self.actuationLock = defer.DeferredLock()
self.actuateOk = False
self.actuateAt = None
self.actuateAtTimer = None
self.reason = util.ascii2unicode(reason % {'name': name})
self.branch = branch
self.change_filter = ChangeFilter.fromSchedulerConstructorArgs(change_filter=change_filter)
self.createAbsoluteSourceStamps = createAbsoluteSourceStamps
self.onlyIfChanged = onlyIfChanged
if fileIsImportant and not callable(fileIsImportant):
config.error(
"fileIsImportant must be a callable")
self.fileIsImportant = fileIsImportant
# If True, only important changes will be added to the buildset.
self.onlyImportant = onlyImportant
self._reactor = reactor # patched by tests
def addChangeFromEvent(self, properties, event):
if "change" in event and "patchSet" in event:
event_change = event["change"]
return self.addChange({
'author':
_gerrit_user_to_author(event_change["owner"]),
'project':
util.ascii2unicode(event_change["project"]),
'repository':
u"%s/%s" % (self.gitBaseURL, event_change["project"]),
'branch':
self.getGroupingPolicyFromEvent(event),
'revision':
event["patchSet"]["revision"],
'revlink':
event_change["url"],
'comments':
event_change["subject"],
'files': [u"unknown"],
'category':
event["type"],
'properties':
properties
})
def __init__(self, repourl, branch='default',
workdir=None, pollInterval=10 * 60,
hgbin='hg', usetimestamps=True,
category=None, project='', pollinterval=-2,
encoding='utf-8', name=None, pollAtLaunch=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = "%s[%s]" % (repourl, branch)
self.repourl = repourl
self.branch = branch
base.PollingChangeSource.__init__(
self, name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch)
self.encoding = encoding
self.lastChange = time.time()
self.lastPoll = time.time()
self.hgbin = hgbin
self.workdir = workdir
self.usetimestamps = usetimestamps
self.category = category if callable(
category) else ascii2unicode(category)
self.project = project
self.commitInfo = {}
self.initLock = defer.DeferredLock()
if self.workdir is None:
config.error("workdir is mandatory for now in HgPoller")
def reconfigService(self,
owner,
repo,
branches=None,
pollInterval=10 * 60,
category=None,
baseURL=None,
project='',
pullrequest_filter=True,
token=None,
pollAtLaunch=False,
repository_link="https",
**kwargs):
yield base.ReconfigurablePollingChangeSource.reconfigService(
self, name=self.name, **kwargs)
if baseURL is None:
baseURL = HOSTED_BASE_URL
if baseURL.endswith('/'):
baseURL = baseURL[:-1]
http_headers = {'User-Agent': 'Buildbot'}
if token is not None:
http_headers.update({'Authorization': 'token ' + token})
self._http = yield httpclientservice.HTTPClientService.getService(
self.master, baseURL, headers=http_headers)
if not branches:
branches = ['master']
self.token = token
self.owner = owner
self.repo = repo
self.branches = branches
self.project = project
self.pollInterval = pollInterval
self.repository_link = link_urls[repository_link]
if callable(pullrequest_filter):
self.pullrequest_filter = pullrequest_filter
else:
self.pullrequest_filter = (lambda _: pullrequest_filter)
self.category = category if callable(category) else ascii2unicode(
category)
self.project = ascii2unicode(project)
def _maybeStartBuildsOnBuilder(self, bldr, _reactor=reactor):
# create a chooser to give us our next builds
# this object is temporary and will go away when we're done
bc = self.createBuildChooser(bldr, self.master)
while True:
slave, breqs = yield bc.chooseNextBuild()
if not slave or not breqs:
break
# claim brid's
brids = [br.id for br in breqs]
claimed_at_epoch = _reactor.seconds()
claimed_at = epoch2datetime(claimed_at_epoch)
if not (yield self.master.data.updates.claimBuildRequests(
brids, claimed_at=claimed_at)):
# some brids were already claimed, so start over
bc = self.createBuildChooser(bldr, self.master)
continue
# the claim was successful, so publish a message for each brid
for brid in brids:
# TODO: inefficient..
brdict = yield self.master.db.buildrequests.getBuildRequest(
brid)
key = ('buildsets', str(brdict['buildsetid']), 'builders',
str(-1), 'buildrequests', str(brdict['buildrequestid']),
'claimed')
msg = dict(
bsid=brdict['buildsetid'],
brid=brdict['buildrequestid'],
buildername=brdict['buildername'],
builderid=-1,
# TODO:
# claimed_at=claimed_at_epoch,
# masterid=masterid)
)
self.master.mq.produce(key, msg)
buildStarted = yield bldr.maybeStartBuild(slave, breqs)
if not buildStarted:
yield self.master.data.updates.unclaimBuildRequests(brids)
for breq in breqs:
bsid = breq.bsid
buildername = ascii2unicode(breq.buildername)
brid = breq.id
key = ('buildsets', str(brdict['buildsetid']), 'builders',
str(-1), 'buildrequests',
str(brdict['buildrequestid']), 'unclaimed')
msg = dict(brid=brid,
bsid=bsid,
buildername=buildername,
builderid=-1)
self.master.mq.produce(key, msg)
# and try starting builds again. If we still have a working slave,
# then this may re-claim the same buildrequests
self.botmaster.maybeStartBuildsForBuilder(self.name)
def reconfigService(self,
owner,
repo,
branches=None,
pollInterval=10 * 60,
category=None,
baseURL=None,
project='',
pullrequest_filter=True,
token=None,
pollAtLaunch=False,
magic_link=False,
repository_type="https",
**kwargs):
yield base.ReconfigurablePollingChangeSource.reconfigService(
self, name=self.name, **kwargs)
if baseURL is None:
baseURL = HOSTED_BASE_URL
if baseURL.endswith('/'):
baseURL = baseURL[:-1]
http_headers = {'User-Agent': 'Buildbot'}
if token is not None:
http_headers.update({'Authorization': 'token ' + token})
self._http = yield httpclientservice.HTTPClientService.getService(
self.master, baseURL, headers=http_headers)
self.token = token
self.owner = owner
self.repo = repo
self.branches = branches
self.project = project
self.pollInterval = pollInterval
self.repository_type = link_urls[repository_type]
self.magic_link = magic_link
if callable(pullrequest_filter):
self.pullrequest_filter = pullrequest_filter
else:
self.pullrequest_filter = (lambda _: pullrequest_filter)
self.category = category if callable(category) else ascii2unicode(
category)
self.project = ascii2unicode(project)
def addLog(self, name, type="s", logEncoding=None):
d = self.master.data.updates.addLog(self.stepid, util.ascii2unicode(name), unicode(type))
@d.addCallback
def newLog(logid):
return self._newLog(name, type, logid, logEncoding)
return d
def __init__(self, name):
# service.Service.__init__(self) # there is none, oddly
name = util.ascii2unicode(name)
self.setName(name)
self.serviceid = None
self.active = False
def __init__(self, name):
# service.Service.__init__(self) # there is none, oddly
name = util.ascii2unicode(name)
self.setName(name)
self.serviceid = None
self.active = False
def _maybeStartBuildsOnBuilder(self, bldr, _reactor=reactor):
# create a chooser to give us our next builds
# this object is temporary and will go away when we're done
bc = self.createBuildChooser(bldr, self.master)
while True:
slave, breqs = yield bc.chooseNextBuild()
if not slave or not breqs:
break
# claim brid's
brids = [br.id for br in breqs]
claimed_at_epoch = _reactor.seconds()
claimed_at = epoch2datetime(claimed_at_epoch)
if not (yield self.master.data.updates.claimBuildRequests(
brids, claimed_at=claimed_at)):
# some brids were already claimed, so start over
bc = self.createBuildChooser(bldr, self.master)
continue
# the claim was successful, so publish a message for each brid
for brid in brids:
# TODO: inefficient..
brdict = yield self.master.db.buildrequests.getBuildRequest(brid)
key = ('buildsets', str(brdict['buildsetid']),
'builders', str(-1),
'buildrequests', str(brdict['buildrequestid']), 'claimed')
msg = dict(
bsid=brdict['buildsetid'],
brid=brdict['buildrequestid'],
buildername=brdict['buildername'],
builderid=-1,
# TODO:
# claimed_at=claimed_at_epoch,
# masterid=masterid)
)
self.master.mq.produce(key, msg)
buildStarted = yield bldr.maybeStartBuild(slave, breqs)
if not buildStarted:
yield self.master.data.updates.unclaimBuildRequests(brids)
for breq in breqs:
bsid = breq.bsid
buildername = ascii2unicode(breq.buildername)
brid = breq.id
key = ('buildsets', str(brdict['buildsetid']),
'builders', str(-1),
'buildrequests', str(brdict['buildrequestid']), 'unclaimed')
msg = dict(brid=brid, bsid=bsid, buildername=buildername,
builderid=-1)
self.master.mq.produce(key, msg)
# and try starting builds again. If we still have a working slave,
# then this may re-claim the same buildrequests
self.botmaster.maybeStartBuildsForBuilder(self.name)
def addLog(self, name, type='s', logEncoding=None):
d = self.master.data.updates.addLog(self.stepid,
util.ascii2unicode(name),
unicode(type))
@d.addCallback
def newLog(logid):
return self._newLog(name, type, logid, logEncoding)
return d
def __init__(self,
p4port=None,
p4user=None,
p4passwd=None,
p4base='//',
p4bin='p4',
split_file=lambda branchfile: (None, branchfile),
pollInterval=60 * 10,
histmax=None,
pollinterval=-2,
encoding='utf8',
project=None,
name=None,
use_tickets=False,
ticket_login_interval=60 * 60 * 24,
server_tz=None,
pollAtLaunch=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = "P4Source:%s:%s" % (p4port, p4base)
base.PollingChangeSource.__init__(self,
name=name,
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch)
if project is None:
project = ''
if use_tickets and not p4passwd:
config.error(
"You need to provide a P4 password to use ticket authentication"
)
self.p4port = p4port
self.p4user = p4user
self.p4passwd = p4passwd
self.p4base = p4base
self.p4bin = p4bin
self.split_file = split_file
self.encoding = encoding
self.project = util.ascii2unicode(project)
self.use_tickets = use_tickets
self.ticket_login_interval = ticket_login_interval
self.server_tz = dateutil.tz.gettz(server_tz) if server_tz else None
if server_tz is not None and self.server_tz is None:
raise P4PollerError(
"Failed to get timezone from server_tz string '{}'".format(
server_tz))
self._ticket_passwd = None
self._ticket_login_counter = 0
def __init__(self, *args, **kwargs):
name = kwargs.pop("name", None)
if name is not None:
self.name = ascii2unicode(name)
self.checkConfig(*args, **kwargs)
if self.name is None:
raise ValueError("%s: must pass a name to constructor" % type(self))
self._config_args = args
self._config_kwargs = kwargs
AsyncMultiService.__init__(self)
def __init__(self, *args, **kwargs):
name = kwargs.pop("name", None)
if name is not None:
self.name = ascii2unicode(name)
self.checkConfig(*args, **kwargs)
if self.name is None:
raise ValueError("%s: must pass a name to constructor" % type(self))
self._config_args = args
self._config_kwargs = kwargs
AsyncMultiService.__init__(self)
def __init__(self,
p4port=None,
p4user=None,
p4passwd=None,
p4base='//',
p4bin='p4',
split_file=lambda branchfile: (None, branchfile),
pollInterval=60 * 10,
histmax=None,
pollinterval=-2,
encoding='utf8',
project=None,
name=None,
use_tickets=False,
ticket_login_interval=60 * 60 * 24,
server_tz=None,
pollAtLaunch=False):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if name is None:
name = "P4Source:%s:%s" % (p4port, p4base)
base.PollingChangeSource.__init__(
self,
name=name,
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch)
if project is None:
project = ''
if use_tickets and not p4passwd:
config.error(
"You need to provide a P4 password to use ticket authentication"
)
self.p4port = p4port
self.p4user = p4user
self.p4passwd = p4passwd
self.p4base = p4base
self.p4bin = p4bin
self.split_file = split_file
self.encoding = encoding
self.project = util.ascii2unicode(project)
self.use_tickets = use_tickets
self.ticket_login_interval = ticket_login_interval
self.server_tz = dateutil.tz.gettz(server_tz) if server_tz else None
self._ticket_passwd = None
self._ticket_login_counter = 0
def getOldestRequestTime(self):
"""Returns the submitted_at of the oldest unclaimed build request for
this builder, or None if there are no build requests.
@returns: datetime instance or None, via Deferred
"""
unclaimed = yield self.master.data.get(('builders', ascii2unicode(self.name), 'buildrequests'),
[resultspec.Filter('claimed', 'eq', [False])])
if unclaimed:
unclaimed = sorted([brd['submitted_at'] for brd in unclaimed])
defer.returnValue(unclaimed[0])
else:
defer.returnValue(None)
def getOldestRequestTime(self):
"""Returns the submitted_at of the oldest unclaimed build request for
this builder, or None if there are no build requests.
@returns: datetime instance or None, via Deferred
"""
unclaimed = yield self.master.data.get(('builders', ascii2unicode(self.name), 'buildrequests'),
[resultspec.Filter('claimed', 'eq', [False])])
if unclaimed:
unclaimed = sorted([brd['submitted_at'] for brd in unclaimed])
defer.returnValue(unclaimed[0])
else:
defer.returnValue(None)
def forceIdentifier(maxLength, str):
if not isinstance(str, string_types):
raise TypeError("%r cannot be coerced to an identifier" % (str,))
# usually ascii2unicode can handle it
str = util.ascii2unicode(str)
if isIdentifier(maxLength, str):
return str
# trim to length and substitute out invalid characters
str = str[:maxLength]
str = initial_re.sub('_', str)
str = subsequent_re.subn('_', str)[0]
return str
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = dict([(b.name, b)
for b in list(self)
if isinstance(b, Builder)])
old_set = set(old_by_name)
new_by_name = dict([(bc.name, bc)
for bc in new_config.builders])
new_set = set(new_by_name)
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
# pylint: disable=cell-var-from-loop
yield defer.maybeDeferred(lambda:
builder.disownServiceParent())
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
yield builder.setServiceParent(self)
self.builderNames = list(self.builders)
yield self.master.data.updates.updateBuilderList(
self.master.masterid,
[util.ascii2unicode(n) for n in self.builderNames])
metrics.MetricCountEvent.log("num_builders",
len(self.builders), absolute=True)
timer.stop()
