def method(testcase):
testcase.expect_commands(ExpectMaster(["command"]))
testcase.add_run_process_expect_env({'key': 'value'})
res = yield runprocess.run_process(None, ["command"],
env={'key': 'value'})
self.assertEqual(res, (0, b'', b''))
testcase.assert_all_commands_ran()
def _dovccmdImpl(self, command, args, path, ssh_workdir):
full_args = []
full_env = os.environ.copy()
if self._isSshPrivateKeyNeededForCommand(command):
key_path = self._getSshPrivateKeyPath(ssh_workdir)
self._downloadSshPrivateKey(key_path)
known_hosts_path = None
if self.sshHostKey is not None or self.sshKnownHosts is not None:
known_hosts_path = self._getSshKnownHostsPath(ssh_workdir)
self._downloadSshKnownHosts(known_hosts_path)
self.adjustCommandParamsForSshPrivateKey(full_args, full_env,
key_path, None,
known_hosts_path)
full_args += [command] + args
res = yield runprocess.run_process(self.master.reactor,
[self.gitbin] + full_args,
path,
env=full_env)
(code, stdout, stderr) = res
stdout = bytes2unicode(stdout, self.encoding)
stderr = bytes2unicode(stderr, self.encoding)
if code != 0:
if code == 128:
raise GitError(
'command {} in {} on repourl {} failed with exit code {}: {}'
.format(full_args, path, self.repourl, code, stderr))
raise EnvironmentError(
('command {} in {} on repourl {} failed with exit code {}: {}'
).format(full_args, path, self.repourl, code, stderr))
return stdout.strip()
def runProcessLogFailures(reactor, args, expectedCode=0):
code, stdout, stderr = yield runprocess.run_process(reactor, args)
if code != expectedCode:
log.err(f'Got unexpected return code when running {args}: '
f'code: {code}, stdout: {stdout}, stderr: {stderr}')
return False
return True
def _initRepository(self):
"""Have mercurial init the workdir as a repository (hg init) if needed.
hg init will also create all needed intermediate directories.
"""
if self._isRepositoryReady():
return
log.msg('hgpoller: initializing working dir from {}'.format(
self.repourl))
rc = yield runprocess.run_process(
self.master.reactor,
[self.hgbin, 'init', self._absWorkdir()],
env=os.environ,
collect_stdout=False,
collect_stderr=False)
if rc != 0:
self._stopOnFailure()
raise EnvironmentError(
'{}: repository init failed with exit code {}'.format(
self, rc))
log.msg("hgpoller: finished initializing working dir {}".format(
self.workdir))
def method(testcase):
testcase.expect_commands(
ExpectMasterShell(["command"
]).stdout(b'stdout').stderr(b'stderr'))
res = yield runprocess.run_process(None, ["command"])
self.assertEqual(res, (0, b'stdout', b'stderr'))
testcase.assert_all_commands_ran()
def _getRevDetails(self, rev):
"""Return a deferred for (date, author, files, comments) of given rev.
Deferred will be in error if rev is unknown.
"""
command = [
self.hgbin, 'log', '-r', rev, os.linesep.join((
'--template={date|hgdate}',
'{author}',
"{files % '{file}" + os.pathsep + "'}",
'{desc|strip}'))]
# Mercurial fails with status 255 if rev is unknown
rc, output = yield runprocess.run_process(self.master.reactor,
command, workdir=self._absWorkdir(),
env=os.environ, collect_stderr=False,
stderr_is_error=True)
if rc != 0:
msg = f'{self}: got error {rc} when getting details for revision {rev}'
raise Exception(msg)
# all file names are on one line
output = output.decode(self.encoding, "replace")
date, author, files, comments = output.split(os.linesep, 3)
if not self.usetimestamps:
stamp = None
else:
try:
stamp = float(date.split()[0])
except Exception:
log.msg(f'hgpoller: caught exception converting output {repr(date)} to timestamp')
raise
return stamp, author.strip(), files.split(os.pathsep)[:-1], comments.strip()
def _getHead(self, branch):
"""Return a deferred for branch head revision or None.
We'll get an error if there is no head for this branch, which is
probably a good thing, since it's probably a misspelling
(if really buildbotting a branch that does not have any changeset
yet, one shouldn't be surprised to get errors)
"""
rc, stdout = yield runprocess.run_process(self.master.reactor,
[self.hgbin, 'heads', branch,
'--template={rev}' + os.linesep],
workdir=self._absWorkdir(), env=os.environ,
collect_stderr=False, stderr_is_error=True)
if rc != 0:
log.err(f"{self}: could not find revision {branch} in repository {self.repourl}")
return None
if not stdout:
return None
if len(stdout.split()) > 1:
log.err(f"{self}: caught several heads in branch {branch} "
f"from repository {self.repourl}. Staying at previous revision"
"You should wait until the situation is normal again "
"due to a merge or directly strip if remote repo "
"gets stripped later.")
return None
# in case of whole reconstruction, are we sure that we'll get the
# same node -> rev assignations ?
return stdout.strip().decode(self.encoding)
def _prepare_base_image(self):
"""
I am a private method for creating (possibly cheap) copies of a
base_image for start_instance to boot.
"""
if not self.base_image:
return
if self.cheap_copy:
clone_cmd = [
'qemu-img', 'create', '-b', self.base_image, '-f', 'qcow2',
self.image
]
else:
clone_cmd = ['cp', self.base_image, self.image]
log.msg(f"Cloning base image: {clone_cmd}'")
try:
rc = yield runprocess.run_process(self.master.reactor,
clone_cmd,
collect_stdout=False,
collect_stderr=False)
if rc != 0:
raise LatentWorkerFailedToSubstantiate(
f'Failed to clone image (rc={rc})')
except Exception as e:
log.err(f"Cloning failed: {e}")
raise
def method(testcase):
testcase.expect_commands(
ExpectMaster(["command"]).stdout(b'stdout').stderr(b'stderr'))
res = yield runprocess.run_process(None, ["command"],
collect_stdout=False,
collect_stderr=False)
self.assertEqual(res, 0)
testcase.assert_all_commands_ran()
def apply_acls(self, bind):
for cmd in [["setfacl", "-m", "user:{0}:rwx".format(buildbot_uid), bind],
["setfacl", "-m", "group:{0}:rwx".format(buildbot_gid), bind]]:
rc, out, err = yield runprocess.run_process(self.master.reactor, cmd, env=None,
collect_stdout=True,
collect_stderr=True)
if rc != 0:
raise RuntimeError("setfacl '{0}' failed with result {1}:\n{2}".format(cmd, rc, err))
def method(testcase):
testcase.expect_commands(
ExpectMaster(["command"]).stderr(b"some test"))
res = yield runprocess.run_process(None, ["command"],
collect_stderr=False,
stderr_is_error=True)
self.assertEqual(res, (-1, b''))
testcase.assert_all_commands_ran()
def runProcessLogFailures(reactor, args, expectedCode=0):
code, stdout, stderr = yield runprocess.run_process(reactor, args)
if code != expectedCode:
log.err(('Got unexpected return code when running {}: '
'code: {}, stdout: {}, stderr: {}').format(
args, code, stdout, stderr))
return False
return True
def _get_process_output(self, args):
env = {e: os.environ.get(e)
for e in self.env_vars if os.environ.get(e)}
res, out = yield runprocess.run_process(self.master.reactor, [self.p4bin] + args,
env=env, collect_stderr=False,
stderr_is_error=True)
if res != 0:
raise P4PollerError('Failed to run {}'.format(self.p4bin))
return out
def get(self, entry):
"""
get the value from pass identified by 'entry'
"""
try:
rc, output = yield runprocess.run_process(self.master.reactor,
['pass', entry],
env=self._env,
collect_stderr=False,
stderr_is_error=True)
if rc != 0:
return None
return output.decode("utf-8", "ignore").splitlines()[0]
except IOError:
return None
def get_prefix(self):
command = [
self.svnbin, "info", "--xml", "--non-interactive", self.repourl
]
if self.svnuser:
command.append("--username={}".format(self.svnuser))
if self.svnpasswd is not None:
command.append("--password={}".format(self.svnpasswd))
if self.extra_args:
command.extend(self.extra_args)
rc, output = yield runprocess.run_process(self.master.reactor,
command,
env=self.environ,
collect_stderr=False,
stderr_is_error=True)
if rc != 0:
raise EnvironmentError(
'{}: Got error when retrieving svn prefix'.format(self))
try:
doc = xml.dom.minidom.parseString(output)
except xml.parsers.expat.ExpatError:
log.msg(
"SVNPoller: SVNPoller.get_prefix: ExpatError in '{}'".format(
output))
raise
rootnodes = doc.getElementsByTagName("root")
if not rootnodes:
# this happens if the URL we gave was already the root. In this
# case, our prefix is empty.
self._prefix = ""
return self._prefix
rootnode = rootnodes[0]
root = "".join([c.data for c in rootnode.childNodes])
# root will be a unicode string
if not self.repourl.startswith(root):
log.msg(format="Got root %(root)r from `svn info`, but it is "
"not a prefix of the configured repourl",
repourl=self.repourl,
root=root)
raise RuntimeError("Configured repourl doesn't match svn root")
prefix = self.repourl[len(root):]
if prefix.startswith("/"):
prefix = prefix[1:]
log.msg("SVNPoller: repourl={}, root={}, so prefix={}".format(
self.repourl, root, prefix))
return prefix
def _getRevNodeList(self, revset):
rc, stdout = yield runprocess.run_process(self.master.reactor,
[self.hgbin, 'log', '-r', revset,
r'--template={rev}:{node}\n'],
workdir=self._absWorkdir(), env=os.environ,
collect_stdout=True, collect_stderr=False,
stderr_is_error=True)
if rc != 0:
raise EnvironmentError(f'{self}: could not get rev node list: {rc}')
results = stdout.decode(self.encoding)
revNodeList = [rn.split(':', 1) for rn in results.strip().split()]
return revNodeList
def _getChanges(self):
self.lastPoll = time.time()
yield self._initRepository()
log.msg(f"{self}: polling hg repo at {self.repourl}")
command = [self.hgbin, 'pull']
for name in self.branches:
command += ['-b', name]
for name in self.bookmarks:
command += ['-B', name]
command += [self.repourl]
yield runprocess.run_process(self.master.reactor, command, workdir=self._absWorkdir(),
env=os.environ, collect_stdout=False,
collect_stderr=False)
def get_logs(self, _):
command = [self.svnbin, "log", "--xml", "--verbose", "--non-interactive"]
if self.svnuser:
command.extend([f"--username={self.svnuser}"])
if self.svnpasswd is not None:
command.extend([f"--password={self.svnpasswd}"])
if self.extra_args:
command.extend(self.extra_args)
command.extend([f"--limit={(self.histmax)}", self.repourl])
rc, output = yield runprocess.run_process(self.master.reactor, command, env=self.environ,
collect_stderr=False, stderr_is_error=True)
if rc != 0:
raise EnvironmentError(f'{self}: Got error when retrieving svn logs')
return output
def getFiles(self, change, patchset):
cmd = self._buildGerritCommand("query", str(change), "--format", "JSON",
"--files", "--patch-sets")
if self.debug:
log.msg(f"{self.name}: querying for changed files in change {change}/{patchset}: {cmd}")
rc, out = yield runprocess.run_process(self.master.reactor, cmd, env=None,
collect_stderr=False)
if rc != 0:
return ["unknown"]
out = out.splitlines()[0]
res = json.loads(bytes2unicode(out))
if res.get("rowCount") == 0:
return ["unknown"]
patchsets = {i["number"]: i["files"] for i in res["patchSets"]}
return [i["file"] for i in patchsets[int(patchset)]]
def method(testcase):
testcase.expect_commands(ExpectMaster(["command2"]))
yield runprocess.run_process(None, ["command"])
def method(testcase):
testcase.expect_commands(ExpectMasterShell(["command", "arg"]))
yield runprocess.run_process(None, ["command", "otherarg"])
testcase.assert_all_commands_ran()
def method(testcase):
testcase.expect_commands(
ExpectMaster(["command"]).workdir("/home"))
yield runprocess.run_process(None, ["command"])
testcase.assert_all_commands_ran()
def method(testcase):
testcase.expect_commands(ExpectMaster(["command"]))
testcase.add_run_process_expect_env({'key': 'value'})
d = runprocess.run_process(None, ["command"])
return d
def method(testcase):
testcase.expect_commands(ExpectMaster(["command"]))
testcase.add_run_process_expect_env({'key': 'value'})
yield runprocess.run_process(None, ["command"],
env={'key': 'wrongvalue'})
testcase.assert_all_commands_ran()
def method(testcase):
testcase.expect_commands(ExpectMaster(["command", "arg"]))
yield runprocess.run_process(None, ["command"], workdir="/path")
testcase.assert_all_commands_ran()
def method(testcase):
testcase.expect_commands(ExpectMaster(["command"]).exit(1))
res = yield runprocess.run_process(None, ["command"])
self.assertEqual(res, (1, b'', b''))
testcase.assert_all_commands_ran()
