def getConnection(self):
maker = SSHConnectionMaker()
if not DOCKER:
raise SkipTest('docker is not installed')
# Start a docker container.
log.msg('Spawning container...', system='docker')
stdout, sdterr, rc = yield utils.getProcessOutputAndValue(
DOCKER,
['run', '-p', '22', '-d', 'moldit/ubuntu-ssh'],
)
if rc != 0:
self.fail("Could not spawn docker instance:\n%s\n%s" % (
stdout, stderr))
container_id = stdout.strip()
log.msg('Spawned container %s' % (container_id,), system='docker')
self.addCleanup(self.killDockerContainer, container_id)
# Generate an SSH uri for connecting to the container.
stdout, stderr, rc = yield utils.getProcessOutputAndValue(
DOCKER,
['port', container_id, '22'],
)
port = int(stdout.strip().split(':')[-1])
uri = 'ssh://*****:*****@127.0.0.1:%d' % (port,)
log.msg('Generated SSH URI: %r' % (uri,))
conn = yield maker.getConnection(uri)
defer.returnValue(conn)
def test_send_file_pre_generated_code(self):
code = "1-abc"
filename = "testfile"
message = "test message"
send_dir = self.mktemp()
os.mkdir(send_dir)
with open(os.path.join(send_dir, filename), "w") as f:
f.write(message)
wormhole = self.find_executable()
server_args = ["--relay-url", self.relayurl]
send_args = server_args + [
"send",
"--code", code,
filename,
]
d1 = getProcessOutputAndValue(wormhole, send_args, path=send_dir)
receive_dir = self.mktemp()
os.mkdir(receive_dir)
receive_args = server_args + [
"receive", "--accept-file",
code,
]
d2 = getProcessOutputAndValue(wormhole, receive_args, path=receive_dir)
def _check_sender(res):
out, err, rc = res
out = out.decode("utf-8")
err = err.decode("utf-8")
self.failUnlessIn("Sending %d byte file named '%s'\n" %
(len(message), filename), out)
self.failUnlessIn("On the other computer, please run: "
"wormhole receive\n"
"Wormhole code is: %s\n\n" % code,
out)
self.failUnlessIn("File sent.. waiting for confirmation\n"
"Confirmation received. Transfer complete.\n",
out)
self.failUnlessEqual(err, "")
self.failUnlessEqual(rc, 0)
return d2
d1.addCallback(_check_sender)
def _check_receiver(res):
out, err, rc = res
out = out.decode("utf-8")
err = err.decode("utf-8")
self.failUnlessIn("Receiving %d bytes for '%s'" %
(len(message), filename), out)
self.failUnlessIn("Received file written to ", out)
self.failUnlessEqual(err, "")
self.failUnlessEqual(rc, 0)
fn = os.path.join(receive_dir, filename)
self.failUnless(os.path.exists(fn))
with open(fn, "r") as f:
self.failUnlessEqual(f.read(), message)
d1.addCallback(_check_receiver)
return d1
def set_master(_):
log.msg('gitpoller: checking out %s' % self.branch)
if self.branch == 'master': # repo is already on branch 'master', so reset
d = utils.getProcessOutputAndValue(self.gitbin,
['reset', '--hard', 'origin/%s' % self.branch],
path=self.workdir, env=os.environ)
else:
d = utils.getProcessOutputAndValue(self.gitbin,
['checkout', '-b', self.branch, 'origin/%s' % self.branch],
path=self.workdir, env=os.environ)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
return d
def execute(self):
"""Execute the CLI command.
Returns a deferred that will be called when the operation completes. The
parameter to the deferred is this object.
"""
def __cli_output_callback(result):
"""Callback from getProcessOutputAndValue"""
self._set_properties(result)
LOGGER.debug("Asterisk CLI %s exited %d" %
(self.host, self.exitcode))
if self.err:
LOGGER.debug(self.err)
if self.exitcode:
self._deferred.errback(self)
else:
self._deferred.callback(self)
def __cli_error_callback(result):
"""Errback from getProcessOutputAndValue"""
self._set_properties(result.value)
LOGGER.warning("Asterisk CLI %s exited %d with error: %s" %
(self.host, self.exitcode, self.err))
if self.err:
LOGGER.debug(self.err)
self._deferred.errback(self)
self._deferred = defer.Deferred()
deferred = utils.getProcessOutputAndValue(self._cmd[0],
self._cmd,
env=os.environ)
deferred.addCallbacks(callback=__cli_output_callback,
errback=__cli_error_callback,)
return self._deferred
def _getBatchOutput(self, f):
fn = self.mktemp()
open(fn, 'w').write(f)
port = self.server.getHost().port
cmds = ('-p %i -l testuser '
'--known-hosts kh_test '
'--user-authentications publickey '
'--host-key-algorithms ssh-rsa '
'-i dsa_test '
'-a '
'-v -b %s 127.0.0.1') % (port, fn)
cmds = test_conch._makeArgs(cmds.split(), mod='cftp')[1:]
log.msg('running %s %s' % (sys.executable, cmds))
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
self.server.factory.expectedLoseConnection = 1
d = getProcessOutputAndValue(sys.executable, cmds, env=env)
def _cleanup(res):
os.remove(fn)
return res
d.addCallback(lambda res: res[0])
d.addBoth(_cleanup)
return d
def _dovccmd(self, command, args, path=None):
def encodeArg(arg):
if isinstance(arg, list):
return [encodeArg(a) for a in arg]
elif isinstance(arg, text_type):
return arg.encode("ascii")
return arg
d = utils.getProcessOutputAndValue(encodeArg(self.gitbin),
encodeArg([command] + args),
path=encodeArg(path),
env=os.environ)
def _convert_nonzero_to_failure(res, command, args, path):
"utility to handle the result of getProcessOutputAndValue"
(stdout, stderr, code) = res
if code != 0:
if code == 128:
raise GitError(
'command %s %s in %s on repourl %s failed with exit code %d: %s'
% (command, args, path, self.repourl, code, stderr))
raise EnvironmentError(
'command %s %s in %s on repourl %s failed with exit code %d: %s'
% (command, args, path, self.repourl, code, stderr))
return stdout.strip()
d.addCallback(_convert_nonzero_to_failure, command, args, path)
return d
def _dovccmdImpl(self, command, args, path):
full_args = []
full_env = os.environ.copy()
if self._isSshPrivateKeyNeededForCommand(command):
key_path = self._getSshPrivateKeyPath()
self._downloadSshPrivateKey(key_path)
known_hosts_path = None
if self.sshHostKey is not None:
known_hosts_path = self._getSshKnownHostsPath()
self._downloadSshKnownHosts(known_hosts_path)
self.adjustCommandParamsForSshPrivateKey(full_args, full_env,
key_path, None,
known_hosts_path)
full_args += [command] + args
res = yield utils.getProcessOutputAndValue(self.gitbin,
full_args,
path=path,
env=full_env)
(stdout, stderr, code) = res
stdout = bytes2unicode(stdout, self.encoding)
stderr = bytes2unicode(stderr, self.encoding)
if code != 0:
if code == 128:
raise GitError(
'command {} in {} on repourl {} failed with exit code {}: {}'
.format(full_args, path, self.repourl, code, stderr))
raise EnvironmentError(
'command {} in {} on repourl {} failed with exit code {}: {}'.
format(full_args, path, self.repourl, code, stderr))
defer.returnValue(stdout.strip())
def _getBatchOutput(self, f):
fn = self.mktemp()
with open(fn, "w") as fp:
fp.write(f)
port = self.server.getHost().port
cmds = ("-p %i -l testuser "
"--known-hosts kh_test "
"--user-authentications publickey "
"--host-key-algorithms ssh-rsa "
"-i dsa_test "
"-a "
"-v -b %s 127.0.0.1") % (port, fn)
cmds = test_conch._makeArgs(cmds.split(), mod="cftp")[1:]
log.msg(f"running {sys.executable} {cmds}")
env = os.environ.copy()
env["PYTHONPATH"] = os.pathsep.join(sys.path)
self.server.factory.expectedLoseConnection = 1
d = getProcessOutputAndValue(sys.executable, cmds, env=env)
def _cleanup(res):
os.remove(fn)
return res
d.addCallback(lambda res: res[0])
d.addBoth(_cleanup)
return d
def test_outputAndValue(self):
"""
The L{Deferred} returned by L{getProcessOutputAndValue} fires with a
three-tuple, the elements of which give the data written to the child's
stdout, the data written to the child's stderr, and the exit status of
the child.
"""
scriptFile = self.makeSourceFile([
"import sys", "if hasattr(sys.stdout, 'buffer'):",
" # Python 3",
" sys.stdout.buffer.write(b'hello world!\\n')",
" sys.stderr.buffer.write(b'goodbye world!\\n')", "else:",
" # Python 2", " sys.stdout.write(b'hello world!\\n')",
" sys.stderr.write(b'goodbye world!\\n')", "sys.exit(1)"
])
def gotOutputAndValue(out_err_code):
out, err, code = out_err_code
self.assertEqual(out, b"hello world!\n")
if _PY3:
self.assertEqual(err, b"goodbye world!\n")
else:
self.assertEqual(err, b"goodbye world!" + os.linesep)
self.assertEqual(code, 1)
d = utils.getProcessOutputAndValue(self.exe, ["-u", scriptFile])
return d.addCallback(gotOutputAndValue)
def _run_test(self, test):
d = getProcessOutputAndValue(
'bash',
[
get_test_data_file('testssl.sh'),
'--quiet',
'--nodns',
'--color',
'0',
test,
'127.0.0.1:%d' % self.port,
],
)
def on_result(results):
if test == '-e':
results = results[0].split('\n')[7:-6]
self.assertTrue(len(results) > 3)
else:
self.assertIn('OK', results[0])
self.assertNotIn('NOT ok', results[0])
d.addCallback(on_result)
return d
def _started(res):
open(HOTLINE_FILE, "w").write("")
self.failUnless(os.path.exists(TWISTD_PID_FILE))
# rm this so we can detect when the second incarnation is ready
os.unlink(PORTNUMFILE)
return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "restart", c1], env=os.environ)
def gpg_verify(filename, signature, gpg="/usr/bin/gpg"):
"""Verify the GPG signature of a file.
@param filename: Path to the file to verify the signature against.
@param signature: Path to signature to use.
@param gpg: Optionally, path to the GPG binary to use.
@return: a C{Deferred} resulting in C{True} if the signature is
valid, C{False} otherwise.
"""
def remove_gpg_home(ignored):
shutil.rmtree(gpg_home)
return ignored
def check_gpg_exit_code(args):
out, err, code = args
# We want a nice error message with Python 3 as well, so decode the
# bytes here.
out, err = out.decode("ascii"), err.decode("ascii")
if code != 0:
raise InvalidGPGSignature("%s failed (out='%s', err='%s', "
"code='%d')" % (gpg, out, err, code))
gpg_home = tempfile.mkdtemp()
args = ("--no-options", "--homedir", gpg_home, "--no-default-keyring",
"--ignore-time-conflict", "--keyring", "/etc/apt/trusted.gpg",
"--verify", signature, filename)
result = getProcessOutputAndValue(gpg, args=args)
result.addBoth(remove_gpg_home)
result.addCallback(check_gpg_exit_code)
return result
def getProcessOutput(binary,
args,
getProcessOutputAndValue=getProcessOutputAndValue,
checkStatus=_checkStatus):
"""
Spawn a new process and return the standard output.
@type binary: C{str}
@param binary: Path to the binary to spawn.
@type args: C{sequence} of C{str}
@param args: Arguments to pass when spawning L{binary}.
@param getProcessOutputAndValue: Callable to spawn a process and return
C{(standard output, standard error, exit status)}. Defaults to
L{twisted.internet.utils.getProcessOutputAndValue}
@raise L{ExternalProcessError}: If the exit status is non-zero.
@rtype: C{Deferred<str>}
@return: Deferred firing with the standard output of L{binary}.
"""
def _log((out, err, code)):
name = FilePath(binary).basename()
log.msg('%s command: %r' % (name, (binary, args)))
def _check((out, err, code)):
if not checkStatus((out, err, code)):
raise ExternalProcessError(binary, args, code, (out, err))
return out
d = getProcessOutputAndValue(binary, args)
d.addCallback(defertee, _log)
d.addCallback(_check)
return d
def git_init(_):
log.msg('gitpoller: initializing working dir from %s' % self.repourl)
d = utils.getProcessOutputAndValue(self.gitbin,
['init', '--bare', self.workdir], env=os.environ)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
return d
def git_remote_add(_):
d = utils.getProcessOutputAndValue(self.gitbin,
['remote', 'add', 'origin', self.repourl],
path=self.workdir, env=os.environ)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
return d
def test_buildbot_cvs_mail_with_unknown_opt_exits_with_error(self):
d = utils.getProcessOutputAndValue(
sys.executable,
[
self.buildbot_cvs_mail_path,
"[email protected]",
"-P",
"test",
"--path",
"test",
"-R",
"*****@*****.**",
"-t",
"README",
"1.1",
"1.2",
"hello.c",
"2.2",
"2.3",
],
)
def check(xxx_todo_changeme3):
(stdout, stderr, code) = xxx_todo_changeme3
self.assertEqual(code, 2)
d.addCallback(check)
return d
def _dovccmdImpl(self, command, args, path, ssh_workdir):
full_args = []
full_env = os.environ.copy()
if self._isSshPrivateKeyNeededForCommand(command):
key_path = self._getSshPrivateKeyPath(ssh_workdir)
self._downloadSshPrivateKey(key_path)
known_hosts_path = None
if self.sshHostKey is not None or self.sshKnownHosts is not None:
known_hosts_path = self._getSshKnownHostsPath(ssh_workdir)
self._downloadSshKnownHosts(known_hosts_path)
self.adjustCommandParamsForSshPrivateKey(full_args, full_env,
key_path, None,
known_hosts_path)
full_args += [command] + args
res = yield utils.getProcessOutputAndValue(self.gitbin,
full_args, path=path, env=full_env)
(stdout, stderr, code) = res
stdout = bytes2unicode(stdout, self.encoding)
stderr = bytes2unicode(stderr, self.encoding)
if code != 0:
if code == 128:
raise GitError('command {} in {} on repourl {} failed with exit code {}: {}'.format(
full_args, path, self.repourl, code, stderr))
raise EnvironmentError('command {} in {} on repourl {} failed with exit code {}: {}'.format(
full_args, path, self.repourl, code, stderr))
return stdout.strip()
def test_baddir(self):
self.skip_if_cannot_daemonize()
basedir = self.workdir("test_baddir")
fileutil.make_dirs(basedir)
d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", "--basedir", basedir], env=os.environ)
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 1)
self.failUnless("does not look like a node directory" in err, err)
d.addCallback(_cb)
def _then_stop_it(res):
return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", "--basedir", basedir], env=os.environ)
d.addCallback(_then_stop_it)
def _cb2(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 2)
self.failUnless("does not look like a running node directory" in err)
d.addCallback(_cb2)
def _then_start_in_bogus_basedir(res):
not_a_dir = os.path.join(basedir, "bogus")
return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", "--basedir", not_a_dir], env=os.environ)
d.addCallback(_then_start_in_bogus_basedir)
def _cb3(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 1)
self.failUnless("does not look like a directory at all" in err, err)
d.addCallback(_cb3)
return d
def test_get_processOutputAndValueStdin(self):
"""
Standard input can be made available to the child process by passing
bytes for the `stdinBytes` parameter.
"""
scriptFile = self.makeSourceFile([
"import sys",
"sys.stdout.write(sys.stdin.read())",
])
stdinBytes = b"These are the bytes to see."
d = utils.getProcessOutputAndValue(
self.exe,
['-u', scriptFile],
stdinBytes=stdinBytes,
)
def gotOutputAndValue(out_err_code):
out, err, code = out_err_code
# Avoid making an exact equality comparison in case there is extra
# random output on stdout (warnings, stray print statements,
# logging, who knows).
self.assertIn(stdinBytes, out)
self.assertEqual(0, code)
d.addCallback(gotOutputAndValue)
return d
def test_outputAndValue(self):
"""
The L{Deferred} returned by L{getProcessOutputAndValue} fires with a
three-tuple, the elements of which give the data written to the child's
stdout, the data written to the child's stderr, and the exit status of
the child.
"""
scriptFile = self.makeSourceFile([
"import sys",
"if hasattr(sys.stdout, 'buffer'):",
" # Python 3",
" sys.stdout.buffer.write(b'hello world!\\n')",
" sys.stderr.buffer.write(b'goodbye world!\\n')",
"else:",
" # Python 2",
" sys.stdout.write(b'hello world!\\n')",
" sys.stderr.write(b'goodbye world!\\n')",
"sys.exit(1)"
])
def gotOutputAndValue(out_err_code):
out, err, code = out_err_code
self.assertEqual(out, b"hello world!\n")
if _PY3:
self.assertEqual(err, b"goodbye world!\n")
else:
self.assertEqual(err, b"goodbye world!" +
os.linesep)
self.assertEqual(code, 1)
d = utils.getProcessOutputAndValue(self.exe, ["-u", scriptFile])
return d.addCallback(gotOutputAndValue)
def spawnChild():
d = succeed(None)
d.addCallback(lambda dummy: utils.getProcessOutputAndValue(
pyExe, [b"-m", us] + args, env=properEnv,
reactor=reactor))
d.addCallback(processFinished)
d.addBoth(shutdown)
def git_fetch_origin(_):
d = utils.getProcessOutputAndValue(
self.gitbin, ["fetch", "origin"], path=self.workdir, env=dict(PATH=os.environ["PATH"])
)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
return d
def getQemuOutputAndValue(executable,
args=(),
env={},
path=None,
reactor=None):
exe = abspath_qemu(executable)
return getProcessOutputAndValue(exe, args, env, path, reactor)
def query(self, changenumber):
"""
Run gerrit query and return the data result
"""
cmd = self._gerritCmd(
"query --format json --patch-sets limit:1 change:%s" %
(changenumber, ))
result = yield getProcessOutputAndValue(cmd[0], cmd[1:])
(out, err, status) = result
if status != 0:
log.msg("forcegerritrebuild: gerrit status error: %d %s" %
(status, err))
raise ValidationError("Error response from Gerrit %s" %
(self.gerrit_server, ))
try:
out = out.strip()
reslist = [
json.loads(bytes2unicode(_.strip()))
for _ in out.split(b"\n")
]
qstatus = reslist[-1]
if qstatus['rowCount'] != 1:
return None
dataresult = reslist[0]
except Exception as e:
raise ValidationError(
"Error processing response from Gerrit: %s" % (e, ))
return dataresult
def test_outputSignal(self):
"""
If the child process exits because of a signal, the L{Deferred}
returned by L{getProcessOutputAndValue} fires a L{Failure} of a tuple
containing the child's stdout, stderr, and the signal which caused
it to exit.
"""
# Use SIGKILL here because it's guaranteed to be delivered. Using
# SIGHUP might not work in, e.g., a buildbot slave run under the
# 'nohup' command.
scriptFile = self.makeSourceFile([
"import sys, os, signal", "sys.stdout.write('stdout bytes\\n')",
"sys.stderr.write('stderr bytes\\n')", "sys.stdout.flush()",
"sys.stderr.flush()", "os.kill(os.getpid(), signal.SIGKILL)"
])
def gotOutputAndValue(out_err_sig):
out, err, sig = out_err_sig
self.assertEqual(out, "stdout bytes\n")
self.assertEqual(err, "stderr bytes\n")
self.assertEqual(sig, signal.SIGKILL)
d = utils.getProcessOutputAndValue(self.exe, ['-u', scriptFile])
d = self.assertFailure(d, tuple)
return d.addCallback(gotOutputAndValue)
def _run(self, program, arguments, error_message):
out, err, code = yield getProcessOutputAndValue(program, args=arguments, env=os.environ)
if code not in (0, 1): # Why do we allow 1? Without it uploads seem to sometimes say they fail. IDK
self.log("{:d}", code)
self.log(out)
self.log(err)
raise self.master.modules["commands"].exception("{} [CODE: {:d}] [OUT: {}] [ERR: {}]".format(error_message, code, out.replace("\n"," ")[-60:], err.replace("\n"," ")[-60:]))
def method(testcase):
testcase.expectCommands(Expect("command"))
testcase.expectCommands(Expect("command2"))
d = utils.getProcessOutputAndValue("command", ())
d.addCallback(self.assertEqual, ('', '', 0))
d.addCallback(lambda _: testcase.assertAllCommandsRan())
return d
def call(args, options=None):
PIOCoreRPC.setup_multithreading_std_streams()
cwd = (options or {}).get("cwd") or os.getcwd()
for i, arg in enumerate(args):
if isinstance(arg, string_types):
args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg
else:
args[i] = str(arg)
def _call_inline():
with util.cd(cwd):
exit_code = __main__.main(["-c"] + args)
return (PIOCoreRPC.thread_stdout.get_value_and_reset(),
PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code)
if args and args[0] in ("account", "remote"):
d = utils.getProcessOutputAndValue(
helpers.get_core_fullpath(),
args,
path=cwd,
env={k: v
for k, v in os.environ.items() if "%" not in k})
else:
d = threads.deferToThread(_call_inline)
d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args)
d.addErrback(PIOCoreRPC._call_errback)
return d
def _finished(ignored):
fd.close()
d = getProcessOutputAndValue(
'aunpack', (tp.path, '-X', downloadTempPath.path))
d.addErrback(log.err)
d.addCallback(_extracted)
return d
def _find_utf8_locale(self):
# Click really wants to be running under a unicode-capable locale,
# especially on python3. macOS has en-US.UTF-8 but not C.UTF-8, and
# most linux boxes have C.UTF-8 but not en-US.UTF-8 . For tests,
# figure out which one is present and use that. For runtime, it's a
# mess, as really the user must take responsibility for setting their
# locale properly. I'm thinking of abandoning Click and going back to
# twisted.python.usage to avoid this problem in the future.
(out, err, rc) = yield getProcessOutputAndValue("locale", ["-a"])
if rc != 0:
log.msg("error running 'locale -a', rc=%s" % (rc, ))
log.msg("stderr: %s" % (err, ))
returnValue(None)
out = out.decode("utf-8") # make sure we get a string
utf8_locales = {}
for locale in out.splitlines():
locale = locale.strip()
if locale.lower().endswith((".utf-8", ".utf8")):
utf8_locales[locale.lower()] = locale
for wanted in ["C.utf8", "C.UTF-8", "en_US.utf8", "en_US.UTF-8"]:
if wanted.lower() in utf8_locales:
returnValue(utf8_locales[wanted.lower()])
if utf8_locales:
returnValue(list(utf8_locales.values())[0])
returnValue(None)
def test_buildbot_cvs_mail_without_cvsroot_opt_exits_with_error(self):
d = utils.getProcessOutputAndValue(
sys.executable,
[
self.buildbot_cvs_mail_path,
"--complete-garbage-opt=gomi",
'--cvsroot="ext:example.com:/cvsroot"',
"[email protected]",
"-P",
"test",
"--path",
"test",
"-R",
"*****@*****.**",
"-t",
"README",
"1.1",
"1.2",
"hello.c",
"2.2",
"2.3",
],
)
def check(xxx_todo_changeme2):
(stdout, stderr, code) = xxx_todo_changeme2
self.assertEqual(code, 2)
d.addCallback(check)
return d
def is_runnable(self):
# One property of Versioneer is that many changes to the source tree
# (making a commit, dirtying a previously-clean tree) will change the
# version string. Entrypoint scripts frequently insist upon importing
# a library version that matches the script version (whatever was
# reported when 'pip install' was run), and throw a
# DistributionNotFound error when they don't match. This is really
# annoying in a workspace created with "pip install -e .", as you
# must re-run pip after each commit.
# So let's report just one error in this case (from test_version),
# and skip the other tests that we know will fail.
# Setting LANG/LC_ALL to a unicode-capable locale is necessary to
# convince Click to not complain about a forced-ascii locale. My
# apologies to folks who want to run tests on a machine that doesn't
# have the C.UTF-8 locale installed.
locale = yield locale_finder.find_utf8_locale()
if not locale:
raise unittest.SkipTest("unable to find UTF-8 locale")
locale_env = dict(LC_ALL=locale, LANG=locale)
wormhole = self.find_executable()
res = yield getProcessOutputAndValue(wormhole, ["--version"],
env=locale_env)
out, err, rc = res
if rc != 0:
log.msg("wormhole not runnable in this tree:")
log.msg("out", out)
log.msg("err", err)
log.msg("rc", rc)
raise unittest.SkipTest("wormhole is not runnable in this tree")
returnValue(locale_env)
def _finished(ignored):
fd.close()
d = getProcessOutputAndValue(
'aunpack', (tp.path, '-X', downloadTempPath.path))
d.addErrback(log.err)
d.addCallback(_extracted)
return d
def _find_utf8_locale(self):
# Click really wants to be running under a unicode-capable locale,
# especially on python3. macOS has en-US.UTF-8 but not C.UTF-8, and
# most linux boxes have C.UTF-8 but not en-US.UTF-8 . For tests,
# figure out which one is present and use that. For runtime, it's a
# mess, as really the user must take responsibility for setting their
# locale properly. I'm thinking of abandoning Click and going back to
# twisted.python.usage to avoid this problem in the future.
(out, err, rc) = yield getProcessOutputAndValue("locale", ["-a"])
if rc != 0:
log.msg("error running 'locale -a', rc=%s" % (rc,))
log.msg("stderr: %s" % (err,))
returnValue(None)
out = out.decode("utf-8") # make sure we get a string
utf8_locales = {}
for locale in out.splitlines():
locale = locale.strip()
if locale.lower().endswith((".utf-8", ".utf8")):
utf8_locales[locale.lower()] = locale
for wanted in ["C.utf8", "C.UTF-8", "en_US.utf8", "en_US.UTF-8"]:
if wanted.lower() in utf8_locales:
returnValue(utf8_locales[wanted.lower()])
if utf8_locales:
returnValue(list(utf8_locales.values())[0])
returnValue(None)
def render_POST(self, request):
challenge, data = decrypt_from_appengine(self.secret, request.content.read(), self.serverTime)
register_details = json.loads(data)
username = register_details["username"]
server = register_details["server"]
def result(results):
out, err, code = results
if code == 0:
log.msg("Successfully removed Rogerthat mobile account!\noutput: %s\nerrors: %s\nexit code: %s" % results)
else:
log.msg("Failed to remove Rogerthat mobile account!\noutput: %s\nerrors: %s\nexit code: %s" % results)
data = json.dumps((code == 0, code, out, err))
request.write(encrypt_for_appengine(self.secret, challenge, data))
request.finish()
def killed(results):
out, err, signalNum = results
log.msg("Command to remove Rogerthat mobile account was killed!\noutput: %s\nerrors: %s\nsignal: %s" % results)
data = json.dumps((False, signalNum, out, err))
request.write(encrypt_for_appengine(self.secret, challenge, data))
request.finish()
d = getProcessOutputAndValue(_UNREGISTER_CMD, (username, server))
d.addCallback(result)
d.addErrback(killed)
return NOT_DONE_YET
def _pingIp(self):
maxTries = self.config.tries
attempts = 0
exitCode = -1
timestamp = None
while attempts < maxTries:
attempts += 1
cmd, args = _getPingCmd(ip=self.config.ip, version=self.config.ipVersion,
ttl=64, timeout=float(self._preferences.pingTimeOut),
datalength=self._daemon.options.dataLength)
log.debug("%s %s", cmd, " ".join(args))
timestamp = time.time()
out, err, exitCode = yield utils.getProcessOutputAndValue(cmd, args)
pingResult = PingResult(self.config.ip, exitCode, out, timestamp)
self.logPingResult(pingResult)
if not self.config.points and exitCode == 0:
# if there are no datapoints to store
# and there is at least 1 ping up, then go on
break
if self.isUp:
log.debug("%s is up!", self.config.ip)
self.sendPingUp()
else:
log.debug("%s is down", self.config.ip)
self.sendPingDown()
self.storeResults()
def spawnChild():
d = succeed(None)
d.addCallback(lambda dummy: utils.getProcessOutputAndValue(
pyExe, [b"-m", us] + args, env=properEnv,
reactor=reactor))
d.addCallback(processFinished)
d.addBoth(shutdown)
def method(testcase):
testcase.expectCommands(Expect("command"))
testcase.expectCommands(Expect("command2"))
d = utils.getProcessOutputAndValue("command", ())
d.addCallback(self.assertEqual, ('', '', 0))
d.addCallback(lambda _: testcase.assertAllCommandsRan())
return d
def git_init(_):
log.msg('gitpoller: initializing working dir from %s' % self.repourl)
d = utils.getProcessOutputAndValue(self.gitbin,
['init', self.workdir], env=os.environ)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
return d
def testLingeringClose(self):
args = ('-u', util.sibpath(__file__, "simple_client.py"),
"lingeringClose", str(self.port), self.type)
d = waitForDeferred(utils.getProcessOutputAndValue(sys.executable, args=args))
yield d; out,err,code = d.getResult()
self.assertEquals(code, 0, "Error output:\n%s" % (err,))
self.assertEquals(out, "HTTP/1.1 402 Payment Required\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
def execute_commands(self, torrent_id, event, *arg):
torrent = component.get("TorrentManager").torrents[torrent_id]
info = torrent.get_status(["name", "save_path", "move_on_completed", "move_on_completed_path"])
# Grab the torrent name and save path
torrent_name = info["name"]
if event == "complete":
save_path = info["move_on_completed_path"] if info["move_on_completed"] else info["save_path"]
elif event == "added" and arg[0]:
# No futher action as from_state (arg[0]) is True
return
else:
save_path = info["save_path"]
log.debug("[execute] Running commands for %s", event)
def log_error(result, command):
(stdout, stderr, exit_code) = result
if exit_code:
log.warn("[execute] command '%s' failed with exit code %d", command, exit_code)
if stdout:
log.warn("[execute] stdout: %s", stdout)
if stderr:
log.warn("[execute] stderr: %s", stderr)
# Go through and execute all the commands
for command in self.config["commands"]:
if command[EXECUTE_EVENT] == event:
command = os.path.expandvars(command[EXECUTE_COMMAND])
command = os.path.expanduser(command)
log.debug("[execute] running %s", command)
d = getProcessOutputAndValue(command, (torrent_id, torrent_name, save_path), env=os.environ)
d.addCallback(log_error, command)
def _dovccmd(self, command, args, path=None):
def encodeArg(arg):
if isinstance(arg, list):
return [encodeArg(a) for a in arg]
elif isinstance(arg, unicode):
return arg.encode("ascii")
return arg
d = utils.getProcessOutputAndValue(encodeArg(self.gitbin),
encodeArg([command] + args),
path=encodeArg(path), env=os.environ)
def _convert_nonzero_to_failure(res,
command,
args,
path):
"utility to handle the result of getProcessOutputAndValue"
(stdout, stderr, code) = res
if code != 0:
if code == 128:
raise GitError('command %s %s in %s on repourl %s failed with exit code %d: %s'
% (command, args, path, self.repourl, code, stderr))
raise EnvironmentError('command %s %s in %s on repourl %s failed with exit code %d: %s'
% (command, args, path, self.repourl, code, stderr))
return stdout.strip()
d.addCallback(_convert_nonzero_to_failure,
command,
args,
path)
return d
def _pingIp(self):
maxTries = self.config.tries
attempts = 0
exitCode = -1
timestamp = None
while attempts < maxTries:
attempts += 1
cmd, args = _getPingCmd(ip=self.config.ip,
version=self.config.ipVersion,
ttl=64,
timeout=float(
self._preferences.pingTimeOut),
datalength=self._daemon.options.dataLength)
log.debug("%s %s", cmd, " ".join(args))
timestamp = time.time()
out, err, exitCode = yield utils.getProcessOutputAndValue(
cmd, args)
pingResult = PingResult(self.config.ip, exitCode, out, timestamp)
self.logPingResult(pingResult)
if not self.config.points and exitCode == 0:
# if there are no datapoints to store
# and there is at least 1 ping up, then go on
break
if self.isUp:
log.debug("%s is up!", self.config.ip)
self.sendPingUp()
else:
log.debug("%s is down", self.config.ip)
self.sendPingDown()
self.storeResults()
def git_remote_add(_):
d = utils.getProcessOutputAndValue(self.gitbin,
['remote', 'add', 'origin', self.repourl],
path=self.workdir, env=os.environ)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
return d
def execute(self):
"""Execute the CLI command.
Returns a deferred that will be called when the operation completes. The
parameter to the deferred is this object.
"""
def __cli_output_callback(result):
"""Callback from getProcessOutputAndValue"""
self._set_properties(result)
LOGGER.debug("Asterisk CLI %s exited %d" %
(self.host, self.exitcode))
if self.err:
LOGGER.debug(self.err)
if self.exitcode:
self._deferred.errback(self)
else:
self._deferred.callback(self)
def __cli_error_callback(result):
"""Errback from getProcessOutputAndValue"""
self._set_properties(result)
LOGGER.warning("Asterisk CLI %s exited %d with error: %s" %
(self.host, self.exitcode, self.err))
if self.err:
LOGGER.debug(self.err)
self._deferred.errback(self)
self._deferred = defer.Deferred()
deferred = utils.getProcessOutputAndValue(self._cmd[0],
self._cmd,
env=os.environ)
deferred.addCallbacks(callback=__cli_output_callback,
errback=__cli_error_callback,)
return self._deferred
def _send_configuration(self,
application_config_yaml=COMPLEX_APPLICATION_YAML,
deployment_config_yaml=COMPLEX_DEPLOYMENT_YAML):
"""
Run ``flocker-deploy`` against the API server.
:param application_config: Application configuration dictionary.
:param deployment_config: Deployment configuration dictionary.
:return: ``Deferred`` that fires with a tuple (stdout, stderr,
exit code).
"""
app_config = FilePath(self.mktemp())
app_config.setContent(safe_dump(application_config_yaml))
deployment_config = FilePath(self.mktemp())
deployment_config.setContent(safe_dump(deployment_config_yaml))
# This duplicates some code in the acceptance tests...
# https://clusterhq.atlassian.net/browse/FLOC-1904
return getProcessOutputAndValue(
b"flocker-deploy", [
b"--certificates-directory", self.certificate_path.path,
b"--port",
unicode(self.port_number).encode("ascii"), b"localhost",
deployment_config.path, app_config.path
],
env=environ)
def _getBatchOutput(self, f):
fn = self.mktemp()
open(fn, 'w').write(f)
port = self.server.getHost().port
cmds = ('-p %i -l testuser '
'--known-hosts kh_test '
'--user-authentications publickey '
'--host-key-algorithms ssh-rsa '
'-i dsa_test '
'-a '
'-v -b %s 127.0.0.1') % (port, fn)
cmds = test_conch._makeArgs(cmds.split(), mod='cftp')[1:]
log.msg('running %s %s' % (sys.executable, cmds))
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
self.server.factory.expectedLoseConnection = 1
d = getProcessOutputAndValue(sys.executable, cmds, env=env)
def _cleanup(res):
os.remove(fn)
return res
d.addCallback(lambda res: res[0])
d.addBoth(_cleanup)
return d
def test_extendedAttributes(self):
"""
Test the return of extended attributes by the server: the sftp client
should ignore them, but still be able to parse the response correctly.
This test is mainly here to check that
L{filetransfer.FILEXFER_ATTR_EXTENDED} has the correct value.
"""
fn = self.mktemp()
open(fn, 'w').write("ls .\nexit")
port = self.server.getHost().port
oldGetAttr = FileTransferForTestAvatar._getAttrs
def _getAttrs(self, s):
attrs = oldGetAttr(self, s)
attrs["ext_foo"] = "bar"
return attrs
self.patch(FileTransferForTestAvatar, "_getAttrs", _getAttrs)
self.server.factory.expectedLoseConnection = True
cmds = ('-o', 'IdentityFile=dsa_test',
'-o', 'UserKnownHostsFile=kh_test',
'-o', 'HostKeyAlgorithms=ssh-rsa',
'-o', 'Port=%i' % (port,), '-b', fn, '[email protected]')
d = getProcessOutputAndValue("sftp", cmds)
def check(result):
self.assertEqual(result[2], 0)
for i in ['testDirectory', 'testRemoveFile',
'testRenameFile', 'testfile1']:
self.assertIn(i, result[0])
return d.addCallback(check)
def initiateMotionCaptureCycle(self):
if self.state != TreatCam.IDLE:
return
LOGGER.debug("Initiating motion capture cycle")
self.state = TreatCam.PENDING_MOTION_CAPTURE
deferred = utils.getProcessOutputAndValue(executable=self.config.motionCaptureProgram, args=self.config.motionCaptureProgramArgs.split(' '), reactor=self.reactor)
deferred.addCallbacks(self.motionCapture, self.motionCaptureError)
def test_outputSignal(self):
"""
If the child process exits because of a signal, the L{Deferred}
returned by L{getProcessOutputAndValue} fires a L{Failure} of a tuple
containing the the child's stdout, stderr, and the signal which caused
it to exit.
"""
# Use SIGKILL here because it's guaranteed to be delivered. Using
# SIGHUP might not work in, e.g., a buildbot slave run under the
# 'nohup' command.
scriptFile = self.makeSourceFile([
"import sys, os, signal",
"sys.stdout.write('stdout bytes\\n')",
"sys.stderr.write('stderr bytes\\n')",
"sys.stdout.flush()",
"sys.stderr.flush()",
"os.kill(os.getpid(), signal.SIGKILL)"])
def gotOutputAndValue((out, err, sig)):
self.assertEquals(out, "stdout bytes\n")
self.assertEquals(err, "stderr bytes\n")
self.assertEquals(sig, signal.SIGKILL)
d = utils.getProcessOutputAndValue(self.exe, ['-u', scriptFile])
d = self.assertFailure(d, tuple)
return d.addCallback(gotOutputAndValue)
def _getBatchOutput(self, f):
fn = tempfile.mktemp()
open(fn, 'w').write(f)
l = []
port = self.server.getHost().port
cmds = ('-p %i -l testuser '
'--known-hosts kh_test '
'--user-authentications publickey '
'--host-key-algorithms ssh-rsa '
'-K direct '
'-i dsa_test '
'-a --nocache '
'-v -b %s 127.0.0.1') % (port, fn)
cmds = test_conch._makeArgs(cmds.split(), mod='cftp')[1:]
log.msg('running %s %s' % (sys.executable, cmds))
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
d = getProcessOutputAndValue(sys.executable, cmds, env=env)
d.setTimeout(10)
d.addBoth(l.append)
while not l:
if hasattr(self.server.factory, 'proto'):
self.server.factory.proto.expectedLoseConnection = 1
reactor.iterate(0.1)
os.remove(fn)
result = l[0]
if isinstance(result, failure.Failure):
raise result.value
else:
log.msg(result[1])
return result[0]
def is_runnable(self):
# One property of Versioneer is that many changes to the source tree
# (making a commit, dirtying a previously-clean tree) will change the
# version string. Entrypoint scripts frequently insist upon importing
# a library version that matches the script version (whatever was
# reported when 'pip install' was run), and throw a
# DistributionNotFound error when they don't match. This is really
# annoying in a workspace created with "pip install -e .", as you
# must re-run pip after each commit.
# So let's report just one error in this case (from test_version),
# and skip the other tests that we know will fail.
# Setting LANG/LC_ALL to a unicode-capable locale is necessary to
# convince Click to not complain about a forced-ascii locale. My
# apologies to folks who want to run tests on a machine that doesn't
# have the en_US.UTF-8 locale installed.
wormhole = self.find_executable()
d = getProcessOutputAndValue(wormhole, ["--version"],
env=dict(LC_ALL="en_US.UTF-8",
LANG="en_US.UTF-8"))
def _check(res):
out, err, rc = res
if rc != 0:
log.msg("wormhole not runnable in this tree:")
log.msg("out", out)
log.msg("err", err)
log.msg("rc", rc)
raise unittest.SkipTest("wormhole is not runnable in this tree")
d.addCallback(_check)
return d
def process_changes(self, changes):
logging.debug(">>>>>>>> processing changes: %s" % changes)
logging.debug(">>>>>>>> git fetch:")
x = defer.waitForDeferred(
utils.getProcessOutputAndValue('/usr/bin/git',
['--no-pager', 'fetch', '--all'],
path=self.src_dir))
yield x
out, err, code = x.getResult()
if code != 0:
logging.error('(%s) in git fetch: %s' % (code, err))
return
logging.debug(">>>>>>>> git reset:")
logging.debug("processing changes: %s" % changes)
x = defer.waitForDeferred(
utils.getProcessOutputAndValue(
'/usr/bin/git',
['--no-pager', 'reset', '--hard', 'origin/master'],
path=self.src_dir))
yield x
out, err, code = x.getResult()
if code != 0:
logging.error('(%s) in git reset: %s' % (code, err))
return
logging.debug(">>>>>>>> git submodule:")
x = defer.waitForDeferred(
utils.getProcessOutputAndValue(
'/usr/bin/git',
['--no-pager', 'submodule', 'update', '--init'],
path=self.src_dir))
yield x
out, err, code = x.getResult()
if code != 0:
logging.error('(%s) in git submodule: %s' % (code, err))
return
logging.debug(">>>>>>>> buildbot reconfig:")
x = utils.getProcessOutputAndValue('/usr/bin/buildbot',
['reconfig', self.master_dir])
yield x
if code != 0:
logging.error('(%s) in buildbot reconfig: %s' % (code, err))
return
logging.debug(">>>>>>>> done.")
def runProcessLogFailures(bin, args, expectedCode=0):
stdout, stderr, code = yield utils.getProcessOutputAndValue(bin, args)
if code != expectedCode:
log.err(('Got unexpected return code when running {} {}: '
'code: {}, stdout: {}, stderr: {}').format(
bin, args, code, stdout, stderr))
return False
return True
def git_fetch_origin(_):
args = ['fetch', 'origin']
self._extend_with_fetch_refspec(args)
d = utils.getProcessOutputAndValue(self.gitbin, args,
path=self.workdir, env=os.environ)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
return d
def get_rev(_):
d = utils.getProcessOutputAndValue(self.gitbin,
['rev-parse', self.branch],
path=self.workdir, env=os.environ)
d.addCallback(self._convert_nonzero_to_failure)
d.addErrback(self._stop_on_failure)
d.addCallback(lambda (out, err, code) : out.strip())
return d
def runStart(self, **config):
args=[
'-c',
'from buildbot.scripts.start import start; start(%r)' % (mkconfig(**config),),
]
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
return getProcessOutputAndValue(sys.executable, args=args, env=env)
def _call_subprocess(args, options):
cwd = (options or {}).get("cwd") or os.getcwd()
return utils.getProcessOutputAndValue(
helpers.get_core_fullpath(),
args,
path=cwd,
env={k: v for k, v in os.environ.items() if "%" not in k},
)
def test_buildbot_cvs_mail_no_args_exits_with_error(self):
d = utils.getProcessOutputAndValue(sys.executable, [self.buildbot_cvs_mail_path])
def check(xxx_todo_changeme):
(stdout, stderr, code) = xxx_todo_changeme
self.assertEqual(code, 2)
d.addCallback(check)
return d
def _on_torrent_finished(self, torrent_id):
"""
This is called when a torrent finishes and checks if any files to extract.
"""
tid = component.get('TorrentManager').torrents[torrent_id]
tid_status = tid.get_status(['download_location', 'name'])
files = tid.get_files()
for f in files:
file_root, file_ext = os.path.splitext(f['path'])
file_ext_sec = os.path.splitext(file_root)[1]
if file_ext_sec and file_ext_sec + file_ext in EXTRACT_COMMANDS:
file_ext = file_ext_sec + file_ext
elif file_ext not in EXTRACT_COMMANDS or file_ext_sec == '.tar':
log.debug('Cannot extract file with unknown file type: %s',
f['path'])
continue
elif file_ext == '.rar' and 'part' in file_ext_sec:
part_num = file_ext_sec.split('part')[1]
if part_num.isdigit() and int(part_num) != 1:
log.debug('Skipping remaining multi-part rar files: %s',
f['path'])
continue
cmd = EXTRACT_COMMANDS[file_ext]
fpath = os.path.join(tid_status['download_location'],
os.path.normpath(f['path']))
dest = os.path.normpath(self.config['extract_path'])
if self.config['use_name_folder']:
dest = os.path.join(dest, tid_status['name'])
try:
os.makedirs(dest)
except OSError as ex:
if not (ex.errno == errno.EEXIST and os.path.isdir(dest)):
log.error('Error creating destination folder: %s', ex)
break
def on_extract(result, torrent_id, fpath):
# Check command exit code.
if not result[2]:
log.info('Extract successful: %s (%s)', fpath, torrent_id)
else:
log.error('Extract failed: %s (%s) %s', fpath, torrent_id,
result[1])
# Run the command and add callback.
log.debug(
'Extracting %s from %s with %s %s to %s',
fpath,
torrent_id,
cmd[0],
cmd[1],
dest,
)
d = getProcessOutputAndValue(cmd[0], cmd[1].split() + [str(fpath)],
os.environ, str(dest))
d.addCallback(on_extract, torrent_id, fpath)
