def test_run_move_dir_with_logfile(self):
# Write log into a directory that gets renamed by the called command.
# The file should be opened just once at the start.
tmpdir = tempfile.mkdtemp(prefix='kobo-bug-')
logfile = os.path.join(tmpdir, 'file.log')
destdir = tmpdir + '.moved'
run(['mv', '-v', tmpdir, destdir], logfile=logfile, show_cmd=True)
self.assertTrue(os.path.isfile(os.path.join(destdir, 'file.log')))
def test_run_move_dir_with_logfile(self):
# Write log into a directory that gets renamed by the called command.
# The file should be opened just once at the start.
tmpdir = tempfile.mkdtemp(prefix='kobo-bug-')
logfile = os.path.join(tmpdir, 'file.log')
destdir = tmpdir + '.moved'
run(['mv', '-v', tmpdir, destdir], logfile=logfile, show_cmd=True)
self.assertTrue(os.path.isfile(os.path.join(destdir, 'file.log')))
def _gzip_log(self, name):
"""gzip one log, do *not* throw any error on failure"""
# compress only log files
if not name.endswith(".log"):
return
import pipes
path = self._get_absolute_log_path(name)
if not os.path.isfile(path + ".gz"):
run("gzip %s" % pipes.quote(path), can_fail=True, stdout=False)
def _gzip_log(self, name):
"""gzip one log, do *not* throw any error on failure"""
# compress only log files
if not name.endswith(".log"):
return
import pipes
path = self._get_absolute_log_path(name)
if not os.path.isfile(path + ".gz"):
run("gzip %s" % pipes.quote(path), can_fail=True, stdout=False)
def test_run_univ_nl_show_cmd_stdout(self, mock_out):
ret, out = run("echo foo", universal_newlines=True, show_cmd=True,
stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
self.assertEqual(mock_out.getvalue(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def test_run_logfile(self):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", logfile=logfile)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(), 'foo\n')
def test_run_univ_nl_logfile(self):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", universal_newlines=True, logfile=logfile)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(), 'foo\n')
def test_run_in_text_mode(self):
"""test run with kwargs 'text', 'encoding' or/and 'errors' set. These kwargs are
added to Popen from python3.6(encoding, errors) and python3.7(text). Running test
with python version older than 3.7 is expected to fail
"""
ret, out = run("echo hello", text=True)
self.assertEqual(ret, 0)
self.assertEqual(out, "hello\n")
ret, out = run("echo hello", encoding="utf-8")
self.assertEqual(ret, 0)
self.assertEqual(out, "hello\n")
ret, out = run("echo hello", errors="strict")
self.assertEqual(ret, 0)
self.assertEqual(out, "hello\n")
def get_ip_from_leases(mac):
cmd = ('cat /var/lib/libvirt/dnsmasq/default.leases |'
'grep "%s"' % mac)
rc, out = run(cmd, can_fail=True)
if int(rc):
return None
return out.split(' ')[2]
def test_run_logfile(self):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", logfile=logfile)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(), 'foo\n')
def test_run_logfile_stdout(self, mock_out):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", logfile=logfile, stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(), 'foo\n')
self.assertEqual(mock_out.getvalue(), 'foo\n')
def test_run_show_cmd_logfile(self):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", show_cmd=True, logfile=logfile)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def test_run_univ_nl_logfile(self):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", universal_newlines=True,
logfile=logfile)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(), 'foo\n')
def test_run_show_cmd_logfile(self):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", show_cmd=True, logfile=logfile)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def download_submodules(modules, destination):
for mod in modules:
rc, out = run('wget %(download_url)s' % mod,
can_fail=True, workdir=destination)
if rc:
sys.stdout.write('Failed to download %(fullname)s (%(commit)s)\n' % mod)
else:
sys.stdout.write('Downloaded %(fullname)s (%(commit)s)\n' % mod)
sys.stdout.flush()
def test_run_univ_nl_show_cmd_stdout(self, mock_out):
ret, out = run("echo foo",
universal_newlines=True,
show_cmd=True,
stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
self.assertEqual(mock_out.getvalue(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def test_run_logfile_stdout(self, mock_out):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", logfile=logfile, stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(), 'foo\n')
self.assertEqual(mock_out.getvalue(),
'foo\n')
def test_run_univ_nl_show_cmd_logfile_stdout(self, mock_out):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo", universal_newlines=True, show_cmd=True,
logfile=logfile, stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(),
'COMMAND: echo foo\n-----------------\nfoo\n')
self.assertEqual(mock_out.getvalue(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def test_run_univ_nl_show_cmd_logfile_stdout(self, mock_out):
logfile = os.path.join(self.tmp_dir, 'output.log')
ret, out = run("echo foo",
universal_newlines=True,
show_cmd=True,
logfile=logfile,
stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
with open(logfile) as f:
self.assertEqual(f.read(),
'COMMAND: echo foo\n-----------------\nfoo\n')
self.assertEqual(mock_out.getvalue(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def get_mac(name):
cmd = ('sudo virsh dumpxml "%s"' % name)
rc, out = run(cmd, can_fail=True)
if int(rc):
return None
content = fromstring(out)
output = []
ifsel = CSSSelector('interface[type="network"]')
macsel = CSSSelector('mac')
for interface in ifsel(content):
hwaddr = macsel(interface)[0]
output.append(hwaddr.get('address'))
return output
def run_test(test_with_args):
print "Executing tests in %-40s" % test_with_args[0]
retcode, output = run(['python'] + test_with_args, can_fail=True)
if retcode == 0:
# If tests were skipped, give caller a hint that this happened.
# Summary line example:
# "OK (skipped=9)"
if 'skipped=' in output.splitlines()[-1]:
print "[ SKIP ]"
else:
print "[ OK ]"
return True
else:
print "[ FAILED ]"
print output
return False
def run_test(test_with_args):
print("Executing tests in %-40s" % test_with_args[0])
retcode, output = run(['python'] + test_with_args, can_fail=True)
if retcode == 0:
# If tests were skipped, give caller a hint that this happened.
# Summary line example:
# "OK (skipped=9)"
if b'skipped=' in output.splitlines()[-1]:
print("[ SKIP ]")
else:
print("[ OK ]")
return True
else:
print("[ FAILED ]")
print(output)
return False
def test_run_show_cmd_stdout(self, mock_out):
ret, out = run("echo foo", show_cmd=True, stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
self.assertEqual(mock_out.getvalue(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def _curl(url,
method='GET',
content='application/json',
output='application/json',
data=None):
cmd = [
'curl', '-X', method, '-H',
'Accept: %s' % output, '-k', '-A', USER_AGENT, '-s'
]
if AUTH_USER:
# unusual case, perhaps a development/testing server.
# Note: curl wipes out the -u argument from the process table so it's not _too_
# insecure...
cmd += ['--anyauth', '-u', AUTH_USER]
else:
# typical case, authenticating by kerberos
cmd += ['--negotiate', '-u', ':']
data_file = None
if data:
# If there is data to send in the request, put it in a file to avoid hitting
# arg length limits
data_file = pack_data(data)
cmd += ['-H', 'Content-Type: %s' % content, '-d', '@' + data_file]
elif method in ['POST', 'PUT', 'PATCH']:
# Explicitly tell curl that we have no data; otherwise it will not set
# a Content-Length header, which breaks on some servers
cmd += ['-d', '']
# attempt the command
try:
if DEBUG:
print('Attempting URL: %s' % url)
print('full command: %s' % cmd)
cmd += [url]
rv, output = run(cmd, can_fail=True)
if DEBUG and output: print('OUTPUT:\n', output)
finally:
if data_file:
# Make sure to remove the tempory data file once curl is complete
try:
os.unlink(data_file)
except:
print("Warning: could not delete %s" % data_file)
# Handle exceptions
if rv:
raise ErrataToolError("failed: %s" % rv)
if '401 Authorization Required' in output:
raise ErrataToolError('Could not authenticate with ET. kinit?')
# output can be empty, for example, a successful update (PUT method) to an
# advisory will return no advisory information
if not output.strip():
return "{}"
einfo = simplejson.loads(output.replace('\n', ''))
if 'error' in einfo and einfo['error'] != '':
raise ErrataToolError("API Error: %s" % einfo['error'])
if 'errors' in einfo and len(einfo['errors']):
raise ErrataToolError("API Errors: %s" % str(einfo['errors']))
return output
def test_run_show_cmd(self):
ret, out = run("echo foo", show_cmd=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
def test_run_univ_nl(self):
ret, out = run("echo foo", universal_newlines=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
def test_run_show_cmd(self):
ret, out = run("echo foo", show_cmd=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
def test_run_stdout(self, mock_out):
ret, out = run("echo foo", stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
self.assertEqual(mock_out.getvalue(),
'foo\n')
def test_run_stdout(self, mock_out):
ret, out = run("echo foo", stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
self.assertEqual(mock_out.getvalue(), 'foo\n')
def test_run_univ_nl(self):
ret, out = run("echo foo", universal_newlines=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 'foo\n')
def test_run_show_cmd_stdout(self, mock_out):
ret, out = run("echo foo", show_cmd=True, stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b'foo\n')
self.assertEqual(mock_out.getvalue(),
'COMMAND: echo foo\n-----------------\nfoo\n')
def format_submodules(installdir, print_output=True):
"""Returns formated git module urls in GitHub format and current commit
hashes for SPEC file.
"""
# get urls
cmd = ('cat .gitmodules | grep "url"')
rc, out = run(cmd, can_fail=True)
modules = []
for line in out.split('\n'):
cont = line.strip().split('=')
if len(cont) < 2:
if line:
sys.stdout.write('Invalid line: %s\n' % line)
continue
mod = {}
mod['raw_url'] = cont[1].strip()
mod['base_url'] = (mod['raw_url'].endswith('.git')
and mod['raw_url'][:-4]
or mod['raw_url'])
mod['fullname'] = mod['base_url'].split('/')[-1]
modules.append(mod)
# get path for modules and github url
cmd = ('cat .gitmodules | grep "path"')
rc, out = run(cmd, can_fail=True)
index = 0
fmt = ('%(base_url)s/archive/%%{%(name)s_commit}/'
'%(name)s-%%{%(name)s_commit}.tar.gz')
for line in out.split('\n'):
cont = line.strip().split('=')
if len(cont) < 2:
if line:
sys.stdout.write('Invalid line: %s\n' % line)
continue
mod = modules[index]
mod['path'] = cont[1].strip()
mod['name'] = mod['path'].split('/')[-1]
mod['github_url'] = fmt % mod
mod['destination'] = '%s/%s' % (installdir, mod['name'])
index += 1
# sort according to name
modules.sort(key=lambda x: x['name'])
# get commit hashes
cmd = ('git submodule status')
rc, out = run(cmd, can_fail=True)
index = 0
for line in out.split('\n'):
cont = line.strip().split(' ')
# some modules just have one field
if len(cont) < 2:
if line:
sys.stdout.write('Invalid line: %s\n' % line)
continue
for mod in modules:
if mod['path'] == cont[1].strip():
mod['index'] = index
mod['commit'] = cont[0].strip()
url = mod['github_url'].replace('%%{%(name)s_commit}' % mod,
mod['commit'])
mod['download_url'] = url.replace('git://', 'https://')
index += 1
# print output
if print_output:
for head, fmt in [
('Globals:\n',
'%%global %(name)s_commit\t%(commit)s\n'),
('Source list:\n',
'Source%(index)s:\t%(github_url)s\n'),
('Installation script:\n',
'cp -r %(fullname)s-%%{%(name)s_commit} %(destination)s\n')]:
sys.stdout.write(head)
for mod in modules:
try:
prefix, suffix = fmt.split('\t')
except Exception:
sys.stdout.write(fmt % mod)
else:
prefix = prefix % mod
suffix = suffix % mod
space = 30 - len(prefix) + len(suffix)
sys.stdout.write(
'%s %s' % (
prefix, suffix.rjust(space)
)
)
sys.stdout.write('\n')
sys.stdout.flush()
return modules
def test_run(self):
ret, out = run("echo hello")
self.assertEqual(ret, 0)
self.assertEqual(out, b"hello\n")
ret, out = run(["echo", "'hello'"])
self.assertEqual(ret, 0)
self.assertEqual(out, b"'hello'\n")
ret, out = run(["echo", "\" ' "])
self.assertEqual(ret, 0)
self.assertEqual(out, b"\" ' \n")
# test a longer output that needs to be read in several chunks
ret, out = run("echo -n '%s'; sleep 0.2; echo -n '%s'" % (10000 * "x", 10 * "a"), logfile=self.tmp_file, can_fail=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 10000 * b"x" + 10 * b"a")
# check if log file is written properly; it is supposed to append data to existing content
self.assertEqual("\n".join(read_from_file(self.tmp_file)), "test" + 10000 * "x" + 10 * "a")
ret, out = run("exit 1", can_fail=True)
self.assertEqual(ret, 1)
self.assertRaises(RuntimeError, run, "exit 1")
# stdin test
ret, out = run("xargs -0 echo -n", stdin_data=b"\0".join([str(i).encode() for i in range(10000)]))
self.assertEqual(out, b" ".join([str(i).encode() for i in range(10000)]))
# return None
ret, out = run("xargs echo", stdin_data=b"\n".join([str(i).encode() for i in range(1000000)]), return_stdout=False)
self.assertEqual(out, None)
# log file with absolute path
log_file = os.path.join(self.tmp_dir, "a.log")
ret, out = run("echo XXX", logfile=log_file)
self.assertEqual(open(log_file, "r").read(), "XXX\n")
# log file with relative path
log_file = "b.log"
cwd = os.getcwd()
os.chdir(self.tmp_dir)
ret, out = run("echo XXX", logfile=log_file)
self.assertEqual(open(log_file, "r").read(), "XXX\n")
os.chdir(cwd)
# bashism - output redirection to subshells
# fails in default shell (/bin/sh)
self.assertRaises(RuntimeError, run, "echo foo | tee >(md5sum -b) >/dev/null")
# passes in bash
run("echo foo | tee >(md5sum -b) >/dev/null", executable="/bin/bash")
def test_run_split_in_middle_of_utf8_sequence(self):
cmd = "printf ' ' && bash -c \"printf 'č%.0s' {1..10000}\""
ret, out = run(cmd, stdout=True)
self.assertEqual(ret, 0)
self.assertEqual(out, b" " + b"\xc4\x8d" * 10000)
def test_run(self):
ret, out = run("echo hello")
self.assertEqual(ret, 0)
self.assertEqual(out, b"hello\n")
ret, out = run(["echo", "'hello'"])
self.assertEqual(ret, 0)
self.assertEqual(out, b"'hello'\n")
ret, out = run(["echo", "\" ' "])
self.assertEqual(ret, 0)
self.assertEqual(out, b"\" ' \n")
# test a longer output that needs to be read in several chunks
ret, out = run("echo -n '%s'; sleep 0.2; echo -n '%s'" %
(10000 * "x", 10 * "a"),
logfile=self.tmp_file,
can_fail=True)
self.assertEqual(ret, 0)
self.assertEqual(out, 10000 * b"x" + 10 * b"a")
# check if log file is written properly; it is supposed to append data to existing content
self.assertEqual("\n".join(read_from_file(self.tmp_file)),
"test" + 10000 * "x" + 10 * "a")
ret, out = run("exit 1", can_fail=True)
self.assertEqual(ret, 1)
self.assertRaises(RuntimeError, run, "exit 1")
# stdin test
ret, out = run("xargs -0 echo -n",
stdin_data=b"\0".join(
[str(i).encode() for i in range(10000)]))
self.assertEqual(out,
b" ".join([str(i).encode() for i in range(10000)]))
# return None
ret, out = run("xargs echo",
stdin_data=b"\n".join(
[str(i).encode() for i in range(1000000)]),
return_stdout=False)
self.assertEqual(out, None)
# log file with absolute path
log_file = os.path.join(self.tmp_dir, "a.log")
ret, out = run("echo XXX", logfile=log_file)
self.assertEqual(open(log_file, "r").read(), "XXX\n")
# log file with relative path
log_file = "b.log"
cwd = os.getcwd()
os.chdir(self.tmp_dir)
ret, out = run("echo XXX", logfile=log_file)
self.assertEqual(open(log_file, "r").read(), "XXX\n")
os.chdir(cwd)
# bashism - output redirection to subshells
# fails in default shell (/bin/sh)
self.assertRaises(RuntimeError, run,
"echo foo | tee >(md5sum -b) >/dev/null")
# passes in bash
run("echo foo | tee >(md5sum -b) >/dev/null", executable="/bin/bash")
# prepare environment for tests
PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
os.environ["PYTHONPATH"] = PROJECT_DIR
sys.path.insert(0, PROJECT_DIR)
from kobo.shortcuts import run
if __name__ == '__main__':
failed = False
for test in sorted(os.listdir(os.path.dirname(__file__))):
# run all tests that match the 'test_*.py" pattern
if not test.startswith("test_"):
continue
if not test.endswith(".py"):
continue
print "Executing tests in %-40s" % test,
retcode, output = run("python %s" % test, can_fail=True)
if retcode == 0:
print "[ OK ]"
print output
else:
failed = True
print "[ FAILED ]"
print output
if failed:
sys.exit(1)
sys.exit(0)
os.environ["PYTHONPATH"] = PROJECT_DIR
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
sys.path.insert(0, PROJECT_DIR)
from kobo.shortcuts import run
if __name__ == '__main__':
failed = False
for test in sorted(os.listdir(os.path.dirname(__file__))):
# run all tests that match the 'test_*.py" pattern
if not test.startswith("test_"):
continue
if not test.endswith(".py"):
continue
print "Executing tests in %-40s" % test,
retcode, output = run("python %s" % test, can_fail=True)
if retcode == 0:
print "[ OK ]"
else:
failed = True
print "[ FAILED ]"
print output
if failed:
sys.exit(1)
sys.exit(0)
