def ipexec(fname, options=None, commands=()):
"""Utility to call 'ipython filename'.
Starts IPython with a minimal and safe configuration to make startup as fast
as possible.
Note that this starts IPython in a subprocess!
Parameters
----------
fname : str, Path
Name of file to be executed (should have .py or .ipy extension).
options : optional, list
Extra command-line flags to be passed to IPython.
commands : optional, list
Commands to send in on stdin
Returns
-------
``(stdout, stderr)`` of ipython subprocess.
"""
__tracebackhide__ = True
if options is None:
options = []
cmdargs = default_argv() + options
test_dir = os.path.dirname(__file__)
ipython_cmd = get_ipython_cmd()
# Absolute path for filename
full_fname = os.path.join(test_dir, fname)
full_cmd = ipython_cmd + cmdargs + ['--', full_fname]
env = os.environ.copy()
# FIXME: ignore all warnings in ipexec while we have shims
# should we keep suppressing warnings here, even after removing shims?
env['PYTHONWARNINGS'] = 'ignore'
# env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
# Prevent coloring under PyCharm ("\x1b[0m" at the end of the stdout)
env.pop("PYCHARM_HOSTED", None)
for k, v in env.items():
# Debug a bizarre failure we've seen on Windows:
# TypeError: environment can only contain strings
if not isinstance(v, str):
print(k, v)
p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
out, err = p.communicate(
input=py3compat.encode('\n'.join(commands)) or None)
out, err = py3compat.decode(out), py3compat.decode(err)
# `import readline` causes 'ESC[?1034h' to be output sometimes,
# so strip that out before doing comparisons
if out:
out = re.sub(r'\x1b\[[^h]+h', '', out)
return out, err
def ipexec(fname, options=None, commands=()):
"""Utility to call 'ipython filename'.
Starts IPython with a minimal and safe configuration to make startup as fast
as possible.
Note that this starts IPython in a subprocess!
Parameters
----------
fname : str
Name of file to be executed (should have .py or .ipy extension).
options : optional, list
Extra command-line flags to be passed to IPython.
commands : optional, list
Commands to send in on stdin
Returns
-------
``(stdout, stderr)`` of ipython subprocess.
"""
if options is None: options = []
cmdargs = default_argv() + options
test_dir = os.path.dirname(__file__)
ipython_cmd = get_ipython_cmd()
# Absolute path for filename
full_fname = os.path.join(test_dir, fname)
full_cmd = ipython_cmd + cmdargs + [full_fname]
env = os.environ.copy()
# FIXME: ignore all warnings in ipexec while we have shims
# should we keep suppressing warnings here, even after removing shims?
env['PYTHONWARNINGS'] = 'ignore'
# env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
for k, v in env.items():
# Debug a bizarre failure we've seen on Windows:
# TypeError: environment can only contain strings
if not isinstance(v, str):
print(k, v)
p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
out, err = p.communicate(input=py3compat.encode('\n'.join(commands)) or None)
out, err = py3compat.decode(out), py3compat.decode(err)
# `import readline` causes 'ESC[?1034h' to be output sometimes,
# so strip that out before doing comparisons
if out:
out = re.sub(r'\x1b\[[^h]+h', '', out)
return out, err
def _find_cmd(cmd):
"""Find the full path to a command using which."""
path = sp.Popen(['/usr/bin/env', 'which', cmd],
stdout=sp.PIPE,
stderr=sp.PIPE).communicate()[0]
return py3compat.decode(path)
def _find_cmd(cmd):
"""Find the full path to a command using which."""
paths = System.Environment.GetEnvironmentVariable("PATH").Split(os.pathsep)
for path in paths:
filename = os.path.join(path, cmd)
if System.IO.File.Exists(filename):
return py3compat.decode(filename)
raise OSError("command %r not found" % cmd)
def _find_cmd(cmd):
"""Find the full path to a command using which."""
paths = System.Environment.GetEnvironmentVariable("PATH").Split(os.pathsep)
for path in paths:
filename = os.path.join(path, cmd)
if System.IO.File.Exists(filename):
return py3compat.decode(filename)
raise OSError("command %r not found" % cmd)
def osx_clipboard_get() -> str:
""" Get the clipboard's text on OS X.
"""
p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'],
stdout=subprocess.PIPE)
bytes_, stderr = p.communicate()
# Text comes in with old Mac \r line endings. Change them to \n.
bytes_ = bytes_.replace(b'\r', b'\n')
text = py3compat.decode(bytes_)
return text
def run(self):
self.started = True
while not self.stop.is_set():
chunk = os.read(self.readfd, 1024)
with self.buffer_lock:
self.buffer.write(chunk)
if self.echo:
sys.stdout.write(decode(chunk))
os.close(self.readfd)
os.close(self.writefd)
def run(self):
self.started = True
while not self.stop.is_set():
chunk = os.read(self.readfd, 1024)
with self.buffer_lock:
self.buffer.write(chunk)
if self.echo:
sys.stdout.write(decode(chunk))
os.close(self.readfd)
os.close(self.writefd)
def get_output_error_code(cmd):
"""Return (standard output, standard error, return code) of executing cmd
in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str or list
A command to be executed in the system shell.
Returns
-------
stdout : str
stderr : str
returncode: int
"""
out_err, p = process_handler(cmd, lambda p: (p.communicate(), p))
if out_err is None:
return '', '', p.returncode
out, err = out_err
return py3compat.decode(out), py3compat.decode(err), p.returncode
def get_output_error_code(cmd):
"""Return (standard output, standard error, return code) of executing cmd
in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str or list
A command to be executed in the system shell.
Returns
-------
stdout : str
stderr : str
returncode: int
"""
out_err, p = process_handler(cmd, lambda p: (p.communicate(), p))
if out_err is None:
return '', '', p.returncode
out, err = out_err
return py3compat.decode(out), py3compat.decode(err), p.returncode
def add_anchor(html):
"""Add an anchor-link to an html header tag
For use in heading cells
"""
h = ElementTree.fromstring(py3compat.cast_bytes_py2(html))
link = html2text(h).replace(' ', '-')
h.set('id', link)
a = ElementTree.Element("a", {"class": "anchor-link", "href": "#" + link})
a.text = u'¶'
h.append(a)
# Known issue of Python3.x, ElementTree.tostring() returns a byte string
# instead of a text string. See issue http://bugs.python.org/issue10942
# Workaround is to make sure the bytes are casted to a string.
return py3compat.decode(ElementTree.tostring(h), 'utf-8')
def add_anchor(html):
"""Add an anchor-link to an html header tag
For use in heading cells
"""
h = ElementTree.fromstring(py3compat.cast_bytes_py2(html, encoding='utf-8'))
link = html2text(h).replace(' ', '-')
h.set('id', link)
a = ElementTree.Element("a", {"class" : "anchor-link", "href" : "#" + link})
a.text = u'¶'
h.append(a)
# Known issue of Python3.x, ElementTree.tostring() returns a byte string
# instead of a text string. See issue http://bugs.python.org/issue10942
# Workaround is to make sure the bytes are casted to a string.
return py3compat.decode(ElementTree.tostring(h), 'utf-8')
def add_anchor(html):
"""Add an anchor-link to an html header tag
For use in heading cells
"""
try:
h = ElementTree.fromstring(py3compat.cast_bytes_py2(html, encoding="utf-8"))
except Exception:
# failed to parse, just return it unmodified
return html
link = html2text(h).replace(" ", "-")
h.set("id", link)
a = ElementTree.Element("a", {"class": "anchor-link", "href": "#" + link})
a.text = u"¶"
h.append(a)
# Known issue of Python3.x, ElementTree.tostring() returns a byte string
# instead of a text string. See issue http://bugs.python.org/issue10942
# Workaround is to make sure the bytes are casted to a string.
return py3compat.decode(ElementTree.tostring(h), "utf-8")
def getoutput(cmd):
"""Run a command and return its stdout/stderr as a string.
Parameters
----------
cmd : str or list
A command to be executed in the system shell.
Returns
-------
output : str
A string containing the combination of stdout and stderr from the
subprocess, in whatever order the subprocess originally wrote to its
file descriptors (so the order of the information in this string is the
correct order as would be seen if running the command in a terminal).
"""
out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
if out is None:
return ''
return py3compat.decode(out)
def getoutput(cmd):
"""Run a command and return its stdout/stderr as a string.
Parameters
----------
cmd : str or list
A command to be executed in the system shell.
Returns
-------
output : str
A string containing the combination of stdout and stderr from the
subprocess, in whatever order the subprocess originally wrote to its
file descriptors (so the order of the information in this string is the
correct order as would be seen if running the command in a terminal).
"""
out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
if out is None:
return ''
return py3compat.decode(out)
def _find_cmd(cmd):
"""Find the full path to a command using which."""
path = sp.Popen(['/usr/bin/env', 'which', cmd],
stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0]
return py3compat.decode(path)
def run_iptestall(options):
"""Run the entire IPython test suite by calling nose and trial.
This function constructs :class:`IPTester` instances for all IPython
modules and package and then runs each of them. This causes the modules
and packages of IPython to be tested each in their own subprocess using
nose.
Parameters
----------
All parameters are passed as attributes of the options object.
testgroups : list of str
Run only these sections of the test suite. If empty, run all the available
sections.
fast : int or None
Run the test suite in parallel, using n simultaneous processes. If None
is passed, one process is used per CPU core. Default 1 (i.e. sequential)
inc_slow : bool
Include slow tests. By default, these tests aren't run.
url : unicode
Address:port to use when running the JS tests.
xunit : bool
Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
coverage : bool or str
Measure code coverage from tests. True will store the raw coverage data,
or pass 'html' or 'xml' to get reports.
extra_args : list
Extra arguments to pass to the test subprocesses, e.g. '-v'
"""
to_run, not_run = prepare_controllers(options)
def justify(ltext, rtext, width=70, fill="-"):
ltext += " "
rtext = (" " + rtext).rjust(width - len(ltext), fill)
return ltext + rtext
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
print()
if options.fast == 1:
# This actually means sequential, i.e. with 1 job
for controller in to_run:
print("Test group:", controller.section)
sys.stdout.flush() # Show in correct order when output is piped
controller, res = do_run(controller, buffer_output=False)
if res:
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
print()
else:
# Run tests concurrently
try:
pool = multiprocessing.pool.ThreadPool(options.fast)
for (controller, res) in pool.imap_unordered(do_run, to_run):
res_string = "OK" if res == 0 else "FAILED"
print(justify("Test group: " + controller.section, res_string))
if res:
print(decode(controller.stdout))
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
except KeyboardInterrupt:
return
for controller in not_run:
print(justify("Test group: " + controller.section, "NOT RUN"))
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(to_run)
nfail = len(failed)
# summarize results
print("_" * 70)
print("Test suite completed for system with the following information:")
print(report())
took = "Took %.3fs." % t_tests
print("Status: ", end="")
if not failed:
print("OK (%d test groups)." % nrunners, took)
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
failed_sections = [c.section for c in failed]
print(
"ERROR - {} out of {} test groups failed ({}).".format(
nfail, nrunners, ", ".join(failed_sections)
),
took,
)
print()
print("You may wish to rerun these, with:")
print(" iptest", *failed_sections)
print()
if options.coverage:
from coverage import coverage, CoverageException
cov = coverage(data_file=".coverage")
cov.combine()
cov.save()
# Coverage HTML report
if options.coverage == "html":
html_dir = "ipy_htmlcov"
shutil.rmtree(html_dir, ignore_errors=True)
print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
sys.stdout.flush()
# Custom HTML reporter to clean up module names.
from coverage.html import HtmlReporter
class CustomHtmlReporter(HtmlReporter):
def find_code_units(self, morfs):
super(CustomHtmlReporter, self).find_code_units(morfs)
for cu in self.code_units:
nameparts = cu.name.split(os.sep)
if "IPython" not in nameparts:
continue
ix = nameparts.index("IPython")
cu.name = ".".join(nameparts[ix:])
# Reimplement the html_report method with our custom reporter
cov.get_data()
cov.config.from_args(
omit="*{0}tests{0}*".format(os.sep),
html_dir=html_dir,
html_title="IPython test coverage",
)
reporter = CustomHtmlReporter(cov, cov.config)
reporter.report(None)
print("done.")
# Coverage XML report
elif options.coverage == "xml":
try:
cov.xml_report(outfile="ipy_coverage.xml")
except CoverageException as e:
print(
"Generating coverage report failed. Are you running javascript tests only?"
)
import traceback
traceback.print_exc()
if failed:
# Ensure that our exit code indicates failure
sys.exit(1)
def shebang(self, line, cell):
"""Run a cell via a shell command
The `%%script` line is like the #! line of script,
specifying a program (bash, perl, ruby, etc.) with which to run.
The rest of the cell is run by that program.
Examples
--------
::
In [1]: %%script bash
...: for i in 1 2 3; do
...: echo $i
...: done
1
2
3
"""
argv = arg_split(line, posix = not sys.platform.startswith('win'))
args, cmd = self.shebang.parser.parse_known_args(argv)
try:
p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
except OSError as e:
if e.errno == errno.ENOENT:
print("Couldn't find program: %r" % cmd[0])
return
else:
raise
if not cell.endswith('\n'):
cell += '\n'
cell = cell.encode('utf8', 'replace')
if args.bg:
self.bg_processes.append(p)
self._gc_bg_processes()
if args.out:
self.shell.user_ns[args.out] = p.stdout
if args.err:
self.shell.user_ns[args.err] = p.stderr
self.job_manager.new(self._run_script, p, cell, daemon=True)
if args.proc:
self.shell.user_ns[args.proc] = p
return
try:
out, err = p.communicate(cell)
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print("Process is interrupted.")
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print("Process is terminated.")
return
p.kill()
print("Process is killed.")
except OSError:
pass
except Exception as e:
print("Error while terminating subprocess (pid=%i): %s" \
% (p.pid, e))
return
out = py3compat.decode(out)
err = py3compat.decode(err)
if args.out:
self.shell.user_ns[args.out] = out
else:
sys.stdout.write(out)
sys.stdout.flush()
if args.err:
self.shell.user_ns[args.err] = err
else:
sys.stderr.write(err)
sys.stderr.flush()
def run_iptestall(options):
"""Run the entire IPython test suite by calling nose and trial.
This function constructs :class:`IPTester` instances for all IPython
modules and package and then runs each of them. This causes the modules
and packages of IPython to be tested each in their own subprocess using
nose.
Parameters
----------
All parameters are passed as attributes of the options object.
testgroups : list of str
Run only these sections of the test suite. If empty, run all the available
sections.
fast : int or None
Run the test suite in parallel, using n simultaneous processes. If None
is passed, one process is used per CPU core. Default 1 (i.e. sequential)
inc_slow : bool
Include slow tests. By default, these tests aren't run.
url : unicode
Address:port to use when running the JS tests.
xunit : bool
Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
coverage : bool or str
Measure code coverage from tests. True will store the raw coverage data,
or pass 'html' or 'xml' to get reports.
extra_args : list
Extra arguments to pass to the test subprocesses, e.g. '-v'
"""
to_run, not_run = prepare_controllers(options)
def justify(ltext, rtext, width=70, fill='-'):
ltext += ' '
rtext = (' ' + rtext).rjust(width - len(ltext), fill)
return ltext + rtext
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
print()
if options.fast == 1:
# This actually means sequential, i.e. with 1 job
for controller in to_run:
print('Test group:', controller.section)
sys.stdout.flush() # Show in correct order when output is piped
controller, res = do_run(controller, buffer_output=False)
if res:
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
print()
else:
# Run tests concurrently
try:
pool = multiprocessing.pool.ThreadPool(options.fast)
for (controller, res) in pool.imap_unordered(do_run, to_run):
res_string = 'OK' if res == 0 else 'FAILED'
print(justify('Test group: ' + controller.section, res_string))
if res:
controller.print_extra_info()
print(decode(controller.stdout))
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
except KeyboardInterrupt:
return
for controller in not_run:
print(justify('Test group: ' + controller.section, 'NOT RUN'))
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(to_run)
nfail = len(failed)
# summarize results
print('_'*70)
print('Test suite completed for system with the following information:')
print(report())
took = "Took %.3fs." % t_tests
print('Status: ', end='')
if not failed:
print('OK (%d test groups).' % nrunners, took)
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
failed_sections = [c.section for c in failed]
print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
nrunners, ', '.join(failed_sections)), took)
print()
print('You may wish to rerun these, with:')
print(' iptest', *failed_sections)
print()
if options.coverage:
from coverage import coverage, CoverageException
cov = coverage(data_file='.coverage')
cov.combine()
cov.save()
# Coverage HTML report
if options.coverage == 'html':
html_dir = 'ipy_htmlcov'
shutil.rmtree(html_dir, ignore_errors=True)
print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
sys.stdout.flush()
# Custom HTML reporter to clean up module names.
from coverage.html import HtmlReporter
class CustomHtmlReporter(HtmlReporter):
def find_code_units(self, morfs):
super(CustomHtmlReporter, self).find_code_units(morfs)
for cu in self.code_units:
nameparts = cu.name.split(os.sep)
if 'IPython' not in nameparts:
continue
ix = nameparts.index('IPython')
cu.name = '.'.join(nameparts[ix:])
# Reimplement the html_report method with our custom reporter
cov.get_data()
cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
html_title='IPython test coverage',
)
reporter = CustomHtmlReporter(cov, cov.config)
reporter.report(None)
print('done.')
# Coverage XML report
elif options.coverage == 'xml':
try:
cov.xml_report(outfile='ipy_coverage.xml')
except CoverageException as e:
print('Generating coverage report failed. Are you running javascript tests only?')
import traceback
traceback.print_exc()
if failed:
# Ensure that our exit code indicates failure
sys.exit(1)