def ipexec(fname, options=None, commands=()):
"""Utility to call 'ipython filename'.
Starts IPython with a minimal and safe configuration to make startup as fast
as possible.
Note that this starts IPython in a subprocess!
Parameters
----------
fname : str
Name of file to be executed (should have .py or .ipy extension).
options : optional, list
Extra command-line flags to be passed to IPython.
commands : optional, list
Commands to send in on stdin
Returns
-------
(stdout, stderr) of ipython subprocess.
"""
if options is None:
options = []
# For these subprocess calls, eliminate all prompt printing so we only see
# output from script execution
prompt_opts = [
'--PromptManager.in_template=""',
'--PromptManager.in2_template=""',
'--PromptManager.out_template=""',
]
cmdargs = default_argv() + prompt_opts + options
test_dir = os.path.dirname(__file__)
ipython_cmd = get_ipython_cmd()
# Absolute path for filename
full_fname = os.path.join(test_dir, fname)
full_cmd = ipython_cmd + cmdargs + [full_fname]
env = os.environ.copy()
# FIXME: ignore all warnings in ipexec while we have shims
# should we keep suppressing warnings here, even after removing shims?
env["PYTHONWARNINGS"] = "ignore"
# env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
for k, v in env.items():
# Debug a bizarre failure we've seen on Windows:
# TypeError: environment can only contain strings
if not isinstance(v, str):
print(k, v)
p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
out, err = p.communicate(input=py3compat.str_to_bytes("\n".join(commands)) or None)
out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
# `import readline` causes 'ESC[?1034h' to be output sometimes,
# so strip that out before doing comparisons
if out:
out = re.sub(r"\x1b\[[^h]+h", "", out)
return out, err
def ipexec(fname, options=None, commands=()):
"""Utility to call 'ipython filename'.
Starts IPython with a minimal and safe configuration to make startup as fast
as possible.
Note that this starts IPython in a subprocess!
Parameters
----------
fname : str
Name of file to be executed (should have .py or .ipy extension).
options : optional, list
Extra command-line flags to be passed to IPython.
commands : optional, list
Commands to send in on stdin
Returns
-------
(stdout, stderr) of ipython subprocess.
"""
if options is None: options = []
# For these subprocess calls, eliminate all prompt printing so we only see
# output from script execution
prompt_opts = [
'--PromptManager.in_template=""', '--PromptManager.in2_template=""',
'--PromptManager.out_template=""'
]
cmdargs = default_argv() + prompt_opts + options
test_dir = os.path.dirname(__file__)
ipython_cmd = get_ipython_cmd()
# Absolute path for filename
full_fname = os.path.join(test_dir, fname)
full_cmd = ipython_cmd + cmdargs + [full_fname]
env = os.environ.copy()
# FIXME: ignore all warnings in ipexec while we have shims
# should we keep suppressing warnings here, even after removing shims?
env['PYTHONWARNINGS'] = 'ignore'
# env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
for k, v in env.items():
# Debug a bizarre failure we've seen on Windows:
# TypeError: environment can only contain strings
if not isinstance(v, str):
print(k, v)
p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
out, err = p.communicate(
input=py3compat.str_to_bytes('\n'.join(commands)) or None)
out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
# `import readline` causes 'ESC[?1034h' to be output sometimes,
# so strip that out before doing comparisons
if out:
out = re.sub(r'\x1b\[[^h]+h', '', out)
return out, err
def doit(self, doit_args, cell):
with NamedTemporaryFile(delete=False, suffix='.py') as tmp_file:
tmp_name = tmp_file.name
tmp_file.write(cell)
cur_dir = os.getcwd()
doit_args = doit_args.split()
if doit_args:
doit_command = [doit_args.pop(0)]
else:
doit_command = []
cmd = ['doit']
cmd += doit_command
cmd += ['-d', cur_dir, '-f', tmp_name]
cmd += doit_args
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
try:
out, err = p.communicate(cell)
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print("Process is interrupted.")
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print("Process is terminated.")
return
p.kill()
print("Process is killed.")
except OSError:
pass
except Exception as e:
print("Error while terminating subprocess (pid=%i): %s" \
% (p.pid, e))
return
out = py3compat.bytes_to_str(out)
err = py3compat.bytes_to_str(err)
sys.stdout.write(out)
sys.stdout.flush()
sys.stderr.write(err)
sys.stderr.flush()
os.remove(tmp_name)
def doit(self, doit_args, cell):
with NamedTemporaryFile(delete=False, suffix='.py') as tmp_file:
tmp_name = tmp_file.name
tmp_file.write(cell)
cur_dir = os.getcwd()
doit_args = doit_args.split()
if doit_args:
doit_command = [doit_args.pop(0)]
else:
doit_command = []
cmd = ['doit']
cmd += doit_command
cmd += [ '-d', cur_dir, '-f', tmp_name]
cmd += doit_args
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
try:
out, err = p.communicate(cell)
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print("Process is interrupted.")
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print("Process is terminated.")
return
p.kill()
print("Process is killed.")
except OSError:
pass
except Exception as e:
print("Error while terminating subprocess (pid=%i): %s" \
% (p.pid, e))
return
out = py3compat.bytes_to_str(out)
err = py3compat.bytes_to_str(err)
sys.stdout.write(out)
sys.stdout.flush()
sys.stderr.write(err)
sys.stderr.flush()
os.remove(tmp_name)
def ipexec(fname, options=None):
"""Utility to call 'ipython filename'.
Starts IPython with a minimal and safe configuration to make startup as fast
as possible.
Note that this starts IPython in a subprocess!
Parameters
----------
fname : str
Name of file to be executed (should have .py or .ipy extension).
options : optional, list
Extra command-line flags to be passed to IPython.
Returns
-------
(stdout, stderr) of ipython subprocess.
"""
if options is None: options = []
# For these subprocess calls, eliminate all prompt printing so we only see
# output from script execution
prompt_opts = [ '--PromptManager.in_template=""',
'--PromptManager.in2_template=""',
'--PromptManager.out_template=""'
]
cmdargs = default_argv() + prompt_opts + options
_ip = get_ipython()
test_dir = os.path.dirname(__file__)
# FIXME: remove workaround for 2.6 support
if sys.version_info[:2] > (2,6):
ipython_cmd = [sys.executable, "-m", "IPython"]
else:
ipython_cmd = ["ipython"]
# Absolute path for filename
full_fname = os.path.join(test_dir, fname)
full_cmd = ipython_cmd + cmdargs + [full_fname]
p = Popen(full_cmd, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
# `import readline` causes 'ESC[?1034h' to be output sometimes,
# so strip that out before doing comparisons
if out:
out = re.sub(r'\x1b\[[^h]+h', '', out)
return out, err
def ipexec(fname, options=None, commands=()):
"""Utility to call 'ipython filename'.
Starts IPython with a minimal and safe configuration to make startup as fast
as possible.
Note that this starts IPython in a subprocess!
Parameters
----------
fname : str
Name of file to be executed (should have .py or .ipy extension).
options : optional, list
Extra command-line flags to be passed to IPython.
commands : optional, list
Commands to send in on stdin
Returns
-------
(stdout, stderr) of ipython subprocess.
"""
if options is None: options = []
# For these subprocess calls, eliminate all prompt printing so we only see
# output from script execution
prompt_opts = [ '--PromptManager.in_template=""',
'--PromptManager.in2_template=""',
'--PromptManager.out_template=""'
]
cmdargs = default_argv() + prompt_opts + options
test_dir = os.path.dirname(__file__)
ipython_cmd = get_ipython_cmd()
# Absolute path for filename
full_fname = os.path.join(test_dir, fname)
full_cmd = ipython_cmd + cmdargs + [full_fname]
env = os.environ.copy()
env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
out, err = p.communicate(input=py3compat.str_to_bytes('\n'.join(commands)) or None)
out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
# `import readline` causes 'ESC[?1034h' to be output sometimes,
# so strip that out before doing comparisons
if out:
out = re.sub(r'\x1b\[[^h]+h', '', out)
return out, err
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, ip=LOCALHOST, key=b""):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the REQ (raw input) channel.
hb_port : int, optional
The port to use for the hearbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for HMAC authentication.
"""
# default to temporary connector file
if not fname:
fname = tempfile.mktemp(".json")
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + int(stdin_port <= 0) + int(hb_port <= 0)
for i in xrange(ports_needed):
sock = socket.socket()
sock.bind(("", 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict(shell_port=shell_port, iopub_port=iopub_port, stdin_port=stdin_port, hb_port=hb_port)
cfg["ip"] = ip
cfg["key"] = bytes_to_str(key)
with open(fname, "w") as f:
f.write(json.dumps(cfg, indent=2))
return fname, cfg
def _find_cmd(cmd):
"""Find the full path to a command using which."""
path = sp.Popen(['/usr/bin/env', 'which', cmd],
stdout=sp.PIPE,
stderr=sp.PIPE).communicate()[0]
return py3compat.bytes_to_str(path)
def __getitem__(self,attr):
try:
return pickle.loads(
bytes_to_str(base64.b64decode(str_to_bytes(
PermanentStorage._data[attr]))))
except TypeError as e:
return "TypeError: "+str(e)
def ipexec(fname, options=None):
"""Utility to call 'ipython filename'.
Starts IPython with a minimal and safe configuration to make startup as fast
as possible.
Note that this starts IPython in a subprocess!
Parameters
----------
fname : str
Name of file to be executed (should have .py or .ipy extension).
options : optional, list
Extra command-line flags to be passed to IPython.
Returns
-------
(stdout, stderr) of ipython subprocess.
"""
if options is None: options = []
# For these subprocess calls, eliminate all prompt printing so we only see
# output from script execution
prompt_opts = [
'--PromptManager.in_template=""', '--PromptManager.in2_template=""',
'--PromptManager.out_template=""'
]
cmdargs = default_argv() + prompt_opts + options
test_dir = os.path.dirname(__file__)
ipython_cmd = get_ipython_cmd()
# Absolute path for filename
full_fname = os.path.join(test_dir, fname)
full_cmd = ipython_cmd + cmdargs + [full_fname]
env = os.environ.copy()
env.pop('PYTHONWARNINGS',
None) # Avoid extraneous warnings appearing on stderr
p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, env=env)
out, err = p.communicate()
out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
# `import readline` causes 'ESC[?1034h' to be output sometimes,
# so strip that out before doing comparisons
if out:
out = re.sub(r'\x1b\[[^h]+h', '', out)
return out, err
def shebang(self, line, cell):
"""Run a cell via a shell command
The `%%script` line is like the #! line of script,
specifying a program (bash, perl, ruby, etc.) with which to run.
The rest of the cell is run by that program.
Examples
--------
::
In [1]: %%script bash
...: for i in 1 2 3; do
...: echo $i
...: done
1
2
3
"""
argv = arg_split(line, posix = not sys.platform.startswith('win'))
args, cmd = self.shebang.parser.parse_known_args(argv)
p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
cell = cell.encode('utf8', 'replace')
if args.bg:
if args.out:
self.shell.user_ns[args.out] = p.stdout
if args.err:
self.shell.user_ns[args.err] = p.stderr
self.job_manager.new(self._run_script, p, cell)
return
out, err = p.communicate(cell)
out = py3compat.bytes_to_str(out)
err = py3compat.bytes_to_str(err)
if args.out:
self.shell.user_ns[args.out] = out
else:
sys.stdout.write(out)
sys.stdout.flush()
if args.err:
self.shell.user_ns[args.err] = err
else:
sys.stderr.write(err)
sys.stderr.flush()
def _find_cmd(cmd):
"""Find the full path to a command using which."""
paths = System.Environment.GetEnvironmentVariable("PATH").Split(os.pathsep)
for path in paths:
filename = os.path.join(path, cmd)
if System.IO.File.Exists(filename):
return py3compat.bytes_to_str(filename)
raise OSError("command %r not found" % cmd)
def create_new_code_cell(code):
"""Javascript to create and populate a new code cell in the notebook"""
encoded_code = bytes_to_str(b64encode(str_to_bytes(code)))
display(
Javascript("""
var code_cell = IPython.notebook.insert_cell_below('code');
code_cell.set_text(atob("{0}"));
""".format(encoded_code)))
def __getitem__(self, attr):
try:
return pickle.loads(
bytes_to_str(
base64.b64decode(
str_to_bytes(PermanentStorage._data[attr]))))
except TypeError as e:
return "TypeError: " + str(e)
def _find_cmd(cmd):
"""Find the full path to a command using which."""
paths = System.Environment.GetEnvironmentVariable("PATH").Split(os.pathsep)
for path in paths:
filename = os.path.join(path, cmd)
if System.IO.File.Exists(filename):
return py3compat.bytes_to_str(filename)
raise OSError("command %r not found" % cmd)
def __setitem__(self,attr,value):
"""Set property in the metadata"""
pick=bytes_to_str(base64.b64encode(str_to_bytes(pickle.dumps(value))))
name=attr
PermanentStorage._data[attr]=pick
display(Javascript('%s["%s"]="%s";console.log("Setting %s");' % (self.__storePath,
name,
pick,
name)));
def __setitem__(self, attr, value):
"""Set property in the metadata"""
pick = bytes_to_str(
base64.b64encode(str_to_bytes(pickle.dumps(value))))
name = attr
PermanentStorage._data[attr] = pick
display(
Javascript('%s["%s"]="%s";console.log("Setting %s");' %
(self.__storePath, name, pick, name)))
def create_code_cell(code: str = '', where: CellLocation = 'below'):
"""
Create a code cell in the IPython Notebook
:param code: code to fill the new code cell with.
:param where: where to add the new code cell
"""
encoded_code = bytes_to_str(base64.b64encode(str_to_bytes(code)))
display(Javascript('''
var code = IPython.notebook.insert_cell_{0}('code');
code.set_text(atob("{1}"));
'''.format(where, encoded_code)))
def getoutputerror(cmd):
"""Return (standard output, standard error) of executing cmd in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str
A command to be executed in the system shell.
Returns
-------
stdout : str
stderr : str
"""
out_err = process_handler(cmd, lambda p: p.communicate())
if out_err is None:
return '', ''
out, err = out_err
return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
def getoutputerror(cmd):
"""Return (standard output, standard error) of executing cmd in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str
A command to be executed in the system shell.
Returns
-------
stdout : str
stderr : str
"""
out_err = process_handler(cmd, lambda p: p.communicate())
if out_err is None:
return '', ''
out, err = out_err
return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
def run(self):
self.started = True
while not self.stop.is_set():
chunk = os.read(self.readfd, 1024)
with self.buffer_lock:
self.buffer.write(chunk)
if self.echo:
sys.stdout.write(bytes_to_str(chunk))
os.close(self.readfd)
os.close(self.writefd)
def _read(self, silent=True):
"""Read AMPL output until the next prompt"""
stdout = self.process.stdout
out = ""
while True:
header = ""
while True:
c = stdout.read(1)
if c == " ":
break
header += c
length = int(py3compat.bytes_to_str(header))
data = py3compat.bytes_to_str(stdout.read(length))
command, block = data.split("\n", 1)
# TODO: handle errors
if command.startswith("prompt"):
break
if not silent:
sys.stdout.write(block)
sys.stdout.flush()
out += block
return out
def latex_to_html(s, alt='image'):
"""Render LaTeX to HTML with embedded PNG data using data URIs.
Parameters
----------
s : str
The raw string containing valid inline LateX.
alt : str
The alt text to use for the HTML.
"""
base64_data = bytes_to_str(latex_to_png(s, encode=True), 'ascii')
if base64_data:
return _data_uri_template_png % (base64_data, alt)
def latex_to_html(s, alt='image'):
"""Render LaTeX to HTML with embedded PNG data using data URIs.
Parameters
----------
s : str
The raw string containing valid inline LateX.
alt : str
The alt text to use for the HTML.
"""
base64_data = bytes_to_str(latex_to_png(s, encode=True), 'ascii')
if base64_data:
return _data_uri_template_png % (base64_data, alt)
def wait(self, *pargs, **kwargs):
"""Wait for the JSController to finish"""
ret = super(JSController, self).wait(*pargs, **kwargs)
# If this is a SlimerJS controller, check the captured stdout for
# errors. Otherwise, just return the return code.
if self.engine == 'slimerjs':
stdout = bytes_to_str(self.stdout)
if ret != 0:
# This could still happen e.g. if it's stopped by SIGINT
return ret
return bool(self.slimer_failure.search(strip_ansi(stdout)))
else:
return ret
def run(self):
self.started = True
while not self.stop.is_set():
chunk = os.read(self.readfd, 1024)
with self.buffer_lock:
self.buffer.write(chunk)
if self.echo:
sys.stdout.write(bytes_to_str(chunk))
os.close(self.readfd)
os.close(self.writefd)
def _read(self, silent=True):
"""Read AMPL output until the next prompt"""
stdout = self.process.stdout
out = ""
while True:
header = b""
while True:
c = stdout.read(1)
if c == b" ":
break
header += c
length = int(py3compat.bytes_to_str(header))
data = py3compat.bytes_to_str(stdout.read(length))
command, block = data.split("\n", 1)
# TODO: handle errors
if command.startswith("prompt"):
break
if not silent:
sys.stdout.write(block);
sys.stdout.flush()
out += block
return out
def wait(self, *pargs, **kwargs):
"""Wait for the JSController to finish"""
ret = super(JSController, self).wait(*pargs, **kwargs)
# If this is a SlimerJS controller, check the captured stdout for
# errors. Otherwise, just return the return code.
if self.engine == 'slimerjs':
stdout = bytes_to_str(self.stdout)
if ret != 0:
# This could still happen e.g. if it's stopped by SIGINT
return ret
return bool(self.slimer_failure.search(strip_ansi(stdout)))
else:
return ret
def get_output_error_code(cmd):
"""Return (standard output, standard error, return code) of executing cmd
in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str or list
A command to be executed in the system shell.
Returns
-------
stdout : str
stderr : str
returncode: int
"""
out_err, p = process_handler(cmd, lambda p: (p.communicate(), p))
if out_err is None:
return '', '', p.returncode
out, err = out_err
return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err), p.returncode
def get_output_error_code(cmd):
"""Return (standard output, standard error, return code) of executing cmd
in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str or list
A command to be executed in the system shell.
Returns
-------
stdout : str
stderr : str
returncode: int
"""
out_err, p = process_handler(cmd, lambda p: (p.communicate(), p))
if out_err is None:
return '', '', p.returncode
out, err = out_err
return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err), p.returncode
def display_cell(text):
"""Remove cells that start with "# Temp" and add a new one
Arguments:
* `text` -- new cell content
"""
encoded_code = bytes_to_str(base64.b64encode(str_to_bytes(text)))
display(
Javascript("""
$('span:contains("# Temp")').closest('.cell').remove();
var code = IPython.notebook.insert_cell_{0}('code');
code.set_text(atob("{1}"));
""".format('below', encoded_code)))
def new_cell(texto, tipo_celda):
if tipo_celda == "markdown":
display(
Javascript("""
var mark = IPython.notebook.insert_cell_above('markdown')
mark.set_text("{0}")
mark.execute()
""".format(texto.encode('utf-8'))))
if tipo_celda == "code":
texto = bytes_to_str(base64.b64encode(str_to_bytes(texto)))
display(
Javascript("""
var code = IPython.notebook.insert_cell_above('code')
code.set_text(atob("{0}"))
""".format(texto)))
def getoutput(cmd):
"""Return standard output of executing cmd in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str
A command to be executed in the system shell.
Returns
-------
stdout : str
"""
out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
if out is None:
return ''
return py3compat.bytes_to_str(out)
def create_code_cell(code='', where='below'):
"""Create a code cell in the IPython Notebook.
Found at https://github.com/ipython/ipython/issues/4983
Parameters
code: unicode
Code to fill the new code cell with.
where: unicode
Where to add the new code cell.
Possible values include:
at_bottom
above
below"""
encoded_code = bytes_to_str(base64.b64encode(str_to_bytes(code)))
display(Javascript("""
var code = IPython.notebook.insert_cell_{0}('code');
code.set_text(atob("{1}"));
""".format(where, encoded_code)))
def getoutput(cmd):
"""Return standard output of executing cmd in a shell.
Accepts the same arguments as os.system().
Parameters
----------
cmd : str
A command to be executed in the system shell.
Returns
-------
stdout : str
"""
out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
if out is None:
return ''
return py3compat.bytes_to_str(out)
def create_code_cell(code='', where='below'):
"""Create a code cell in the IPython Notebook.
Found at https://github.com/ipython/ipython/issues/4983
Parameters
code: unicode
Code to fill the new code cell with.
where: unicode
Where to add the new code cell.
Possible values include:
at_bottom
above
below"""
encoded_code = bytes_to_str(base64.b64encode(str_to_bytes(code)))
display(
Javascript("""
var code = IPython.notebook.insert_cell_{0}('code');
code.set_text(atob("{1}"));
""".format(where, encoded_code)))
def nbconvert(self, parameters, ignore_return_code=False):
"""
Run nbconvert a, IPython shell command, listening for both Errors and non-zero
return codes.
Parameters
----------
parameters : str, list(str)
List of parameters to pass to IPython.
ignore_return_code : optional bool (default False)
Throw an OSError if the return code
"""
if isinstance(parameters, string_types):
parameters = shlex.split(parameters)
cmd = [sys.executable, '-m', 'jupyter_nbconvert'] + parameters
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
if not (p.returncode == 0 or ignore_return_code):
raise OSError(bytes_to_str(stderr))
return stdout.decode('utf8', 'replace'), stderr.decode('utf8', 'replace')
def getoutput(cmd):
"""Run a command and return its stdout/stderr as a string.
Parameters
----------
cmd : str
A command to be executed in the system shell.
Returns
-------
output : str
A string containing the combination of stdout and stderr from the
subprocess, in whatever order the subprocess originally wrote to its
file descriptors (so the order of the information in this string is the
correct order as would be seen if running the command in a terminal).
"""
out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
if out is None:
return ''
return py3compat.bytes_to_str(out)
def getoutput(cmd):
"""Run a command and return its stdout/stderr as a string.
Parameters
----------
cmd : str or list
A command to be executed in the system shell.
Returns
-------
output : str
A string containing the combination of stdout and stderr from the
subprocess, in whatever order the subprocess originally wrote to its
file descriptors (so the order of the information in this string is the
correct order as would be seen if running the command in a terminal).
"""
out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
if out is None:
return ''
return py3compat.bytes_to_str(out)
def nbconvert(self, parameters, ignore_return_code=False):
"""
Run nbconvert a, IPython shell command, listening for both Errors and non-zero
return codes.
Parameters
----------
parameters : str, list(str)
List of parameters to pass to IPython.
ignore_return_code : optional bool (default False)
Throw an OSError if the return code
"""
if isinstance(parameters, string_types):
parameters = shlex.split(parameters)
cmd = [sys.executable, '-m', 'jupyter_nbconvert'] + parameters
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
if not (p.returncode == 0 or ignore_return_code):
raise OSError(bytes_to_str(stderr))
return stdout.decode('utf8',
'replace'), stderr.decode('utf8', 'replace')
def run_iptestall(options):
"""Run the entire IPython test suite by calling nose and trial.
This function constructs :class:`IPTester` instances for all IPython
modules and package and then runs each of them. This causes the modules
and packages of IPython to be tested each in their own subprocess using
nose.
Parameters
----------
All parameters are passed as attributes of the options object.
testgroups : list of str
Run only these sections of the test suite. If empty, run all the available
sections.
fast : int or None
Run the test suite in parallel, using n simultaneous processes. If None
is passed, one process is used per CPU core. Default 1 (i.e. sequential)
inc_slow : bool
Include slow tests, like IPython.parallel. By default, these tests aren't
run.
xunit : bool
Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
coverage : bool or str
Measure code coverage from tests. True will store the raw coverage data,
or pass 'html' or 'xml' to get reports.
extra_args : list
Extra arguments to pass to the test subprocesses, e.g. '-v'
"""
if options.fast != 1:
# If running in parallel, capture output so it doesn't get interleaved
TestController.buffer_output = True
to_run, not_run = prepare_controllers(options)
def justify(ltext, rtext, width=70, fill='-'):
ltext += ' '
rtext = (' ' + rtext).rjust(width - len(ltext), fill)
return ltext + rtext
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
print()
if options.fast == 1:
# This actually means sequential, i.e. with 1 job
for controller in to_run:
print('IPython test group:', controller.section)
sys.stdout.flush() # Show in correct order when output is piped
controller, res = do_run(controller)
if res:
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
print()
else:
# Run tests concurrently
try:
pool = multiprocessing.pool.ThreadPool(options.fast)
for (controller, res) in pool.imap_unordered(do_run, to_run):
res_string = 'OK' if res == 0 else 'FAILED'
print(justify('IPython test group: ' + controller.section, res_string))
if res:
print(bytes_to_str(controller.stdout))
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
except KeyboardInterrupt:
return
for controller in not_run:
print(justify('IPython test group: ' + controller.section, 'NOT RUN'))
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(to_run)
nfail = len(failed)
# summarize results
print('_'*70)
print('Test suite completed for system with the following information:')
print(report())
took = "Took %.3fs." % t_tests
print('Status: ', end='')
if not failed:
print('OK (%d test groups).' % nrunners, took)
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
failed_sections = [c.section for c in failed]
print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
nrunners, ', '.join(failed_sections)), took)
print()
print('You may wish to rerun these, with:')
print(' iptest', *failed_sections)
print()
if options.coverage:
from coverage import coverage
cov = coverage(data_file='.coverage')
cov.combine()
cov.save()
# Coverage HTML report
if options.coverage == 'html':
html_dir = 'ipy_htmlcov'
shutil.rmtree(html_dir, ignore_errors=True)
print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
sys.stdout.flush()
# Custom HTML reporter to clean up module names.
from coverage.html import HtmlReporter
class CustomHtmlReporter(HtmlReporter):
def find_code_units(self, morfs):
super(CustomHtmlReporter, self).find_code_units(morfs)
for cu in self.code_units:
nameparts = cu.name.split(os.sep)
if 'IPython' not in nameparts:
continue
ix = nameparts.index('IPython')
cu.name = '.'.join(nameparts[ix:])
# Reimplement the html_report method with our custom reporter
cov._harvest_data()
cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
html_title='IPython test coverage',
)
reporter = CustomHtmlReporter(cov, cov.config)
reporter.report(None)
print('done.')
# Coverage XML report
elif options.coverage == 'xml':
cov.xml_report(outfile='ipy_coverage.xml')
if failed:
# Ensure that our exit code indicates failure
sys.exit(1)
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
control_port=0, ip=LOCALHOST, key=b'', transport='tcp',
signature_scheme='hmac-sha256',
):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER (shell) channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the ROUTER (raw input) channel.
control_port : int, optional
The port to use for the ROUTER (control) channel.
hb_port : int, optional
The port to use for the heartbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for message authentication.
signature_scheme : str, optional
The scheme used for message authentication.
This has the form 'digest-hash', where 'digest'
is the scheme used for digests, and 'hash' is the name of the hash function
used by the digest scheme.
Currently, 'hmac' is the only supported digest scheme,
and 'sha256' is the default hash function.
"""
# default to temporary connector file
if not fname:
fname = tempfile.mktemp('.json')
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + \
int(iopub_port <= 0) + \
int(stdin_port <= 0) + \
int(control_port <= 0) + \
int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
sock.bind(('', 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if control_port <= 0:
control_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict( shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
control_port=control_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
cfg['signature_scheme'] = signature_scheme
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
return fname, cfg
def run_cell(self, line, cell):
"""Run a cell via a shell command
The `%%anybody` invokes the anybody console application on the rest of
the cell.
Parameters
----------
--dir <Path>
--out <output var>
--bg <>
--proc <baground process variable >
--anybodycon <path to anybodycon>
Examples
--------
::
In [1]: %%anybody
...: load "mymodel.any"
...: operation Main.MyStudy.Kinematics
...: run
"""
argv = arg_split(line, posix = not sys.platform.startswith('win'))
args, dummy = self.run_cell.parser.parse_known_args(argv)
if args.anybodycon:
if os.path.exists(args.anybodycon):
abcpath = args.anybodycon
elif self.shell.user_ns.has_key(args.anybodycon):
abcpath = self.shell.user_ns[args.anybodycon]
elif sys.platform == 'win32':
import _winreg
try:
abpath = _winreg.QueryValue(_winreg.HKEY_CLASSES_ROOT,
'AnyBody.AnyScript\shell\open\command')
abpath = abpath.rsplit(' ',1)[0].strip('"')
abcpath = os.path.join(os.path.dirname(abpath),'AnyBodyCon.exe')
except:
raise Exception('Could not find AnyBody Modeling System in windows registry')
else:
raise Exception('Cannot find the specified anybodycon')
if not os.path.exists(abcpath):
raise Exception('Cannot find the specified anybodycon: %s'%abcpath)
if args.dir and os.path.isdir(args.dir):
folder = args.dir
elif self.shell.user_ns.has_key(args.dir):
folder = self.shell.user_ns[args.dir]
else:
folder = os.getcwd()
cell = cell.encode('utf8', 'replace')
macro = cell if cell.endswith('\n') else cell+'\n'
macrofile = NamedTemporaryFile(mode='w+b',
prefix ='macro_',
suffix='.anymcr',
dir= folder,
delete = False)
macrofile.write(macro)
macrofile.flush()
cmd = [abcpath ,'/d',folder, '--macro=', macrofile.name, '/ni', "1>&2"]
try:
p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell= True)
except OSError as e:
if e.errno == errno.ENOENT:
print "Couldn't find program: %r" % cmd[0]
return
else:
raise
if args.bg:
self.bg_processes.append(p)
self._gc_bg_processes()
if args.out:
self.shell.user_ns[args.out] = p.stderr
self.job_manager.new(self._run_script, p, macrofile, daemon=True)
if args.proc:
self.shell.user_ns[args.proc] = p
return
random_tag = ''.join( random.sample(string.ascii_uppercase,6) )
def htmlbox(text):
raw_html = """<div id="anyscriptbox_{0}" style="height: 120px ; width : auto; border:1px dotted black;padding:0.5em;overflow:auto;background-color:#E0E0E0 ; font:3px Geogia"><font size="2px" face="courier"> {1} </font></div> <script> var myDiv = document.getElementById("anyscriptbox_{0}");
myDiv.scrollTop = myDiv.scrollHeight;</script> """.format(random_tag, text )
return HTML(raw_html)
try:
raw_out = []
for line in iter(p.stderr.readline,b''):
line = py3compat.bytes_to_str( line )
raw_out.append(line)
if not args.pager:
clear_output()
display( htmlbox("<br/>".join( raw_out ) ) )
#sys.stdout.flush()
if args.pager:
page("".join(raw_out))
p.communicate();
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print "Process is interrupted."
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print "Process is terminated."
return
p.kill()
print "Process is killed."
except OSError:
pass
except Exception as e:
print "Error while terminating subprocess (pid=%i): %s" \
% (p.pid, e)
return
if args.out:
self.shell.user_ns[args.out] = "\n".join(raw_out)
if args.dump:
output = _parse_anybodycon_output( "\n".join(raw_out) )
if len(output.keys()):
print 'Dumped variables:'
for k,v in output.iteritems():
varname = k.replace('.','_')
self.shell.user_ns[varname] = v
print '- ' + varname
try:
macrofile.close()
os.remove(macrofile.name)
except:
print 'Error removing macro file'
def write_connection_file(fname=None,
shell_port=0,
iopub_port=0,
stdin_port=0,
hb_port=0,
ip=LOCALHOST,
key=b'',
transport='tcp'):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the REQ (raw input) channel.
hb_port : int, optional
The port to use for the hearbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for HMAC authentication.
"""
# default to temporary connector file
if not fname:
fname = tempfile.mktemp('.json')
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
int(stdin_port <= 0) + int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
sock.bind(('', 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict(
shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
return fname, cfg
def run_cell(self, line, cell):
"""Run a cell via a shell command
The `%%anybody` invokes the anybody console application on the rest of
the cell.
Parameters
----------
--dir <Path>
--out <output var>
--bg <>
--proc <baground process variable >
--anybodycon <path to anybodycon>
Examples
--------
::
In [1]: %%anybody
...: load "mymodel.any"
...: operation Main.MyStudy.Kinematics
...: run
"""
argv = arg_split(line, posix=not sys.platform.startswith('win'))
args, dummy = self.run_cell.parser.parse_known_args(argv)
if args.anybodycon:
if os.path.exists(args.anybodycon):
abcpath = args.anybodycon
elif self.shell.user_ns.has_key(args.anybodycon):
abcpath = self.shell.user_ns[args.anybodycon]
elif sys.platform == 'win32':
import _winreg
try:
abpath = _winreg.QueryValue(
_winreg.HKEY_CLASSES_ROOT,
'AnyBody.AnyScript\shell\open\command')
abpath = abpath.rsplit(' ', 1)[0].strip('"')
abcpath = os.path.join(os.path.dirname(abpath),
'AnyBodyCon.exe')
except:
raise Exception(
'Could not find AnyBody Modeling System in windows registry'
)
else:
raise Exception('Cannot find the specified anybodycon')
if not os.path.exists(abcpath):
raise Exception('Cannot find the specified anybodycon: %s' %
abcpath)
if args.dir and os.path.isdir(args.dir):
folder = args.dir
elif self.shell.user_ns.has_key(args.dir):
folder = self.shell.user_ns[args.dir]
else:
folder = os.getcwd()
cell = cell.encode('utf8', 'replace')
macro = cell if cell.endswith('\n') else cell + '\n'
macrofile = NamedTemporaryFile(mode='w+b',
prefix='macro_',
suffix='.anymcr',
dir=folder,
delete=False)
macrofile.write(macro)
macrofile.flush()
cmd = [
abcpath, '/d', folder, '--macro=', macrofile.name, '/ni', "1>&2"
]
try:
p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=True)
except OSError as e:
if e.errno == errno.ENOENT:
print "Couldn't find program: %r" % cmd[0]
return
else:
raise
if args.bg:
self.bg_processes.append(p)
self._gc_bg_processes()
if args.out:
self.shell.user_ns[args.out] = p.stderr
self.job_manager.new(self._run_script, p, macrofile, daemon=True)
if args.proc:
self.shell.user_ns[args.proc] = p
return
random_tag = ''.join(random.sample(string.ascii_uppercase, 6))
def htmlbox(text):
raw_html = """<div id="anyscriptbox_{0}" style="height: 120px ; width : auto; border:1px dotted black;padding:0.5em;overflow:auto;background-color:#E0E0E0 ; font:3px Geogia"><font size="2px" face="courier"> {1} </font></div> <script> var myDiv = document.getElementById("anyscriptbox_{0}");
myDiv.scrollTop = myDiv.scrollHeight;</script> """.format(
random_tag, text)
return HTML(raw_html)
try:
raw_out = []
for line in iter(p.stderr.readline, b''):
line = py3compat.bytes_to_str(line)
raw_out.append(line)
if not args.pager:
clear_output()
display(htmlbox("<br/>".join(raw_out)))
#sys.stdout.flush()
if args.pager:
page("".join(raw_out))
p.communicate()
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print "Process is interrupted."
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print "Process is terminated."
return
p.kill()
print "Process is killed."
except OSError:
pass
except Exception as e:
print "Error while terminating subprocess (pid=%i): %s" \
% (p.pid, e)
return
if args.out:
self.shell.user_ns[args.out] = "\n".join(raw_out)
if args.dump:
output = _parse_anybodycon_output("\n".join(raw_out))
if len(output.keys()):
print 'Dumped variables:'
for k, v in output.iteritems():
varname = k.replace('.', '_')
self.shell.user_ns[varname] = v
print '- ' + varname
try:
macrofile.close()
os.remove(macrofile.name)
except:
print 'Error removing macro file'
def run_iptestall(options):
"""Run the entire IPython test suite by calling nose and trial.
This function constructs :class:`IPTester` instances for all IPython
modules and package and then runs each of them. This causes the modules
and packages of IPython to be tested each in their own subprocess using
nose.
Parameters
----------
All parameters are passed as attributes of the options object.
testgroups : list of str
Run only these sections of the test suite. If empty, run all the available
sections.
fast : int or None
Run the test suite in parallel, using n simultaneous processes. If None
is passed, one process is used per CPU core. Default 1 (i.e. sequential)
inc_slow : bool
Include slow tests, like IPython.parallel. By default, these tests aren't
run.
slimerjs : bool
Use slimerjs if it's installed instead of phantomjs for casperjs tests.
xunit : bool
Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
coverage : bool or str
Measure code coverage from tests. True will store the raw coverage data,
or pass 'html' or 'xml' to get reports.
extra_args : list
Extra arguments to pass to the test subprocesses, e.g. '-v'
"""
to_run, not_run = prepare_controllers(options)
def justify(ltext, rtext, width=70, fill='-'):
ltext += ' '
rtext = (' ' + rtext).rjust(width - len(ltext), fill)
return ltext + rtext
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
print()
if options.fast == 1:
# This actually means sequential, i.e. with 1 job
for controller in to_run:
print('Test group:', controller.section)
sys.stdout.flush() # Show in correct order when output is piped
controller, res = do_run(controller, buffer_output=False)
if res:
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
print()
else:
# Run tests concurrently
try:
pool = multiprocessing.pool.ThreadPool(options.fast)
for (controller, res) in pool.imap_unordered(do_run, to_run):
res_string = 'OK' if res == 0 else 'FAILED'
print(justify('Test group: ' + controller.section, res_string))
if res:
controller.print_extra_info()
print(bytes_to_str(controller.stdout))
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
except KeyboardInterrupt:
return
for controller in not_run:
print(justify('Test group: ' + controller.section, 'NOT RUN'))
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(to_run)
nfail = len(failed)
# summarize results
print('_' * 70)
print('Test suite completed for system with the following information:')
print(report())
took = "Took %.3fs." % t_tests
print('Status: ', end='')
if not failed:
print('OK (%d test groups).' % nrunners, took)
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
failed_sections = [c.section for c in failed]
print(
'ERROR - {} out of {} test groups failed ({}).'.format(
nfail, nrunners, ', '.join(failed_sections)), took)
print()
print('You may wish to rerun these, with:')
print(' iptest', *failed_sections)
print()
if options.coverage:
from coverage import coverage
cov = coverage(data_file='.coverage')
cov.combine()
cov.save()
# Coverage HTML report
if options.coverage == 'html':
html_dir = 'ipy_htmlcov'
shutil.rmtree(html_dir, ignore_errors=True)
print("Writing HTML coverage report to %s/ ... " % html_dir,
end="")
sys.stdout.flush()
# Custom HTML reporter to clean up module names.
from coverage.html import HtmlReporter
class CustomHtmlReporter(HtmlReporter):
def find_code_units(self, morfs):
super(CustomHtmlReporter, self).find_code_units(morfs)
for cu in self.code_units:
nameparts = cu.name.split(os.sep)
if 'IPython' not in nameparts:
continue
ix = nameparts.index('IPython')
cu.name = '.'.join(nameparts[ix:])
# Reimplement the html_report method with our custom reporter
cov._harvest_data()
cov.config.from_args(
omit='*{0}tests{0}*'.format(os.sep),
html_dir=html_dir,
html_title='IPython test coverage',
)
reporter = CustomHtmlReporter(cov, cov.config)
reporter.report(None)
print('done.')
# Coverage XML report
elif options.coverage == 'xml':
cov.xml_report(outfile='ipy_coverage.xml')
if failed:
# Ensure that our exit code indicates failure
sys.exit(1)
def write_connection_file(
fname=None,
shell_port=0,
iopub_port=0,
stdin_port=0,
hb_port=0,
control_port=0,
ip='',
key=b'',
transport='tcp',
signature_scheme='hmac-sha256',
):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER (shell) channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the ROUTER (raw input) channel.
control_port : int, optional
The port to use for the ROUTER (control) channel.
hb_port : int, optional
The port to use for the heartbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for message authentication.
signature_scheme : str, optional
The scheme used for message authentication.
This has the form 'digest-hash', where 'digest'
is the scheme used for digests, and 'hash' is the name of the hash function
used by the digest scheme.
Currently, 'hmac' is the only supported digest scheme,
and 'sha256' is the default hash function.
"""
if not ip:
ip = localhost()
# default to temporary connector file
if not fname:
fname = tempfile.mktemp('.json')
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + \
int(iopub_port <= 0) + \
int(stdin_port <= 0) + \
int(control_port <= 0) + \
int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
sock.bind(('', 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if control_port <= 0:
control_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict(
shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
control_port=control_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
cfg['signature_scheme'] = signature_scheme
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
return fname, cfg
def shebang(self, line, cell):
"""Run a cell via a shell command
The `%%script` line is like the #! line of script,
specifying a program (bash, perl, ruby, etc.) with which to run.
The rest of the cell is run by that program.
Examples
--------
::
In [1]: %%script bash
...: for i in 1 2 3; do
...: echo $i
...: done
1
2
3
"""
argv = arg_split(line, posix = not sys.platform.startswith('win'))
args, cmd = self.shebang.parser.parse_known_args(argv)
try:
p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
except OSError as e:
if e.errno == errno.ENOENT:
print("Couldn't find program: %r" % cmd[0])
return
else:
raise
cell = cell.encode('utf8', 'replace')
if args.bg:
self.bg_processes.append(p)
self._gc_bg_processes()
if args.out:
self.shell.user_ns[args.out] = p.stdout
if args.err:
self.shell.user_ns[args.err] = p.stderr
self.job_manager.new(self._run_script, p, cell, daemon=True)
if args.proc:
self.shell.user_ns[args.proc] = p
return
try:
out, err = p.communicate(cell)
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print("Process is interrupted.")
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print("Process is terminated.")
return
p.kill()
print("Process is killed.")
except OSError:
pass
except Exception as e:
print("Error while terminating subprocess (pid=%i): %s" \
% (p.pid, e))
return
out = py3compat.bytes_to_str(out)
err = py3compat.bytes_to_str(err)
if args.out:
self.shell.user_ns[args.out] = out
else:
sys.stdout.write(out)
sys.stdout.flush()
if args.err:
self.shell.user_ns[args.err] = err
else:
sys.stderr.write(err)
sys.stderr.flush()
def shebang(self, _, cell):
"""Run a cell via a shell command
The `%%script` line is like the #! line of script,
specifying a program (bash, perl, ruby, etc.) with which to run.
The rest of the cell is run by that program.
Examples
--------
::
In [1]: %%script bash
...: for i in 1 2 3; do
...: echo $i
...: done
1
2
3
"""
try:
p = Popen("bash", stdout=PIPE, stderr=PIPE, stdin=PIPE)
except OSError as e:
if e.errno == errno.ENOENT:
print("Couldn't find program: bash")
return
else:
raise
# Inject built in commands
cell = '\n'.join([image_setup_cmd, cell])
if not cell.endswith('\n'):
cell += '\n'
cell = cell.encode('utf8', 'replace')
try:
out, err = p.communicate(cell)
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print("Process is interrupted.")
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print("Process is terminated.")
return
p.kill()
print("Process is killed.")
except OSError:
pass
except Exception as e:
print("Error while terminating subprocess (pid=%i): %s" %
(p.pid, e))
return
out = py3compat.bytes_to_str(out)
err = py3compat.bytes_to_str(err)
filenames, out = extract_display_filenames(out)
for filename in filenames:
# noinspection PyBroadException
image = Image.open(filename)
display(image)
sys.stdout.write(out)
sys.stdout.flush()
sys.stderr.write(err)
sys.stderr.flush()
def create_code_cell(code, execute = True, where = 'below'):
_load_ipython()
encoded_code = bytes_to_str(base64.b64encode(str_to_bytes(code)))
Idp.display(Idp.Javascript(_insert_code_cell_template.format(where, encoded_code, "code.execute();" if execute else "")))
def _find_cmd(cmd):
"""Find the full path to a command using which."""
path = sp.Popen(['/usr/bin/env', 'which', cmd],
stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0]
return py3compat.bytes_to_str(path)
def run_iptestall(options):
"""Run the entire IPython test suite by calling nose and trial.
This function constructs :class:`IPTester` instances for all IPython
modules and package and then runs each of them. This causes the modules
and packages of IPython to be tested each in their own subprocess using
nose.
Parameters
----------
All parameters are passed as attributes of the options object.
testgroups : list of str
Run only these sections of the test suite. If empty, run all the available
sections.
fast : int or None
Run the test suite in parallel, using n simultaneous processes. If None
is passed, one process is used per CPU core. Default 1 (i.e. sequential)
inc_slow : bool
Include slow tests, like IPython.parallel. By default, these tests aren't
run.
xunit : bool
Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
coverage : bool or str
Measure code coverage from tests. True will store the raw coverage data,
or pass 'html' or 'xml' to get reports.
"""
if options.fast != 1:
# If running in parallel, capture output so it doesn't get interleaved
TestController.buffer_output = True
if options.testgroups:
to_run = [PyTestController(name) for name in options.testgroups]
not_run = []
else:
to_run, not_run = prepare_py_test_controllers(inc_slow=options.all)
configure_controllers(to_run, xunit=options.xunit, coverage=options.coverage)
def justify(ltext, rtext, width=70, fill="-"):
ltext += " "
rtext = (" " + rtext).rjust(width - len(ltext), fill)
return ltext + rtext
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
print()
if options.fast == 1:
# This actually means sequential, i.e. with 1 job
for controller in to_run:
print("IPython test group:", controller.section)
controller, res = do_run(controller)
if res:
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
print()
else:
# Run tests concurrently
try:
pool = multiprocessing.pool.ThreadPool(options.fast)
for (controller, res) in pool.imap_unordered(do_run, to_run):
res_string = "OK" if res == 0 else "FAILED"
print(justify("IPython test group: " + controller.section, res_string))
if res:
print(bytes_to_str(controller.stdout))
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
except KeyboardInterrupt:
return
for controller in not_run:
print(justify("IPython test group: " + controller.section, "NOT RUN"))
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(to_run)
nfail = len(failed)
# summarize results
print("_" * 70)
print("Test suite completed for system with the following information:")
print(report())
print("Ran %s test groups in %.3fs" % (nrunners, t_tests))
print()
print("Status: ", end="")
if not failed:
print("OK")
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
failed_sections = [c.section for c in failed]
print("ERROR - {} out of {} test groups failed ({}).".format(nfail, nrunners, ", ".join(failed_sections)))
print()
print("You may wish to rerun these, with:")
print(" iptest", *failed_sections)
print()
if options.coverage:
from coverage import coverage
cov = coverage(data_file=".coverage")
cov.combine()
cov.save()
# Coverage HTML report
if options.coverage == "html":
html_dir = "ipy_htmlcov"
shutil.rmtree(html_dir, ignore_errors=True)
print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
sys.stdout.flush()
# Custom HTML reporter to clean up module names.
from coverage.html import HtmlReporter
class CustomHtmlReporter(HtmlReporter):
def find_code_units(self, morfs):
super(CustomHtmlReporter, self).find_code_units(morfs)
for cu in self.code_units:
nameparts = cu.name.split(os.sep)
if "IPython" not in nameparts:
continue
ix = nameparts.index("IPython")
cu.name = ".".join(nameparts[ix:])
# Reimplement the html_report method with our custom reporter
cov._harvest_data()
cov.config.from_args(omit="*%stests" % os.sep, html_dir=html_dir, html_title="IPython test coverage")
reporter = CustomHtmlReporter(cov, cov.config)
reporter.report(None)
print("done.")
# Coverage XML report
elif options.coverage == "xml":
cov.xml_report(outfile="ipy_coverage.xml")
if failed:
# Ensure that our exit code indicates failure
sys.exit(1)
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
ip=LOCALHOST, key=b'', transport='tcp'):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the REQ (raw input) channel.
hb_port : int, optional
The port to use for the hearbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for HMAC authentication.
"""
# default to temporary connector file
if not fname:
fname = tempfile.mktemp('.json')
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
int(stdin_port <= 0) + int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
sock.bind(('', 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict( shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
return fname, cfg
def shebang(self, line, cell):
"""Run a cell via a shell command
The `%%script` line is like the #! line of script,
specifying a program (bash, perl, ruby, etc.) with which to run.
The rest of the cell is run by that program.
Examples
--------
::
In [1]: %%script bash
...: for i in 1 2 3; do
...: echo $i
...: done
1
2
3
"""
argv = arg_split(line, posix=not sys.platform.startswith('win'))
args, cmd = self.shebang.parser.parse_known_args(argv)
try:
p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
except OSError as e:
if e.errno == errno.ENOENT:
print("Couldn't find program: %r" % cmd[0])
return
else:
raise
cell = cell.encode('utf8', 'replace')
if args.bg:
self.bg_processes.append(p)
self._gc_bg_processes()
if args.out:
self.shell.user_ns[args.out] = p.stdout
if args.err:
self.shell.user_ns[args.err] = p.stderr
self.job_manager.new(self._run_script, p, cell, daemon=True)
if args.proc:
self.shell.user_ns[args.proc] = p
return
try:
out, err = p.communicate(cell)
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
time.sleep(0.1)
if p.poll() is not None:
print("Process is interrupted.")
return
p.terminate()
time.sleep(0.1)
if p.poll() is not None:
print("Process is terminated.")
return
p.kill()
print("Process is killed.")
except OSError:
pass
except Exception as e:
print("Error while terminating subprocess (pid=%i): %s" \
% (p.pid, e))
return
out = py3compat.bytes_to_str(out)
err = py3compat.bytes_to_str(err)
if args.out:
self.shell.user_ns[args.out] = out
else:
sys.stdout.write(out)
sys.stdout.flush()
if args.err:
self.shell.user_ns[args.err] = err
else:
sys.stderr.write(err)
sys.stderr.flush()