def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None,
pgo_step_name=None, _build_ext=build_ext):
self._clear_distutils_mkpath_cache()
dist = Distribution()
config_files = dist.find_config_files()
try:
config_files.remove('setup.cfg')
except ValueError:
pass
dist.parse_config_files(config_files)
if not temp_dir:
temp_dir = lib_dir
add_pgo_flags = self._add_pgo_flags
if pgo_step_name:
base_build_ext = _build_ext
class _build_ext(_build_ext):
def build_extensions(self):
add_pgo_flags(self, pgo_step_name, temp_dir)
base_build_ext.build_extensions(self)
build_extension = _build_ext(dist)
build_extension.finalize_options()
if temp_dir:
temp_dir = py3compat.cast_bytes_py2(temp_dir, encoding=sys.getfilesystemencoding())
build_extension.build_temp = temp_dir
if lib_dir:
lib_dir = py3compat.cast_bytes_py2(lib_dir, encoding=sys.getfilesystemencoding())
build_extension.build_lib = lib_dir
if extension is not None:
build_extension.extensions = [extension]
return build_extension
def set_env(self, parameter_s):
"""Set environment variables. Assumptions are that either "val" is a
name in the user namespace, or val is something that evaluates to a
string.
Usage:\\
%set_env var val: set value for var
%set_env var=val: set value for var
%set_env var=$val: set value for var, using python expansion if possible
"""
split = '=' if '=' in parameter_s else ' '
bits = parameter_s.split(split, 1)
if not parameter_s.strip() or len(bits) < 2:
raise UsageError("usage is 'set_env var=val'")
var = bits[0].strip()
val = bits[1].strip()
if re.match(r'.*\s.*', var):
# an environment variable with whitespace is almost certainly
# not what the user intended. what's more likely is the wrong
# split was chosen, ie for "set_env cmd_args A=B", we chose
# '=' for the split and should have chosen ' '. to get around
# this, users should just assign directly to os.environ or use
# standard magic {var} expansion.
err = "refusing to set env var with whitespace: '{0}'"
err = err.format(val)
raise UsageError(err)
os.environ[py3compat.cast_bytes_py2(var)] = py3compat.cast_bytes_py2(
val)
print('env: {0}={1}'.format(var, val))
def _ipython_dir_changed(self, name, old, new):
if old is not None:
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding()
)
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding()
)
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, 'README')
readme_src = os.path.join(
get_ipython_package_dir(), u'config', u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, name, old, new):
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding()
)
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding()
)
sys.path.append(str_path)
if not os.path.isdir(new):
os.makedirs(new, mode=0o777)
readme = os.path.join(new, 'README')
readme_src = os.path.join(
get_ipython_package_dir(), u'config', u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
if not os.path.exists(path):
try:
os.mkdir(path)
except OSError as e:
if e.errno != errno.EEXIST:
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, change):
old = change['old']
new = change['new']
if old is not Undefined:
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding()
)
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding()
)
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, 'README')
readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError as e:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None,
pgo_step_name=None, _build_ext=build_ext):
self._clear_distutils_mkpath_cache()
dist = Distribution()
config_files = dist.find_config_files()
try:
config_files.remove('setup.cfg')
except ValueError:
pass
dist.parse_config_files(config_files)
if not temp_dir:
temp_dir = lib_dir
add_pgo_flags = self._add_pgo_flags
if pgo_step_name:
base_build_ext = _build_ext
class _build_ext(_build_ext):
def build_extensions(self):
add_pgo_flags(self, pgo_step_name, temp_dir)
base_build_ext.build_extensions(self)
build_extension = _build_ext(dist)
build_extension.finalize_options()
if temp_dir:
temp_dir = py3compat.cast_bytes_py2(temp_dir, encoding=sys.getfilesystemencoding())
build_extension.build_temp = temp_dir
if lib_dir:
lib_dir = py3compat.cast_bytes_py2(lib_dir, encoding=sys.getfilesystemencoding())
build_extension.build_lib = lib_dir
if extension is not None:
build_extension.extensions = [extension]
return build_extension
def set_env(self, parameter_s):
"""Set environment variables. Assumptions are that either "val" is a
name in the user namespace, or val is something that evaluates to a
string.
Usage:\\
%set_env var val
"""
split = "=" if "=" in parameter_s else " "
bits = parameter_s.split(split, 1)
if not parameter_s.strip() or len(bits) < 2:
raise UsageError("usage is 'set_env var=val'")
var = bits[0].strip()
val = bits[1].strip()
if re.match(r".*\s.*", var):
# an environment variable with whitespace is almost certainly
# not what the user intended. what's more likely is the wrong
# split was chosen, ie for "set_env cmd_args A=B", we chose
# '=' for the split and should have chosen ' '. to get around
# this, users should just assign directly to os.environ or use
# standard magic {var} expansion.
err = "refusing to set env var with whitespace: '{0}'"
err = err.format(val)
raise UsageError(err)
os.environ[py3compat.cast_bytes_py2(var)] = py3compat.cast_bytes_py2(val)
print("env: {0}={1}".format(var, val))
def _ipython_dir_changed(self, name, old, new):
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding()
)
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding()
)
sys.path.append(str_path)
if not os.path.isdir(new):
os.makedirs(new, mode=0o777)
readme = os.path.join(new, 'README')
readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
if not os.path.exists(path):
try:
os.mkdir(path)
except OSError as e:
if e.errno != errno.EEXIST:
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, name, old, new):
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding())
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding())
sys.path.append(str_path)
if not os.path.isdir(new):
os.makedirs(new, mode=0o777)
readme = os.path.join(new, 'README')
if not os.path.exists(readme):
path = os.path.join(get_ipython_package_dir(), 'config', 'profile')
shutil.copy(os.path.join(path, 'README'), readme)
self.log.debug("IPYTHONDIR set to: %s" % new)
def raw_input(self, prompt=''):
if self.current_command > len(self.commands) - 1:
echo('Do you really want to exit ([y]/n)? ', nl=False)
wait_for(RETURNS)
self.ask_exit()
return ''
# raw_input expects str, but we pass it unicode sometimes
prompt = py3compat.cast_bytes_py2(prompt)
try:
command = self.commands[self.current_command]
magictype(command, prompt_template=prompt, speed=self.speed)
line = py3compat.cast_unicode_py2(command)
except ValueError:
warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
" or sys.stdout.close()!\nExiting IPython!\n")
self.ask_exit()
return ""
# Try to be reasonably smart about not re-indenting pasted input more
# than necessary. We do this by trimming out the auto-indent initial
# spaces, if the user's actual input started itself with whitespace.
if self.autoindent:
if num_ini_spaces(line) > self.indent_current_nsp:
line = line[self.indent_current_nsp:]
self.indent_current_nsp = 0
self.current_command += 1
return line
def raw_input(self, prompt=''):
"""Write a prompt and read a line.
The returned line does not include the trailing newline.
When the user enters the EOF key sequence, EOFError is raised.
Parameters
----------
prompt : str, optional
A string to be printed to prompt the user.
"""
# raw_input expects str, but we pass it unicode sometimes
prompt = py3compat.cast_bytes_py2(prompt)
try:
line = py3compat.cast_unicode_py2(self.raw_input_original(prompt))
except ValueError:
warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
" or sys.stdout.close()!\nExiting IPython!\n")
self.ask_exit()
return ""
# Try to be reasonably smart about not re-indenting pasted input more
# than necessary. We do this by trimming out the auto-indent initial
# spaces, if the user's actual input started itself with whitespace.
if self.autoindent:
if num_ini_spaces(line) > self.indent_current_nsp:
line = line[self.indent_current_nsp:]
self.indent_current_nsp = 0
return line
def raw_input(self, prompt=''):
"""Write a prompt and read a line.
The returned line does not include the trailing newline.
When the user enters the EOF key sequence, EOFError is raised.
Parameters
----------
prompt : str, optional
A string to be printed to prompt the user.
"""
# raw_input expects str, but we pass it unicode sometimes
prompt = py3compat.cast_bytes_py2(prompt)
try:
line = py3compat.str_to_unicode(self.raw_input_original(prompt))
except ValueError:
warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
" or sys.stdout.close()!\nExiting IPython!\n")
self.ask_exit()
return ""
# Try to be reasonably smart about not re-indenting pasted input more
# than necessary. We do this by trimming out the auto-indent initial
# spaces, if the user's actual input started itself with whitespace.
if self.autoindent:
if num_ini_spaces(line) > self.indent_current_nsp:
line = line[self.indent_current_nsp:]
self.indent_current_nsp = 0
return line
def cython(self, line, cell):
"""Compile and import everything from a Cython code cell.
The contents of the cell are written to a `.pyx` file in the
directory `IPYTHONDIR/cython` using a filename with the hash of the code.
This file is then cythonized and compiled. The resulting module
is imported and all of its symbols are injected into the user's
namespace. The usage is similar to that of `%%cython_pyximport` but
you don't have to pass a module name::
%%cython
def f(x):
return 2.0*x
"""
args = parse_argstring(self.cython, line)
code = cell if cell.endswith('\n') else cell + '\n'
lib_dir = os.path.join(self.shell.ipython_dir, 'cython')
cython_include_dirs = ['.']
force = args.force
quiet = True
ctx = Context(cython_include_dirs, default_options)
key = code, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_magic_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
module_path = os.path.join(lib_dir, module_name + self.so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
c_include_dirs = args.include
if 'numpy' in code:
import numpy
c_include_dirs.append(numpy.get_include())
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = py3compat.cast_bytes_py2(
pyx_file, encoding=sys.getfilesystemencoding())
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
extension = Extension(
name=module_name,
sources=[pyx_file],
include_dirs=c_include_dirs,
extra_compile_args=args.compile_args,
libraries=args.lib,
)
build_extension = self._get_build_extension()
try:
build_extension.extensions = cythonize([extension],
ctx=ctx,
quiet=quiet)
except CompileError:
return
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
def _cythonize(self, module_name, code, lib_dir, args, quiet=True):
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
c_include_dirs = args.include
c_src_files = list(map(str, args.src))
if 'numpy' in code:
import numpy
c_include_dirs.append(numpy.get_include())
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
extension = Extension(
name=module_name,
sources=[pyx_file] + c_src_files,
include_dirs=c_include_dirs,
library_dirs=args.library_dirs,
extra_compile_args=args.compile_args,
extra_link_args=args.link_args,
libraries=args.lib,
language='c++' if args.cplus else 'c',
)
try:
opts = dict(
quiet=quiet,
annotate=args.annotate,
force=True,
language_level=min(3, sys.version_info[0]),
)
if args.language_level is not None:
assert args.language_level in (2, 3)
opts['language_level'] = args.language_level
return cythonize([extension], **opts)
except CompileError:
return None
def _ipython_dir_changed(self, name, old, new):
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding()
)
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding()
)
sys.path.append(str_path)
if not os.path.isdir(new):
os.makedirs(new, mode=0o777)
readme = os.path.join(new, 'README')
if not os.path.exists(readme):
path = os.path.join(get_ipython_package_dir(), 'config', 'profile')
shutil.copy(os.path.join(path, 'README'), readme)
self.log.debug("IPYTHONDIR set to: %s" % new)
def quote(s):
"""unicode-safe quote
- Python 2 requires str, not unicode
- always return unicode
"""
s = py3compat.cast_bytes_py2(s)
quoted = stdlib_quote(s)
return py3compat.str_to_unicode(quoted)
def quote(s):
"""unicode-safe quote
- Python 2 requires str, not unicode
- always return unicode
"""
s = py3compat.cast_bytes_py2(s)
quoted = stdlib_quote(s)
return py3compat.str_to_unicode(quoted)
def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0):
"""Show detailed information about an object.
Optional arguments:
- oname: name of the variable pointing to the object.
- formatter: special formatter for docstrings (see pdoc)
- info: a structure with some information fields which may have been
precomputed already.
- detail_level: if set to 1, more information is given.
"""
info = self.info(obj,
oname=oname,
formatter=formatter,
info=info,
detail_level=detail_level)
displayfields = []
for title, key in self.pinfo_fields1:
field = info[key]
if field is not None:
displayfields.append((title, field.rstrip()))
# Source or docstring, depending on detail level and whether
# source found.
if detail_level > 0 and info['source'] is not None:
displayfields.append(
("Source",
self.format(py3compat.cast_bytes_py2(info['source']))))
elif info['docstring'] is not None:
displayfields.append(("Docstring", info["docstring"]))
# Constructor info for classes
if info['isclass']:
if info['init_definition'] or info['init_docstring']:
displayfields.append(("Constructor information", ""))
if info['init_definition'] is not None:
displayfields.append(
(" Definition", info['init_definition'].rstrip()))
if info['init_docstring'] is not None:
displayfields.append(
(" Docstring", indent(info['init_docstring'])))
# Info for objects:
else:
for title, key in self.pinfo_fields_obj:
field = info[key]
if field is not None:
displayfields.append((title, field.rstrip()))
# Finally send to printer/pager:
if displayfields:
page.page(self._format_fields(displayfields))
def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):
numbers_width = INDENT_SIZE - 1
res = []
i = lnum - index
# This lets us get fully syntax-highlighted tracebacks.
if scheme is None:
ipinst = ipapi.get()
if ipinst is not None:
scheme = ipinst.colors
else:
scheme = DEFAULT_SCHEME
_line_format = _parser.format2
for line in lines:
# FIXME: we need to ensure the source is a pure string at this point,
# else the coloring code makes a royal mess. This is in need of a
# serious refactoring, so that all of the ultratb and PyColorize code
# is unicode-safe. So for now this is rather an ugly hack, but
# necessary to at least have readable tracebacks. Improvements welcome!
line = py3compat.cast_bytes_py2(line, 'utf-8')
new_line, err = _line_format(line, 'str', scheme)
if not err: line = new_line
if i == lnum:
# This is the line with the error
pad = numbers_width - len(str(i))
if pad >= 3:
marker = '-'*(pad-3) + '-> '
elif pad == 2:
marker = '> '
elif pad == 1:
marker = '>'
else:
marker = ''
num = marker + str(i)
line = '%s%s%s %s%s' %(Colors.linenoEm, num,
Colors.line, line, Colors.Normal)
else:
num = '%*s' % (numbers_width,i)
line = '%s%s%s %s' %(Colors.lineno, num,
Colors.Normal, line)
res.append(line)
if lvals and i == lnum:
res.append(lvals + '\n')
i = i + 1
return res
def tunnel_to_kernel(connection_info, sshserver, sshkey=None):
"""tunnel connections to a kernel via ssh
This will open four SSH tunnels from localhost on this machine to the
ports associated with the kernel. They can be either direct
localhost-localhost tunnels, or if an intermediate server is necessary,
the kernel must be listening on a public IP.
Parameters
----------
connection_info : dict or str (path)
Either a connection dict, or the path to a JSON connection file
sshserver : str
The ssh sever to use to tunnel to the kernel. Can be a full
`user@server:port` string. ssh config aliases are respected.
sshkey : str [optional]
Path to file containing ssh key to use for authentication.
Only necessary if your ssh config does not already associate
a keyfile with the host.
Returns
-------
(shell, iopub, stdin, hb) : ints
The four ports on localhost that have been forwarded to the kernel.
"""
from zmq.ssh import tunnel
if isinstance(connection_info, string_types):
# it's a path, unpack it
with open(connection_info) as f:
connection_info = json.loads(f.read())
cf = connection_info
lports = tunnel.select_random_ports(4)
rports = cf['shell_port'], cf['iopub_port'], cf['stdin_port'], cf[
'hb_port']
remote_ip = cf['ip']
if tunnel.try_passwordless_ssh(sshserver, sshkey):
password = False
else:
password = getpass("SSH Password for %s: " % cast_bytes_py2(sshserver))
for lp, rp in zip(lports, rports):
tunnel.ssh_tunnel(lp, rp, sshserver, remote_ip, sshkey, password)
return tuple(lports)
def add_anchor(html):
"""Add an anchor-link to an html header tag
For use in heading cells
"""
h = ElementTree.fromstring(py3compat.cast_bytes_py2(html))
link = html2text(h).replace(' ', '-')
h.set('id', link)
a = ElementTree.Element("a", {"class": "anchor-link", "href": "#" + link})
a.text = u'¶'
h.append(a)
# Known issue of Python3.x, ElementTree.tostring() returns a byte string
# instead of a text string. See issue http://bugs.python.org/issue10942
# Workaround is to make sure the bytes are casted to a string.
return py3compat.decode(ElementTree.tostring(h), 'utf-8')
def data(self, svg):
if svg is None:
self._data = None
return
# parse into dom object
from xml.dom import minidom
svg = cast_bytes_py2(svg)
x = minidom.parseString(svg)
# get svg tag (should be 1)
found_svg = x.getElementsByTagName('svg')
if found_svg:
svg = found_svg[0].toxml()
else:
# fallback on the input, trust the user
# but this is probably an error.
pass
svg = cast_unicode(svg)
self._data = svg
def raw_input(self, prompt=''):
"""Write a prompt and read a line.
The returned line does not include the trailing newline.
When the user enters the EOF key sequence, EOFError is raised.
Optional inputs:
- prompt(''): a string to be printed to prompt the user.
- continue_prompt(False): whether this line is the first one or a
continuation in a sequence of inputs.
"""
# Code run by the user may have modified the readline completer state.
# We must ensure that our completer is back in place.
if self.has_readline:
self.set_readline_completer()
# raw_input expects str, but we pass it unicode sometimes
prompt = py3compat.cast_bytes_py2(prompt)
try:
line = py3compat.str_to_unicode(self.raw_input_original(prompt))
except ValueError:
warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
" or sys.stdout.close()!\nExiting IPython!\n")
self.ask_exit()
return ""
# Try to be reasonably smart about not re-indenting pasted input more
# than necessary. We do this by trimming out the auto-indent initial
# spaces, if the user's actual input started itself with whitespace.
if self.autoindent:
if num_ini_spaces(line) > self.indent_current_nsp:
line = line[self.indent_current_nsp:]
self.indent_current_nsp = 0
return line
def pager_page(strng, start=0, screen_lines=0, pager_cmd=None):
"""Display a string, piping through a pager after a certain length.
strng can be a mime-bundle dict, supplying multiple representations,
keyed by mime-type.
The screen_lines parameter specifies the number of *usable* lines of your
terminal screen (total lines minus lines you need to reserve to show other
information).
If you set screen_lines to a number <=0, page() will try to auto-determine
your screen size and will only use up to (screen_size+screen_lines) for
printing, paging after that. That is, if you want auto-detection but need
to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
auto-detection without any lines reserved simply use screen_lines = 0.
If a string won't fit in the allowed lines, it is sent through the
specified pager command. If none given, look for PAGER in the environment,
and ultimately default to less.
If no system pager works, the string is sent through a 'dumb pager'
written in python, very simplistic.
"""
# for compatibility with mime-bundle form:
if isinstance(strng, dict):
strng = strng['text/plain']
# Ugly kludge, but calling curses.initscr() flat out crashes in emacs
TERM = os.environ.get('TERM', 'dumb')
if TERM in ['dumb', 'emacs'] and os.name != 'nt':
print(strng)
return
# chop off the topmost part of the string we don't want to see
str_lines = strng.splitlines()[start:]
str_toprint = os.linesep.join(str_lines)
num_newlines = len(str_lines)
len_str = len(str_toprint)
# Dumb heuristics to guesstimate number of on-screen lines the string
# takes. Very basic, but good enough for docstrings in reasonable
# terminals. If someone later feels like refining it, it's not hard.
numlines = max(num_newlines, int(len_str / 80) + 1)
screen_lines_def = get_terminal_size()[1]
# auto-determine screen size
if screen_lines <= 0:
try:
screen_lines += _detect_screen_size(screen_lines_def)
except (TypeError, UnsupportedOperation):
print(str_toprint, file=io.stdout)
return
#print 'numlines',numlines,'screenlines',screen_lines # dbg
if numlines <= screen_lines:
#print '*** normal print' # dbg
print(str_toprint, file=io.stdout)
else:
# Try to open pager and default to internal one if that fails.
# All failure modes are tagged as 'retval=1', to match the return
# value of a failed system command. If any intermediate attempt
# sets retval to 1, at the end we resort to our own page_dumb() pager.
pager_cmd = get_pager_cmd(pager_cmd)
pager_cmd += ' ' + get_pager_start(pager_cmd, start)
if os.name == 'nt':
if pager_cmd.startswith('type'):
# The default WinXP 'type' command is failing on complex strings.
retval = 1
else:
fd, tmpname = tempfile.mkstemp('.txt')
try:
os.close(fd)
with open(tmpname, 'wt') as tmpfile:
tmpfile.write(strng)
cmd = "%s < %s" % (pager_cmd, tmpname)
# tmpfile needs to be closed for windows
if os.system(cmd):
retval = 1
else:
retval = None
finally:
os.remove(tmpname)
else:
try:
retval = None
# if I use popen4, things hang. No idea why.
#pager,shell_out = os.popen4(pager_cmd)
pager = os.popen(pager_cmd, 'w')
try:
pager_encoding = pager.encoding or sys.stdout.encoding
pager.write(
py3compat.cast_bytes_py2(strng,
encoding=pager_encoding))
finally:
retval = pager.close()
except IOError as msg: # broken pipe when user quits
if msg.args == (32, 'Broken pipe'):
retval = None
else:
retval = 1
except OSError:
# Other strange problems, sometimes seen in Win2k/cygwin
retval = 1
if retval is not None:
page_dumb(strng, screen_lines=screen_lines)
def fortran(self, line, cell):
"""Compile and import everything from a Fortran code cell, using f2py.
The content of the cell is written to a `.f90` file in the
directory `IPYTHONDIR/fortran` using a filename with the hash of the
code. This file is then compiled. The resulting module
is imported and all of its symbols are injected into the user's
namespace.
Usage
=====
Prepend ``%%fortran`` to your fortran code in a cell::
``%%fortran
! put your code here.
``
"""
try:
# custom saved arguments
saved_defaults = vars(
magic_arguments.parse_argstring(self.fortran,
self.shell.db['fortran']))
self.fortran.parser.set_defaults(**saved_defaults)
except KeyError:
saved_defaults = {'verbosity': 0}
# verbosity is a "count" argument were each ocurrence is
# added implicit.
# so, for instance, -vv in %fortran_config and -vvv in %%fortran means
# a nonsense verbosity=5.
# To override: if verbosity is given for the magic cell
# we ignore the saved config.
if '-v' in line:
self.fortran.parser.set_defaults(verbosity=0)
args = magic_arguments.parse_argstring(self.fortran, line)
# boolean flags
f2py_args = ['--%s' % k for k, v in vars(args).items() if v is True]
try:
base_str_class = basestring
except NameError:
# py3
base_str_class = str
kw = [
'--%s=%s' % (k, v) for k, v in vars(args).items()
if isinstance(v, base_str_class)
]
f2py_args.extend(kw)
# link resource
if args.link:
resources = ['--link-%s' % r for r in args.link]
f2py_args.extend(resources)
if args.extra:
extras = ' '.join(map(unquote, args.extra))
extras = extras.split()
f2py_args.extend(extras)
code = cell if cell.endswith('\n') else cell + '\n'
key = code, line, sys.version_info, sys.executable, f2py2e.f2py_version
module_name = "_fortran_magic_" + \
hashlib.md5(str(key).encode('utf-8')).hexdigest()
module_path = os.path.join(self._lib_dir, module_name + self.so_ext)
f90_file = os.path.join(self._lib_dir, module_name + '.f90')
f90_file = py3compat.cast_bytes_py2(
f90_file, encoding=sys.getfilesystemencoding())
with io.open(f90_file, 'w', encoding='utf-8') as f:
f.write(code)
self._run_f2py(f2py_args + ['-m', module_name, '-c', f90_file],
verbosity=args.verbosity)
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module, verbosity=args.verbosity)
def cython(self, line, cell):
"""Compile and import everything from a Cython code cell.
The contents of the cell are written to a `.pyx` file in the
directory `IPYTHONDIR/cython` using a filename with the hash of the
code. This file is then cythonized and compiled. The resulting module
is imported and all of its symbols are injected into the user's
namespace. The usage is similar to that of `%%cython_pyximport` but
you don't have to pass a module name::
%%cython
def f(x):
return 2.0*x
To compile OpenMP codes, pass the required `--compile-args`
and `--link-args`. For example with gcc::
%%cython --compile-args=-fopenmp --link-args=-fopenmp
...
"""
args = magic_arguments.parse_argstring(self.cython, line)
code = cell if cell.endswith('\n') else cell + '\n'
lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
quiet = True
key = code, sys.version_info, sys.executable, Cython.__version__
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if args.force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += time.time(),
module_name = "_cython_magic_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
module_path = os.path.join(lib_dir, module_name + self.so_ext)
have_module = os.path.isfile(module_path)
need_cythonize = not have_module
if args.annotate:
html_file = os.path.join(lib_dir, module_name + '.html')
if not os.path.isfile(html_file):
need_cythonize = True
if need_cythonize:
c_include_dirs = args.include
if 'numpy' in code:
import numpy
c_include_dirs.append(numpy.get_include())
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = py3compat.cast_bytes_py2(
pyx_file, encoding=sys.getfilesystemencoding())
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
extension = Extension(
name=module_name,
sources=[pyx_file],
include_dirs=c_include_dirs,
library_dirs=args.library_dirs,
extra_compile_args=args.compile_args,
extra_link_args=args.link_args,
libraries=args.lib,
language='c++' if args.cplus else 'c',
)
build_extension = self._get_build_extension()
try:
opts = dict(
quiet=quiet,
annotate=args.annotate,
force=True,
)
build_extension.extensions = cythonize([extension], **opts)
except CompileError:
return
if not have_module:
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
if args.annotate:
try:
with io.open(html_file, encoding='utf-8') as f:
annotated_html = f.read()
except IOError as e:
# File could not be opened. Most likely the user has a version
# of Cython before 0.15.1 (when `cythonize` learned the
# `force` keyword argument) and has already compiled this
# exact source without annotation.
print(
'Cython completed successfully but the annotated '
'source could not be read.',
file=sys.stderr)
print(e, file=sys.stderr)
else:
return display.HTML(self.clean_annotated_html(annotated_html))
def __init__(self, dir):
self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None,
independent=False,
cwd=None, ipython_kernel=True,
**kw
):
""" Launches a localhost kernel, binding to the specified ports.
Parameters
----------
cmd : Popen list,
A string of Python code that imports and executes a kernel entry point.
stdin, stdout, stderr : optional (default None)
Standards streams, as defined in subprocess.Popen.
independent : bool, optional (default False)
If set, the kernel process is guaranteed to survive if this process
dies. If not set, an effort is made to ensure that the kernel is killed
when this process dies. Note that in this case it is still good practice
to kill kernels manually before exiting.
cwd : path, optional
The working dir of the kernel process (default: cwd of this process).
ipython_kernel : bool, optional
Whether the kernel is an official IPython one,
and should get a bit of special treatment.
Returns
-------
Popen instance for the kernel subprocess
"""
# Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
# are invalid. Unfortunately, there is in general no way to detect whether
# they are valid. The following two blocks redirect them to (temporary)
# pipes in certain important cases.
# If this process has been backgrounded, our stdin is invalid. Since there
# is no compelling reason for the kernel to inherit our stdin anyway, we'll
# place this one safe and always redirect.
redirect_in = True
_stdin = PIPE if stdin is None else stdin
# If this process in running on pythonw, we know that stdin, stdout, and
# stderr are all invalid.
redirect_out = sys.executable.endswith('pythonw.exe')
if redirect_out:
_stdout = PIPE if stdout is None else stdout
_stderr = PIPE if stderr is None else stderr
else:
_stdout, _stderr = stdout, stderr
env = env if (env is not None) else os.environ.copy()
encoding = getdefaultencoding(prefer_stream=False)
# Spawn a kernel.
if sys.platform == 'win32':
# Popen on Python 2 on Windows cannot handle unicode args or cwd
cmd = [ cast_bytes_py2(c, encoding) for c in cmd ]
if cwd:
cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii')
from IPython.kernel.zmq.parentpoller import ParentPollerWindows
# Create a Win32 event for interrupting the kernel.
interrupt_event = ParentPollerWindows.create_interrupt_event()
# Store this in an environment variable for third party kernels, but at
# present, our own kernel expects this as a command line argument.
env["IPY_INTERRUPT_EVENT"] = str(interrupt_event)
if ipython_kernel:
cmd += [ '--interrupt=%i' % interrupt_event ]
# If the kernel is running on pythonw and stdout/stderr are not been
# re-directed, it will crash when more than 4KB of data is written to
# stdout or stderr. This is a bug that has been with Python for a very
# long time; see http://bugs.python.org/issue706263.
# A cleaner solution to this problem would be to pass os.devnull to
# Popen directly. Unfortunately, that does not work.
if cmd[0].endswith('pythonw.exe'):
if stdout is None:
cmd.append('--no-stdout')
if stderr is None:
cmd.append('--no-stderr')
# Launch the kernel process.
if independent:
proc = Popen(cmd,
creationflags=512, # CREATE_NEW_PROCESS_GROUP
stdin=_stdin, stdout=_stdout, stderr=_stderr, env=env)
else:
if ipython_kernel:
try:
from _winapi import DuplicateHandle, GetCurrentProcess, \
DUPLICATE_SAME_ACCESS
except:
from _subprocess import DuplicateHandle, GetCurrentProcess, \
DUPLICATE_SAME_ACCESS
pid = GetCurrentProcess()
handle = DuplicateHandle(pid, pid, pid, 0,
True, # Inheritable by new processes.
DUPLICATE_SAME_ACCESS)
cmd +=[ '--parent=%i' % handle ]
proc = Popen(cmd,
stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
# Attach the interrupt event to the Popen objet so it can be used later.
proc.win32_interrupt_event = interrupt_event
else:
if independent:
proc = Popen(cmd, preexec_fn=lambda: os.setsid(),
stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
else:
if ipython_kernel:
cmd += ['--parent=1']
proc = Popen(cmd,
stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
# Clean up pipes created to work around Popen bug.
if redirect_in:
if stdin is None:
proc.stdin.close()
if redirect_out:
if stdout is None:
proc.stdout.close()
if stderr is None:
proc.stderr.close()
return proc
def pyd(self, line, cell):
args = magic_arguments.parse_argstring(self.pyd, line)
code = 'import ppyd;\n\n\
extern(C) void PydMain()\n{\n \
registerAll!(Alias!(__traits(parent, PydMain)))();\n\
}\n\n'\
+ cell
code = code if code.endswith('\n') else code + '\n'
key = code, line, sys.version_info, sys.executable
try:
args.dub_config = json.loads(args.dub_config)
except:
args.dub_config = json.loads(ast.literal_eval(args.dub_config))
pass
try:
args.dub_args = ast.literal_eval(args.dub_args)
except:
pass
if args.force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += (time.time(), )
args.dub_args = '--force ' + args.dub_args
if args.name:
module_name = py3compat.unicode_to_str(args.name)
else:
module_name = "_pyd_magic_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
lib_dir = os.path.join(get_ipython_cache_dir(), 'pyd', module_name)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if os.name == 'nt':
so_ext = '.dll'
else:
so_ext = '.so' #might have to go to dylib on OS X at some point???
module_path = os.path.join(lib_dir, 'lib' + module_name + so_ext)
have_module = os.path.isfile(module_path)
need_pydize = not have_module
if need_pydize:
d_include_dirs = args.include
pyd_file = os.path.join(lib_dir, module_name + '.d')
pyd_file = py3compat.cast_bytes_py2(
pyd_file, encoding=sys.getfilesystemencoding())
with io.open(pyd_file, 'w', encoding='utf-8') as f:
f.write(code)
pyd_dub_file = os.path.join(lib_dir, 'dub.json')
pyd_dub_file = py3compat.cast_bytes_py2(
pyd_dub_file, encoding=sys.getfilesystemencoding())
pyd_dub_selections_file = os.path.join(lib_dir,
'dub.selections.json')
pyd_dub_selections_file = py3compat.cast_bytes_py2(
pyd_dub_selections_file, encoding=sys.getfilesystemencoding())
pyd_dub_json = json.loads('{}')
pyd_dub_json['name'] = module_name
pyd_dub_json['dependencies'] = {
"pyd": args.pyd_version,
"ppyd": ">=0.1.2"
}
pyd_dub_json['subConfigurations'] = {
"pyd":
"python{0}{1}".format(sys.version_info.major,
sys.version_info.minor)
}
pyd_dub_json['sourceFiles'] = [pyd_file]
pyd_dub_json['targetType'] = 'dynamicLibrary'
pyd_dub_json['dflags'] = ['-fPIC']
pyd_dub_json['libs'] = ['phobos2']
pyd_dub_json['versions'] = ['PydPythonExtension']
with io.open(pyd_dub_file, 'w', encoding='utf-8') as f:
f.write(
unicode(json.dumps(pyd_dub_json) + '\n', encoding='utf-8'))
try:
os.remove(pyd_dub_selections_file)
except:
pass
dub_desc = json.loads(
subprocess.check_output(
["dub", "describe", "--root=" + lib_dir],
universal_newlines=True))
for pack in dub_desc['packages']:
if pack['name'] == 'pyd':
_infraDir = os.path.join(pack['path'], 'infrastructure')
break
if os.name == 'nt':
boilerplatePath = os.path.join(
_infraDir, 'd', 'python_dll_windows_boilerplate.d')
else:
boilerplatePath = os.path.join(
_infraDir, 'd', 'python_so_linux_boilerplate.d')
pyd_dub_json['sourceFiles'].append(boilerplatePath)
if args.compiler == 'dmd':
so_ctor_path = os.path.join(_infraDir, 'd', 'so_ctor.c')
so_ctor_object_path = os.path.join(lib_dir, "so_ctor.o")
subprocess.check_call([
'cc', "-c", "-fPIC", "-o" + so_ctor_object_path,
so_ctor_path
])
pyd_dub_json['sourceFiles'].append(so_ctor_object_path)
mainTemplate = os.path.join(_infraDir, 'd', 'pydmain_template.d')
mainTemplate = py3compat.cast_bytes_py2(
mainTemplate, encoding=sys.getfilesystemencoding())
mainTemplateOut = os.path.join(lib_dir, 'pydmain.d')
mainTemplateOut = py3compat.cast_bytes_py2(
mainTemplateOut, encoding=sys.getfilesystemencoding())
with io.open(mainTemplate, 'r', encoding='utf-8') as t, io.open(
mainTemplateOut, 'w', encoding='utf-8') as m:
m.write(t.read() % {'modulename': module_name})
pyd_dub_json['sourceFiles'].append(mainTemplateOut)
pyd_dub_json = ConfigDict(pyd_dub_json)
pyd_dub_json.merge(args.dub_config)
with io.open(pyd_dub_file, 'w', encoding='utf-8') as f:
f.write(
unicode(json.dumps(pyd_dub_json) + '\n', encoding='utf-8'))
try:
output = subprocess.check_output(
["dub", "build", "--root=" + lib_dir] +
args.dub_args.split(' '),
universal_newlines=True,
stderr=subprocess.STDOUT)
except (subprocess.CalledProcessError) as e:
print(e.output)
raise e
if args.print_compiler_output:
print(output)
if not have_module:
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None,
independent=False,
cwd=None,
**kw
):
""" Launches a localhost kernel, binding to the specified ports.
Parameters
----------
cmd : Popen list,
A string of Python code that imports and executes a kernel entry point.
stdin, stdout, stderr : optional (default None)
Standards streams, as defined in subprocess.Popen.
independent : bool, optional (default False)
If set, the kernel process is guaranteed to survive if this process
dies. If not set, an effort is made to ensure that the kernel is killed
when this process dies. Note that in this case it is still good practice
to kill kernels manually before exiting.
cwd : path, optional
The working dir of the kernel process (default: cwd of this process).
Returns
-------
Popen instance for the kernel subprocess
"""
# Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
# are invalid. Unfortunately, there is in general no way to detect whether
# they are valid. The following two blocks redirect them to (temporary)
# pipes in certain important cases.
# If this process has been backgrounded, our stdin is invalid. Since there
# is no compelling reason for the kernel to inherit our stdin anyway, we'll
# place this one safe and always redirect.
redirect_in = True
_stdin = PIPE if stdin is None else stdin
# If this process in running on pythonw, we know that stdin, stdout, and
# stderr are all invalid.
redirect_out = sys.executable.endswith('pythonw.exe')
if redirect_out:
blackhole = open(os.devnull, 'w')
_stdout = blackhole if stdout is None else stdout
_stderr = blackhole if stderr is None else stderr
else:
_stdout, _stderr = stdout, stderr
env = env if (env is not None) else os.environ.copy()
encoding = getdefaultencoding(prefer_stream=False)
kwargs = dict(
stdin=_stdin,
stdout=_stdout,
stderr=_stderr,
cwd=cwd,
env=env,
)
# Spawn a kernel.
if sys.platform == 'win32':
# Popen on Python 2 on Windows cannot handle unicode args or cwd
cmd = [ cast_bytes_py2(c, encoding) for c in cmd ]
if cwd:
cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii')
kwargs['cwd'] = cwd
from IPython.kernel.zmq.parentpoller import ParentPollerWindows
# Create a Win32 event for interrupting the kernel
# and store it in an environment variable.
interrupt_event = ParentPollerWindows.create_interrupt_event()
env["JPY_INTERRUPT_EVENT"] = str(interrupt_event)
# deprecated old env name:
env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"]
try:
from _winapi import DuplicateHandle, GetCurrentProcess, \
DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP
except:
from _subprocess import DuplicateHandle, GetCurrentProcess, \
DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP
# Launch the kernel process
if independent:
kwargs['creationflags'] = CREATE_NEW_PROCESS_GROUP
else:
pid = GetCurrentProcess()
handle = DuplicateHandle(pid, pid, pid, 0,
True, # Inheritable by new processes.
DUPLICATE_SAME_ACCESS)
env['JPY_PARENT_PID'] = str(int(handle))
proc = Popen(cmd, **kwargs)
# Attach the interrupt event to the Popen objet so it can be used later.
proc.win32_interrupt_event = interrupt_event
else:
if independent:
kwargs['preexec_fn'] = lambda: os.setsid()
else:
env['JPY_PARENT_PID'] = str(os.getpid())
proc = Popen(cmd, **kwargs)
# Clean up pipes created to work around Popen bug.
if redirect_in:
if stdin is None:
proc.stdin.close()
return proc
