def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None, pgo_step_name=None, _build_ext=build_ext): self._clear_distutils_mkpath_cache() dist = Distribution() config_files = dist.find_config_files() try: config_files.remove('setup.cfg') except ValueError: pass dist.parse_config_files(config_files) if not temp_dir: temp_dir = lib_dir add_pgo_flags = self._add_pgo_flags if pgo_step_name: base_build_ext = _build_ext class _build_ext(_build_ext): def build_extensions(self): add_pgo_flags(self, pgo_step_name, temp_dir) base_build_ext.build_extensions(self) build_extension = _build_ext(dist) build_extension.finalize_options() if temp_dir: temp_dir = py3compat.cast_bytes_py2(temp_dir, encoding=sys.getfilesystemencoding()) build_extension.build_temp = temp_dir if lib_dir: lib_dir = py3compat.cast_bytes_py2(lib_dir, encoding=sys.getfilesystemencoding()) build_extension.build_lib = lib_dir if extension is not None: build_extension.extensions = [extension] return build_extension
def set_env(self, parameter_s): """Set environment variables. Assumptions are that either "val" is a name in the user namespace, or val is something that evaluates to a string. Usage:\\ %set_env var val: set value for var %set_env var=val: set value for var %set_env var=$val: set value for var, using python expansion if possible """ split = '=' if '=' in parameter_s else ' ' bits = parameter_s.split(split, 1) if not parameter_s.strip() or len(bits) < 2: raise UsageError("usage is 'set_env var=val'") var = bits[0].strip() val = bits[1].strip() if re.match(r'.*\s.*', var): # an environment variable with whitespace is almost certainly # not what the user intended. what's more likely is the wrong # split was chosen, ie for "set_env cmd_args A=B", we chose # '=' for the split and should have chosen ' '. to get around # this, users should just assign directly to os.environ or use # standard magic {var} expansion. err = "refusing to set env var with whitespace: '{0}'" err = err.format(val) raise UsageError(err) os.environ[py3compat.cast_bytes_py2(var)] = py3compat.cast_bytes_py2( val) print('env: {0}={1}'.format(var, val))
def _ipython_dir_changed(self, name, old, new): if old is not None: str_old = py3compat.cast_bytes_py2(os.path.abspath(old), sys.getfilesystemencoding() ) if str_old in sys.path: sys.path.remove(str_old) str_path = py3compat.cast_bytes_py2(os.path.abspath(new), sys.getfilesystemencoding() ) sys.path.append(str_path) ensure_dir_exists(new) readme = os.path.join(new, 'README') readme_src = os.path.join( get_ipython_package_dir(), u'config', u'profile', 'README') if not os.path.exists(readme) and os.path.exists(readme_src): shutil.copy(readme_src, readme) for d in ('extensions', 'nbextensions'): path = os.path.join(new, d) try: ensure_dir_exists(path) except OSError: # this will not be EEXIST self.log.error("couldn't create path %s: %s", path, e) self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, name, old, new): str_old = py3compat.cast_bytes_py2(os.path.abspath(old), sys.getfilesystemencoding() ) if str_old in sys.path: sys.path.remove(str_old) str_path = py3compat.cast_bytes_py2(os.path.abspath(new), sys.getfilesystemencoding() ) sys.path.append(str_path) if not os.path.isdir(new): os.makedirs(new, mode=0o777) readme = os.path.join(new, 'README') readme_src = os.path.join( get_ipython_package_dir(), u'config', u'profile', 'README') if not os.path.exists(readme) and os.path.exists(readme_src): shutil.copy(readme_src, readme) for d in ('extensions', 'nbextensions'): path = os.path.join(new, d) if not os.path.exists(path): try: os.mkdir(path) except OSError as e: if e.errno != errno.EEXIST: self.log.error("couldn't create path %s: %s", path, e) self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, change): old = change['old'] new = change['new'] if old is not Undefined: str_old = py3compat.cast_bytes_py2(os.path.abspath(old), sys.getfilesystemencoding() ) if str_old in sys.path: sys.path.remove(str_old) str_path = py3compat.cast_bytes_py2(os.path.abspath(new), sys.getfilesystemencoding() ) sys.path.append(str_path) ensure_dir_exists(new) readme = os.path.join(new, 'README') readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README') if not os.path.exists(readme) and os.path.exists(readme_src): shutil.copy(readme_src, readme) for d in ('extensions', 'nbextensions'): path = os.path.join(new, d) try: ensure_dir_exists(path) except OSError as e: # this will not be EEXIST self.log.error("couldn't create path %s: %s", path, e) self.log.debug("IPYTHONDIR set to: %s" % new)
def set_env(self, parameter_s): """Set environment variables. Assumptions are that either "val" is a name in the user namespace, or val is something that evaluates to a string. Usage:\\ %set_env var val """ split = "=" if "=" in parameter_s else " " bits = parameter_s.split(split, 1) if not parameter_s.strip() or len(bits) < 2: raise UsageError("usage is 'set_env var=val'") var = bits[0].strip() val = bits[1].strip() if re.match(r".*\s.*", var): # an environment variable with whitespace is almost certainly # not what the user intended. what's more likely is the wrong # split was chosen, ie for "set_env cmd_args A=B", we chose # '=' for the split and should have chosen ' '. to get around # this, users should just assign directly to os.environ or use # standard magic {var} expansion. err = "refusing to set env var with whitespace: '{0}'" err = err.format(val) raise UsageError(err) os.environ[py3compat.cast_bytes_py2(var)] = py3compat.cast_bytes_py2(val) print("env: {0}={1}".format(var, val))
def _ipython_dir_changed(self, name, old, new): str_old = py3compat.cast_bytes_py2(os.path.abspath(old), sys.getfilesystemencoding() ) if str_old in sys.path: sys.path.remove(str_old) str_path = py3compat.cast_bytes_py2(os.path.abspath(new), sys.getfilesystemencoding() ) sys.path.append(str_path) if not os.path.isdir(new): os.makedirs(new, mode=0o777) readme = os.path.join(new, 'README') readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README') if not os.path.exists(readme) and os.path.exists(readme_src): shutil.copy(readme_src, readme) for d in ('extensions', 'nbextensions'): path = os.path.join(new, d) if not os.path.exists(path): try: os.mkdir(path) except OSError as e: if e.errno != errno.EEXIST: self.log.error("couldn't create path %s: %s", path, e) self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, name, old, new): str_old = py3compat.cast_bytes_py2(os.path.abspath(old), sys.getfilesystemencoding()) if str_old in sys.path: sys.path.remove(str_old) str_path = py3compat.cast_bytes_py2(os.path.abspath(new), sys.getfilesystemencoding()) sys.path.append(str_path) if not os.path.isdir(new): os.makedirs(new, mode=0o777) readme = os.path.join(new, 'README') if not os.path.exists(readme): path = os.path.join(get_ipython_package_dir(), 'config', 'profile') shutil.copy(os.path.join(path, 'README'), readme) self.log.debug("IPYTHONDIR set to: %s" % new)
def raw_input(self, prompt=''): if self.current_command > len(self.commands) - 1: echo('Do you really want to exit ([y]/n)? ', nl=False) wait_for(RETURNS) self.ask_exit() return '' # raw_input expects str, but we pass it unicode sometimes prompt = py3compat.cast_bytes_py2(prompt) try: command = self.commands[self.current_command] magictype(command, prompt_template=prompt, speed=self.speed) line = py3compat.cast_unicode_py2(command) except ValueError: warn("\n********\nYou or a %run:ed script called sys.stdin.close()" " or sys.stdout.close()!\nExiting IPython!\n") self.ask_exit() return "" # Try to be reasonably smart about not re-indenting pasted input more # than necessary. We do this by trimming out the auto-indent initial # spaces, if the user's actual input started itself with whitespace. if self.autoindent: if num_ini_spaces(line) > self.indent_current_nsp: line = line[self.indent_current_nsp:] self.indent_current_nsp = 0 self.current_command += 1 return line
def raw_input(self, prompt=''): """Write a prompt and read a line. The returned line does not include the trailing newline. When the user enters the EOF key sequence, EOFError is raised. Parameters ---------- prompt : str, optional A string to be printed to prompt the user. """ # raw_input expects str, but we pass it unicode sometimes prompt = py3compat.cast_bytes_py2(prompt) try: line = py3compat.cast_unicode_py2(self.raw_input_original(prompt)) except ValueError: warn("\n********\nYou or a %run:ed script called sys.stdin.close()" " or sys.stdout.close()!\nExiting IPython!\n") self.ask_exit() return "" # Try to be reasonably smart about not re-indenting pasted input more # than necessary. We do this by trimming out the auto-indent initial # spaces, if the user's actual input started itself with whitespace. if self.autoindent: if num_ini_spaces(line) > self.indent_current_nsp: line = line[self.indent_current_nsp:] self.indent_current_nsp = 0 return line
def raw_input(self, prompt=''): """Write a prompt and read a line. The returned line does not include the trailing newline. When the user enters the EOF key sequence, EOFError is raised. Parameters ---------- prompt : str, optional A string to be printed to prompt the user. """ # raw_input expects str, but we pass it unicode sometimes prompt = py3compat.cast_bytes_py2(prompt) try: line = py3compat.str_to_unicode(self.raw_input_original(prompt)) except ValueError: warn("\n********\nYou or a %run:ed script called sys.stdin.close()" " or sys.stdout.close()!\nExiting IPython!\n") self.ask_exit() return "" # Try to be reasonably smart about not re-indenting pasted input more # than necessary. We do this by trimming out the auto-indent initial # spaces, if the user's actual input started itself with whitespace. if self.autoindent: if num_ini_spaces(line) > self.indent_current_nsp: line = line[self.indent_current_nsp:] self.indent_current_nsp = 0 return line
def cython(self, line, cell): """Compile and import everything from a Cython code cell. The contents of the cell are written to a `.pyx` file in the directory `IPYTHONDIR/cython` using a filename with the hash of the code. This file is then cythonized and compiled. The resulting module is imported and all of its symbols are injected into the user's namespace. The usage is similar to that of `%%cython_pyximport` but you don't have to pass a module name:: %%cython def f(x): return 2.0*x """ args = parse_argstring(self.cython, line) code = cell if cell.endswith('\n') else cell + '\n' lib_dir = os.path.join(self.shell.ipython_dir, 'cython') cython_include_dirs = ['.'] force = args.force quiet = True ctx = Context(cython_include_dirs, default_options) key = code, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_magic_" + hashlib.md5( str(key).encode('utf-8')).hexdigest() module_path = os.path.join(lib_dir, module_name + self.so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): c_include_dirs = args.include if 'numpy' in code: import numpy c_include_dirs.append(numpy.get_include()) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_file = py3compat.cast_bytes_py2( pyx_file, encoding=sys.getfilesystemencoding()) with io.open(pyx_file, 'w', encoding='utf-8') as f: f.write(code) extension = Extension( name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=args.compile_args, libraries=args.lib, ) build_extension = self._get_build_extension() try: build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet) except CompileError: return build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() self._code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) self._import_all(module)
def _cythonize(self, module_name, code, lib_dir, args, quiet=True): pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding()) c_include_dirs = args.include c_src_files = list(map(str, args.src)) if 'numpy' in code: import numpy c_include_dirs.append(numpy.get_include()) with io.open(pyx_file, 'w', encoding='utf-8') as f: f.write(code) extension = Extension( name=module_name, sources=[pyx_file] + c_src_files, include_dirs=c_include_dirs, library_dirs=args.library_dirs, extra_compile_args=args.compile_args, extra_link_args=args.link_args, libraries=args.lib, language='c++' if args.cplus else 'c', ) try: opts = dict( quiet=quiet, annotate=args.annotate, force=True, language_level=min(3, sys.version_info[0]), ) if args.language_level is not None: assert args.language_level in (2, 3) opts['language_level'] = args.language_level return cythonize([extension], **opts) except CompileError: return None
def _ipython_dir_changed(self, name, old, new): str_old = py3compat.cast_bytes_py2(os.path.abspath(old), sys.getfilesystemencoding() ) if str_old in sys.path: sys.path.remove(str_old) str_path = py3compat.cast_bytes_py2(os.path.abspath(new), sys.getfilesystemencoding() ) sys.path.append(str_path) if not os.path.isdir(new): os.makedirs(new, mode=0o777) readme = os.path.join(new, 'README') if not os.path.exists(readme): path = os.path.join(get_ipython_package_dir(), 'config', 'profile') shutil.copy(os.path.join(path, 'README'), readme) self.log.debug("IPYTHONDIR set to: %s" % new)
def quote(s): """unicode-safe quote - Python 2 requires str, not unicode - always return unicode """ s = py3compat.cast_bytes_py2(s) quoted = stdlib_quote(s) return py3compat.str_to_unicode(quoted)
def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0): """Show detailed information about an object. Optional arguments: - oname: name of the variable pointing to the object. - formatter: special formatter for docstrings (see pdoc) - info: a structure with some information fields which may have been precomputed already. - detail_level: if set to 1, more information is given. """ info = self.info(obj, oname=oname, formatter=formatter, info=info, detail_level=detail_level) displayfields = [] for title, key in self.pinfo_fields1: field = info[key] if field is not None: displayfields.append((title, field.rstrip())) # Source or docstring, depending on detail level and whether # source found. if detail_level > 0 and info['source'] is not None: displayfields.append( ("Source", self.format(py3compat.cast_bytes_py2(info['source'])))) elif info['docstring'] is not None: displayfields.append(("Docstring", info["docstring"])) # Constructor info for classes if info['isclass']: if info['init_definition'] or info['init_docstring']: displayfields.append(("Constructor information", "")) if info['init_definition'] is not None: displayfields.append( (" Definition", info['init_definition'].rstrip())) if info['init_docstring'] is not None: displayfields.append( (" Docstring", indent(info['init_docstring']))) # Info for objects: else: for title, key in self.pinfo_fields_obj: field = info[key] if field is not None: displayfields.append((title, field.rstrip())) # Finally send to printer/pager: if displayfields: page.page(self._format_fields(displayfields))
def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None): numbers_width = INDENT_SIZE - 1 res = [] i = lnum - index # This lets us get fully syntax-highlighted tracebacks. if scheme is None: ipinst = ipapi.get() if ipinst is not None: scheme = ipinst.colors else: scheme = DEFAULT_SCHEME _line_format = _parser.format2 for line in lines: # FIXME: we need to ensure the source is a pure string at this point, # else the coloring code makes a royal mess. This is in need of a # serious refactoring, so that all of the ultratb and PyColorize code # is unicode-safe. So for now this is rather an ugly hack, but # necessary to at least have readable tracebacks. Improvements welcome! line = py3compat.cast_bytes_py2(line, 'utf-8') new_line, err = _line_format(line, 'str', scheme) if not err: line = new_line if i == lnum: # This is the line with the error pad = numbers_width - len(str(i)) if pad >= 3: marker = '-'*(pad-3) + '-> ' elif pad == 2: marker = '> ' elif pad == 1: marker = '>' else: marker = '' num = marker + str(i) line = '%s%s%s %s%s' %(Colors.linenoEm, num, Colors.line, line, Colors.Normal) else: num = '%*s' % (numbers_width,i) line = '%s%s%s %s' %(Colors.lineno, num, Colors.Normal, line) res.append(line) if lvals and i == lnum: res.append(lvals + '\n') i = i + 1 return res
def tunnel_to_kernel(connection_info, sshserver, sshkey=None): """tunnel connections to a kernel via ssh This will open four SSH tunnels from localhost on this machine to the ports associated with the kernel. They can be either direct localhost-localhost tunnels, or if an intermediate server is necessary, the kernel must be listening on a public IP. Parameters ---------- connection_info : dict or str (path) Either a connection dict, or the path to a JSON connection file sshserver : str The ssh sever to use to tunnel to the kernel. Can be a full `user@server:port` string. ssh config aliases are respected. sshkey : str [optional] Path to file containing ssh key to use for authentication. Only necessary if your ssh config does not already associate a keyfile with the host. Returns ------- (shell, iopub, stdin, hb) : ints The four ports on localhost that have been forwarded to the kernel. """ from zmq.ssh import tunnel if isinstance(connection_info, string_types): # it's a path, unpack it with open(connection_info) as f: connection_info = json.loads(f.read()) cf = connection_info lports = tunnel.select_random_ports(4) rports = cf['shell_port'], cf['iopub_port'], cf['stdin_port'], cf[ 'hb_port'] remote_ip = cf['ip'] if tunnel.try_passwordless_ssh(sshserver, sshkey): password = False else: password = getpass("SSH Password for %s: " % cast_bytes_py2(sshserver)) for lp, rp in zip(lports, rports): tunnel.ssh_tunnel(lp, rp, sshserver, remote_ip, sshkey, password) return tuple(lports)
def add_anchor(html): """Add an anchor-link to an html header tag For use in heading cells """ h = ElementTree.fromstring(py3compat.cast_bytes_py2(html)) link = html2text(h).replace(' ', '-') h.set('id', link) a = ElementTree.Element("a", {"class": "anchor-link", "href": "#" + link}) a.text = u'¶' h.append(a) # Known issue of Python3.x, ElementTree.tostring() returns a byte string # instead of a text string. See issue http://bugs.python.org/issue10942 # Workaround is to make sure the bytes are casted to a string. return py3compat.decode(ElementTree.tostring(h), 'utf-8')
def data(self, svg): if svg is None: self._data = None return # parse into dom object from xml.dom import minidom svg = cast_bytes_py2(svg) x = minidom.parseString(svg) # get svg tag (should be 1) found_svg = x.getElementsByTagName('svg') if found_svg: svg = found_svg[0].toxml() else: # fallback on the input, trust the user # but this is probably an error. pass svg = cast_unicode(svg) self._data = svg
def raw_input(self, prompt=''): """Write a prompt and read a line. The returned line does not include the trailing newline. When the user enters the EOF key sequence, EOFError is raised. Optional inputs: - prompt(''): a string to be printed to prompt the user. - continue_prompt(False): whether this line is the first one or a continuation in a sequence of inputs. """ # Code run by the user may have modified the readline completer state. # We must ensure that our completer is back in place. if self.has_readline: self.set_readline_completer() # raw_input expects str, but we pass it unicode sometimes prompt = py3compat.cast_bytes_py2(prompt) try: line = py3compat.str_to_unicode(self.raw_input_original(prompt)) except ValueError: warn("\n********\nYou or a %run:ed script called sys.stdin.close()" " or sys.stdout.close()!\nExiting IPython!\n") self.ask_exit() return "" # Try to be reasonably smart about not re-indenting pasted input more # than necessary. We do this by trimming out the auto-indent initial # spaces, if the user's actual input started itself with whitespace. if self.autoindent: if num_ini_spaces(line) > self.indent_current_nsp: line = line[self.indent_current_nsp:] self.indent_current_nsp = 0 return line
def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): """Display a string, piping through a pager after a certain length. strng can be a mime-bundle dict, supplying multiple representations, keyed by mime-type. The screen_lines parameter specifies the number of *usable* lines of your terminal screen (total lines minus lines you need to reserve to show other information). If you set screen_lines to a number <=0, page() will try to auto-determine your screen size and will only use up to (screen_size+screen_lines) for printing, paging after that. That is, if you want auto-detection but need to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for auto-detection without any lines reserved simply use screen_lines = 0. If a string won't fit in the allowed lines, it is sent through the specified pager command. If none given, look for PAGER in the environment, and ultimately default to less. If no system pager works, the string is sent through a 'dumb pager' written in python, very simplistic. """ # for compatibility with mime-bundle form: if isinstance(strng, dict): strng = strng['text/plain'] # Ugly kludge, but calling curses.initscr() flat out crashes in emacs TERM = os.environ.get('TERM', 'dumb') if TERM in ['dumb', 'emacs'] and os.name != 'nt': print(strng) return # chop off the topmost part of the string we don't want to see str_lines = strng.splitlines()[start:] str_toprint = os.linesep.join(str_lines) num_newlines = len(str_lines) len_str = len(str_toprint) # Dumb heuristics to guesstimate number of on-screen lines the string # takes. Very basic, but good enough for docstrings in reasonable # terminals. If someone later feels like refining it, it's not hard. numlines = max(num_newlines, int(len_str / 80) + 1) screen_lines_def = get_terminal_size()[1] # auto-determine screen size if screen_lines <= 0: try: screen_lines += _detect_screen_size(screen_lines_def) except (TypeError, UnsupportedOperation): print(str_toprint, file=io.stdout) return #print 'numlines',numlines,'screenlines',screen_lines # dbg if numlines <= screen_lines: #print '*** normal print' # dbg print(str_toprint, file=io.stdout) else: # Try to open pager and default to internal one if that fails. # All failure modes are tagged as 'retval=1', to match the return # value of a failed system command. If any intermediate attempt # sets retval to 1, at the end we resort to our own page_dumb() pager. pager_cmd = get_pager_cmd(pager_cmd) pager_cmd += ' ' + get_pager_start(pager_cmd, start) if os.name == 'nt': if pager_cmd.startswith('type'): # The default WinXP 'type' command is failing on complex strings. retval = 1 else: fd, tmpname = tempfile.mkstemp('.txt') try: os.close(fd) with open(tmpname, 'wt') as tmpfile: tmpfile.write(strng) cmd = "%s < %s" % (pager_cmd, tmpname) # tmpfile needs to be closed for windows if os.system(cmd): retval = 1 else: retval = None finally: os.remove(tmpname) else: try: retval = None # if I use popen4, things hang. No idea why. #pager,shell_out = os.popen4(pager_cmd) pager = os.popen(pager_cmd, 'w') try: pager_encoding = pager.encoding or sys.stdout.encoding pager.write( py3compat.cast_bytes_py2(strng, encoding=pager_encoding)) finally: retval = pager.close() except IOError as msg: # broken pipe when user quits if msg.args == (32, 'Broken pipe'): retval = None else: retval = 1 except OSError: # Other strange problems, sometimes seen in Win2k/cygwin retval = 1 if retval is not None: page_dumb(strng, screen_lines=screen_lines)
def fortran(self, line, cell): """Compile and import everything from a Fortran code cell, using f2py. The content of the cell is written to a `.f90` file in the directory `IPYTHONDIR/fortran` using a filename with the hash of the code. This file is then compiled. The resulting module is imported and all of its symbols are injected into the user's namespace. Usage ===== Prepend ``%%fortran`` to your fortran code in a cell:: ``%%fortran ! put your code here. `` """ try: # custom saved arguments saved_defaults = vars( magic_arguments.parse_argstring(self.fortran, self.shell.db['fortran'])) self.fortran.parser.set_defaults(**saved_defaults) except KeyError: saved_defaults = {'verbosity': 0} # verbosity is a "count" argument were each ocurrence is # added implicit. # so, for instance, -vv in %fortran_config and -vvv in %%fortran means # a nonsense verbosity=5. # To override: if verbosity is given for the magic cell # we ignore the saved config. if '-v' in line: self.fortran.parser.set_defaults(verbosity=0) args = magic_arguments.parse_argstring(self.fortran, line) # boolean flags f2py_args = ['--%s' % k for k, v in vars(args).items() if v is True] try: base_str_class = basestring except NameError: # py3 base_str_class = str kw = [ '--%s=%s' % (k, v) for k, v in vars(args).items() if isinstance(v, base_str_class) ] f2py_args.extend(kw) # link resource if args.link: resources = ['--link-%s' % r for r in args.link] f2py_args.extend(resources) if args.extra: extras = ' '.join(map(unquote, args.extra)) extras = extras.split() f2py_args.extend(extras) code = cell if cell.endswith('\n') else cell + '\n' key = code, line, sys.version_info, sys.executable, f2py2e.f2py_version module_name = "_fortran_magic_" + \ hashlib.md5(str(key).encode('utf-8')).hexdigest() module_path = os.path.join(self._lib_dir, module_name + self.so_ext) f90_file = os.path.join(self._lib_dir, module_name + '.f90') f90_file = py3compat.cast_bytes_py2( f90_file, encoding=sys.getfilesystemencoding()) with io.open(f90_file, 'w', encoding='utf-8') as f: f.write(code) self._run_f2py(f2py_args + ['-m', module_name, '-c', f90_file], verbosity=args.verbosity) self._code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) self._import_all(module, verbosity=args.verbosity)
def cython(self, line, cell): """Compile and import everything from a Cython code cell. The contents of the cell are written to a `.pyx` file in the directory `IPYTHONDIR/cython` using a filename with the hash of the code. This file is then cythonized and compiled. The resulting module is imported and all of its symbols are injected into the user's namespace. The usage is similar to that of `%%cython_pyximport` but you don't have to pass a module name:: %%cython def f(x): return 2.0*x To compile OpenMP codes, pass the required `--compile-args` and `--link-args`. For example with gcc:: %%cython --compile-args=-fopenmp --link-args=-fopenmp ... """ args = magic_arguments.parse_argstring(self.cython, line) code = cell if cell.endswith('\n') else cell + '\n' lib_dir = os.path.join(get_ipython_cache_dir(), 'cython') quiet = True key = code, sys.version_info, sys.executable, Cython.__version__ if not os.path.exists(lib_dir): os.makedirs(lib_dir) if args.force: # Force a new module name by adding the current time to the # key which is hashed to determine the module name. key += time.time(), module_name = "_cython_magic_" + hashlib.md5( str(key).encode('utf-8')).hexdigest() module_path = os.path.join(lib_dir, module_name + self.so_ext) have_module = os.path.isfile(module_path) need_cythonize = not have_module if args.annotate: html_file = os.path.join(lib_dir, module_name + '.html') if not os.path.isfile(html_file): need_cythonize = True if need_cythonize: c_include_dirs = args.include if 'numpy' in code: import numpy c_include_dirs.append(numpy.get_include()) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_file = py3compat.cast_bytes_py2( pyx_file, encoding=sys.getfilesystemencoding()) with io.open(pyx_file, 'w', encoding='utf-8') as f: f.write(code) extension = Extension( name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, library_dirs=args.library_dirs, extra_compile_args=args.compile_args, extra_link_args=args.link_args, libraries=args.lib, language='c++' if args.cplus else 'c', ) build_extension = self._get_build_extension() try: opts = dict( quiet=quiet, annotate=args.annotate, force=True, ) build_extension.extensions = cythonize([extension], **opts) except CompileError: return if not have_module: build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() self._code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) self._import_all(module) if args.annotate: try: with io.open(html_file, encoding='utf-8') as f: annotated_html = f.read() except IOError as e: # File could not be opened. Most likely the user has a version # of Cython before 0.15.1 (when `cythonize` learned the # `force` keyword argument) and has already compiled this # exact source without annotation. print( 'Cython completed successfully but the annotated ' 'source could not be read.', file=sys.stderr) print(e, file=sys.stderr) else: return display.HTML(self.clean_annotated_html(annotated_html))
def __init__(self, dir): self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, independent=False, cwd=None, ipython_kernel=True, **kw ): """ Launches a localhost kernel, binding to the specified ports. Parameters ---------- cmd : Popen list, A string of Python code that imports and executes a kernel entry point. stdin, stdout, stderr : optional (default None) Standards streams, as defined in subprocess.Popen. independent : bool, optional (default False) If set, the kernel process is guaranteed to survive if this process dies. If not set, an effort is made to ensure that the kernel is killed when this process dies. Note that in this case it is still good practice to kill kernels manually before exiting. cwd : path, optional The working dir of the kernel process (default: cwd of this process). ipython_kernel : bool, optional Whether the kernel is an official IPython one, and should get a bit of special treatment. Returns ------- Popen instance for the kernel subprocess """ # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr # are invalid. Unfortunately, there is in general no way to detect whether # they are valid. The following two blocks redirect them to (temporary) # pipes in certain important cases. # If this process has been backgrounded, our stdin is invalid. Since there # is no compelling reason for the kernel to inherit our stdin anyway, we'll # place this one safe and always redirect. redirect_in = True _stdin = PIPE if stdin is None else stdin # If this process in running on pythonw, we know that stdin, stdout, and # stderr are all invalid. redirect_out = sys.executable.endswith('pythonw.exe') if redirect_out: _stdout = PIPE if stdout is None else stdout _stderr = PIPE if stderr is None else stderr else: _stdout, _stderr = stdout, stderr env = env if (env is not None) else os.environ.copy() encoding = getdefaultencoding(prefer_stream=False) # Spawn a kernel. if sys.platform == 'win32': # Popen on Python 2 on Windows cannot handle unicode args or cwd cmd = [ cast_bytes_py2(c, encoding) for c in cmd ] if cwd: cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii') from IPython.kernel.zmq.parentpoller import ParentPollerWindows # Create a Win32 event for interrupting the kernel. interrupt_event = ParentPollerWindows.create_interrupt_event() # Store this in an environment variable for third party kernels, but at # present, our own kernel expects this as a command line argument. env["IPY_INTERRUPT_EVENT"] = str(interrupt_event) if ipython_kernel: cmd += [ '--interrupt=%i' % interrupt_event ] # If the kernel is running on pythonw and stdout/stderr are not been # re-directed, it will crash when more than 4KB of data is written to # stdout or stderr. This is a bug that has been with Python for a very # long time; see http://bugs.python.org/issue706263. # A cleaner solution to this problem would be to pass os.devnull to # Popen directly. Unfortunately, that does not work. if cmd[0].endswith('pythonw.exe'): if stdout is None: cmd.append('--no-stdout') if stderr is None: cmd.append('--no-stderr') # Launch the kernel process. if independent: proc = Popen(cmd, creationflags=512, # CREATE_NEW_PROCESS_GROUP stdin=_stdin, stdout=_stdout, stderr=_stderr, env=env) else: if ipython_kernel: try: from _winapi import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS except: from _subprocess import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS pid = GetCurrentProcess() handle = DuplicateHandle(pid, pid, pid, 0, True, # Inheritable by new processes. DUPLICATE_SAME_ACCESS) cmd +=[ '--parent=%i' % handle ] proc = Popen(cmd, stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env) # Attach the interrupt event to the Popen objet so it can be used later. proc.win32_interrupt_event = interrupt_event else: if independent: proc = Popen(cmd, preexec_fn=lambda: os.setsid(), stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env) else: if ipython_kernel: cmd += ['--parent=1'] proc = Popen(cmd, stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env) # Clean up pipes created to work around Popen bug. if redirect_in: if stdin is None: proc.stdin.close() if redirect_out: if stdout is None: proc.stdout.close() if stderr is None: proc.stderr.close() return proc
def pyd(self, line, cell): args = magic_arguments.parse_argstring(self.pyd, line) code = 'import ppyd;\n\n\ extern(C) void PydMain()\n{\n \ registerAll!(Alias!(__traits(parent, PydMain)))();\n\ }\n\n'\ + cell code = code if code.endswith('\n') else code + '\n' key = code, line, sys.version_info, sys.executable try: args.dub_config = json.loads(args.dub_config) except: args.dub_config = json.loads(ast.literal_eval(args.dub_config)) pass try: args.dub_args = ast.literal_eval(args.dub_args) except: pass if args.force: # Force a new module name by adding the current time to the # key which is hashed to determine the module name. key += (time.time(), ) args.dub_args = '--force ' + args.dub_args if args.name: module_name = py3compat.unicode_to_str(args.name) else: module_name = "_pyd_magic_" + hashlib.md5( str(key).encode('utf-8')).hexdigest() lib_dir = os.path.join(get_ipython_cache_dir(), 'pyd', module_name) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if os.name == 'nt': so_ext = '.dll' else: so_ext = '.so' #might have to go to dylib on OS X at some point??? module_path = os.path.join(lib_dir, 'lib' + module_name + so_ext) have_module = os.path.isfile(module_path) need_pydize = not have_module if need_pydize: d_include_dirs = args.include pyd_file = os.path.join(lib_dir, module_name + '.d') pyd_file = py3compat.cast_bytes_py2( pyd_file, encoding=sys.getfilesystemencoding()) with io.open(pyd_file, 'w', encoding='utf-8') as f: f.write(code) pyd_dub_file = os.path.join(lib_dir, 'dub.json') pyd_dub_file = py3compat.cast_bytes_py2( pyd_dub_file, encoding=sys.getfilesystemencoding()) pyd_dub_selections_file = os.path.join(lib_dir, 'dub.selections.json') pyd_dub_selections_file = py3compat.cast_bytes_py2( pyd_dub_selections_file, encoding=sys.getfilesystemencoding()) pyd_dub_json = json.loads('{}') pyd_dub_json['name'] = module_name pyd_dub_json['dependencies'] = { "pyd": args.pyd_version, "ppyd": ">=0.1.2" } pyd_dub_json['subConfigurations'] = { "pyd": "python{0}{1}".format(sys.version_info.major, sys.version_info.minor) } pyd_dub_json['sourceFiles'] = [pyd_file] pyd_dub_json['targetType'] = 'dynamicLibrary' pyd_dub_json['dflags'] = ['-fPIC'] pyd_dub_json['libs'] = ['phobos2'] pyd_dub_json['versions'] = ['PydPythonExtension'] with io.open(pyd_dub_file, 'w', encoding='utf-8') as f: f.write( unicode(json.dumps(pyd_dub_json) + '\n', encoding='utf-8')) try: os.remove(pyd_dub_selections_file) except: pass dub_desc = json.loads( subprocess.check_output( ["dub", "describe", "--root=" + lib_dir], universal_newlines=True)) for pack in dub_desc['packages']: if pack['name'] == 'pyd': _infraDir = os.path.join(pack['path'], 'infrastructure') break if os.name == 'nt': boilerplatePath = os.path.join( _infraDir, 'd', 'python_dll_windows_boilerplate.d') else: boilerplatePath = os.path.join( _infraDir, 'd', 'python_so_linux_boilerplate.d') pyd_dub_json['sourceFiles'].append(boilerplatePath) if args.compiler == 'dmd': so_ctor_path = os.path.join(_infraDir, 'd', 'so_ctor.c') so_ctor_object_path = os.path.join(lib_dir, "so_ctor.o") subprocess.check_call([ 'cc', "-c", "-fPIC", "-o" + so_ctor_object_path, so_ctor_path ]) pyd_dub_json['sourceFiles'].append(so_ctor_object_path) mainTemplate = os.path.join(_infraDir, 'd', 'pydmain_template.d') mainTemplate = py3compat.cast_bytes_py2( mainTemplate, encoding=sys.getfilesystemencoding()) mainTemplateOut = os.path.join(lib_dir, 'pydmain.d') mainTemplateOut = py3compat.cast_bytes_py2( mainTemplateOut, encoding=sys.getfilesystemencoding()) with io.open(mainTemplate, 'r', encoding='utf-8') as t, io.open( mainTemplateOut, 'w', encoding='utf-8') as m: m.write(t.read() % {'modulename': module_name}) pyd_dub_json['sourceFiles'].append(mainTemplateOut) pyd_dub_json = ConfigDict(pyd_dub_json) pyd_dub_json.merge(args.dub_config) with io.open(pyd_dub_file, 'w', encoding='utf-8') as f: f.write( unicode(json.dumps(pyd_dub_json) + '\n', encoding='utf-8')) try: output = subprocess.check_output( ["dub", "build", "--root=" + lib_dir] + args.dub_args.split(' '), universal_newlines=True, stderr=subprocess.STDOUT) except (subprocess.CalledProcessError) as e: print(e.output) raise e if args.print_compiler_output: print(output) if not have_module: self._code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) self._import_all(module)
def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, independent=False, cwd=None, **kw ): """ Launches a localhost kernel, binding to the specified ports. Parameters ---------- cmd : Popen list, A string of Python code that imports and executes a kernel entry point. stdin, stdout, stderr : optional (default None) Standards streams, as defined in subprocess.Popen. independent : bool, optional (default False) If set, the kernel process is guaranteed to survive if this process dies. If not set, an effort is made to ensure that the kernel is killed when this process dies. Note that in this case it is still good practice to kill kernels manually before exiting. cwd : path, optional The working dir of the kernel process (default: cwd of this process). Returns ------- Popen instance for the kernel subprocess """ # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr # are invalid. Unfortunately, there is in general no way to detect whether # they are valid. The following two blocks redirect them to (temporary) # pipes in certain important cases. # If this process has been backgrounded, our stdin is invalid. Since there # is no compelling reason for the kernel to inherit our stdin anyway, we'll # place this one safe and always redirect. redirect_in = True _stdin = PIPE if stdin is None else stdin # If this process in running on pythonw, we know that stdin, stdout, and # stderr are all invalid. redirect_out = sys.executable.endswith('pythonw.exe') if redirect_out: blackhole = open(os.devnull, 'w') _stdout = blackhole if stdout is None else stdout _stderr = blackhole if stderr is None else stderr else: _stdout, _stderr = stdout, stderr env = env if (env is not None) else os.environ.copy() encoding = getdefaultencoding(prefer_stream=False) kwargs = dict( stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env, ) # Spawn a kernel. if sys.platform == 'win32': # Popen on Python 2 on Windows cannot handle unicode args or cwd cmd = [ cast_bytes_py2(c, encoding) for c in cmd ] if cwd: cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii') kwargs['cwd'] = cwd from IPython.kernel.zmq.parentpoller import ParentPollerWindows # Create a Win32 event for interrupting the kernel # and store it in an environment variable. interrupt_event = ParentPollerWindows.create_interrupt_event() env["JPY_INTERRUPT_EVENT"] = str(interrupt_event) # deprecated old env name: env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"] try: from _winapi import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP except: from _subprocess import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP # Launch the kernel process if independent: kwargs['creationflags'] = CREATE_NEW_PROCESS_GROUP else: pid = GetCurrentProcess() handle = DuplicateHandle(pid, pid, pid, 0, True, # Inheritable by new processes. DUPLICATE_SAME_ACCESS) env['JPY_PARENT_PID'] = str(int(handle)) proc = Popen(cmd, **kwargs) # Attach the interrupt event to the Popen objet so it can be used later. proc.win32_interrupt_event = interrupt_event else: if independent: kwargs['preexec_fn'] = lambda: os.setsid() else: env['JPY_PARENT_PID'] = str(os.getpid()) proc = Popen(cmd, **kwargs) # Clean up pipes created to work around Popen bug. if redirect_in: if stdin is None: proc.stdin.close() return proc