def _exception_handler(self,
i_shell: InteractiveShell,
etype,
value,
tb: TracebackType,
tb_offset=None):
i_shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
self._notify_of_exception(i_shell, etype, value, tb, tb_offset)
if self.failure_audio:
play_sound(self.failure_audio)
if self.scroll_to_exceptions:
scroll_to_current_cell(self.scroll_to_exceptions)
def __init__(self, exec_lines=None):
self.cout = StringIO()
if exec_lines is None:
exec_lines = []
# Create config object for IPython
config = Config()
config.HistoryManager.hist_file = ':memory:'
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = 'NoColor'
# create a profile so instance history isn't saved
tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
profname = 'auto_profile_sphinx_build'
pdir = os.path.join(tmp_profile_dir, profname)
profile = ProfileDir.create_profile_dir(pdir)
# Create and initialize global ipython, but don't start its mainloop.
# This will persist across different EmbededSphinxShell instances.
IP = InteractiveShell.instance(config=config, profile_dir=profile)
atexit.register(self.cleanup)
sys.stdout = self.cout
sys.stderr = self.cout
# For debugging, so we can see normal output, use this:
#from IPython.utils.io import Tee
#sys.stdout = Tee(self.cout, channel='stdout') # dbg
#sys.stderr = Tee(self.cout, channel='stderr') # dbg
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ''
self.output = ''
self.tmp_profile_dir = tmp_profile_dir
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# Optionally, provide more detailed information to shell.
# this is assigned by the SetUp method of IPythonDirective
# to point at itself.
#
# So, you can access handy things at self.directive.state
self.directive = None
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
# Prepopulate the namespace.
for line in exec_lines:
self.process_input_line(line, store_history=False)
def __init__(self, exec_lines=None):
self.cout = StringIO()
if exec_lines is None:
exec_lines = []
# Create config object for IPython
config = Config()
config.HistoryManager.hist_file = ":memory:"
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = "NoColor"
# create a profile so instance history isn't saved
tmp_profile_dir = tempfile.mkdtemp(prefix="profile_")
profname = "auto_profile_sphinx_build"
pdir = os.path.join(tmp_profile_dir, profname)
profile = ProfileDir.create_profile_dir(pdir)
# Create and initialize global ipython, but don't start its mainloop.
# This will persist across different EmbededSphinxShell instances.
IP = InteractiveShell.instance(config=config, profile_dir=profile)
atexit.register(self.cleanup)
sys.stdout = self.cout
sys.stderr = self.cout
# For debugging, so we can see normal output, use this:
# from IPython.utils.io import Tee
# sys.stdout = Tee(self.cout, channel='stdout') # dbg
# sys.stderr = Tee(self.cout, channel='stderr') # dbg
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ""
self.output = ""
self.tmp_profile_dir = tmp_profile_dir
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# Optionally, provide more detailed information to shell.
# this is assigned by the SetUp method of IPythonDirective
# to point at itself.
#
# So, you can access handy things at self.directive.state
self.directive = None
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
# Prepopulate the namespace.
for line in exec_lines:
self.process_input_line(line, store_history=False)
def spark_magic(self, option=""):
ip = InteractiveShell.instance()
ip.register_magics(SparkexMagicsForTests)
code = " from pyspark.sql import SparkSession" \
" \n SparkSession.builder.appName('abc')"
result = ip.run_cell_magic("spark", option, code)
return result
def __init__(self, exec_lines=None, state=None):
self.cout = DecodingStringIO("")
if exec_lines is None:
exec_lines = []
self.state = state
# Create config object for IPython
config = Config()
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = "NoColor"
# create a profile so instance history isn't saved
tmp_profile_dir = tempfile.mkdtemp(prefix="profile_")
profname = "auto_profile_sphinx_build"
pdir = os.path.join(tmp_profile_dir, profname)
profile = ProfileDir.create_profile_dir(pdir)
# Create and initialize global ipython, but don't start its mainloop.
# This will persist across different EmbededSphinxShell instances.
IP = InteractiveShell.instance(config=config, profile_dir=profile)
# io.stdout redirect must be done after instantiating InteractiveShell
io.stdout = self.cout
io.stderr = self.cout
# For debugging, so we can see normal output, use this:
# from IPython.utils.io import Tee
# io.stdout = Tee(self.cout, channel='stdout') # dbg
# io.stderr = Tee(self.cout, channel='stderr') # dbg
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ""
self.output = ""
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# Optionally, provide more detailed information to shell.
self.directive = None
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
# Prepopulate the namespace.
for line in exec_lines:
self.process_input_line(line, store_history=False)
def __init__(self, exec_lines=None,state=None):
self.cout = DecodingStringIO(u'')
if exec_lines is None:
exec_lines = []
self.state = state
# Create config object for IPython
config = Config()
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = 'NoColor'
# create a profile so instance history isn't saved
tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
profname = 'auto_profile_sphinx_build'
pdir = os.path.join(tmp_profile_dir,profname)
profile = ProfileDir.create_profile_dir(pdir)
# Create and initialize global ipython, but don't start its mainloop.
# This will persist across different EmbededSphinxShell instances.
IP = InteractiveShell.instance(config=config, profile_dir=profile)
# io.stdout redirect must be done after instantiating InteractiveShell
io.stdout = self.cout
io.stderr = self.cout
# For debugging, so we can see normal output, use this:
#from IPython.utils.io import Tee
#io.stdout = Tee(self.cout, channel='stdout') # dbg
#io.stderr = Tee(self.cout, channel='stderr') # dbg
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ''
self.output = ''
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# Optionally, provide more detailed information to shell.
self.directive = None
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
# Prepopulate the namespace.
for line in exec_lines:
self.process_input_line(line, store_history=False)
def __init__(self):
# self.cout = cStringIO.StringIO()
self.cout = io.StringIO()
# Create config object for IPython
config = Config()
config.Global.display_banner = False
config.Global.exec_lines = [
'import numpy as np', 'from pylab import *'
]
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = 'NoColor'
# create a profile so instance history isn't saved
tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
profname = 'auto_profile_sphinx_build'
pdir = os.path.join(tmp_profile_dir, profname)
profile = ProfileDir.create_profile_dir(pdir)
# Create and initialize ipython, but don't start its mainloop
IP = InteractiveShell.instance(config=config, profile_dir=profile)
# io.stdout redirect must be done *after* instantiating InteractiveShell
#io.stdout = self.cout
sys.stdout = self.cout
#io.stderr = self.cout
io.stderr = self.cout
# For debugging, so we can see normal output, use this:
#from IPython.utils.io import Tee
#io.stdout = Tee(self.cout, channel='stdout') # dbg
#io.stderr = Tee(self.cout, channel='stderr') # dbg
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ''
self.output = ''
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
def __init__(self):
self.cout = cStringIO.StringIO()
# Create config object for IPython
config = Config()
config.Global.display_banner = False
config.Global.exec_lines = ['import numpy as np',
'from pylab import *'
]
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = 'NoColor'
config.InteractiveShell.cache_size = 0
# create a profile so instance history isn't saved
tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
profname = 'auto_profile_sphinx_build'
pdir = os.path.join(tmp_profile_dir, profname)
profile = ProfileDir.create_profile_dir(pdir)
# Create and initialize ipython, but don't start its mainloop
IP = InteractiveShell.instance(config=config, profile_dir=profile)
# io.stdout redirect must be done *after* instantiating
# InteractiveShell
io.stdout = self.cout
io.stderr = self.cout
# For debugging, so we can see normal output, use this:
# from IPython.utils.io import Tee
# io.stdout = Tee(self.cout, channel='stdout') # dbg
# io.stderr = Tee(self.cout, channel='stderr') # dbg
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ''
self.output = ''
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
def __install():
log = logging.getLogger('tpython')
log.info('setting up twisted reactor in ipython loop')
from twisted.internet import _threadedselect
_threadedselect.install()
from twisted.internet import reactor
from collections import deque
from IPython.lib import inputhook
from IPython import InteractiveShell
q = deque()
def reactor_wake(twisted_loop_next, q=q):
q.append(twisted_loop_next)
def reactor_work(*_args):
if q:
while len(q):
q.popleft()()
return 0
def reactor_start(*_args):
log.info('starting twisted reactor in ipython')
reactor.interleave(reactor_wake) # @UndefinedVariable
inputhook.set_inputhook(reactor_work)
def reactor_stop():
if reactor.threadpool: # @UndefinedVariable
log.info('stopping twisted threads')
reactor.threadpool.stop() # @UndefinedVariable
log.info('shutting down twisted reactor')
reactor._mainLoopShutdown() # @UndefinedVariable
ip = InteractiveShell.instance()
ask_exit = ip.ask_exit
def ipython_exit():
reactor_stop()
return ask_exit()
ip.ask_exit = ipython_exit
reactor_start()
return reactor
def __init__(self):
self.cout = cStringIO.StringIO()
Term.cout = self.cout
Term.cerr = self.cout
# For debugging, so we can see normal output, use this:
# from IPython.utils.io import Tee
# Term.cout = Tee(self.cout, channel='stdout') # dbg
# Term.cerr = Tee(self.cout, channel='stderr') # dbg
# Create config object for IPython
config = Config()
config.Global.display_banner = False
config.Global.exec_lines = [
"import numpy as np", "from pylab import *"
]
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = "NoColor"
# Create and initialize ipython, but don't start its mainloop
IP = InteractiveShell.instance(config=config)
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ""
self.output = ""
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
# we need bookmark the current dir first so we can save
# relative to it
self.process_input_line("bookmark ipy_basedir")
self.cout.seek(0)
self.cout.truncate(0)
def __init__(self):
self.cout = io.StringIO()
Term.cout = self.cout
Term.cerr = self.cout
# For debugging, so we can see normal output, use this:
# from IPython.utils.io import Tee
#Term.cout = Tee(self.cout, channel='stdout') # dbg
#Term.cerr = Tee(self.cout, channel='stderr') # dbg
# Create config object for IPython
config = Config()
config.Global.display_banner = False
config.Global.exec_lines = ['import numpy as np',
'from pylab import *'
]
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = 'NoColor'
# Create and initialize ipython, but don't start its mainloop
IP = InteractiveShell.instance(config=config)
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ''
self.output = ''
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
# we need bookmark the current dir first so we can save
# relative to it
self.process_input_line('bookmark ipy_basedir')
self.cout.seek(0)
self.cout.truncate(0)
def write_n2(configuration_file_path: str,
n2_file_path: str = None,
overwrite: bool = False):
"""
Write the N2 diagram of the problem in file n2.html
:param configuration_file_path:
:param n2_file_path: if None, will default to `n2.html`
:param overwrite:
:return: path of generated file.
:raise FastFileExistsError: if overwrite==False and n2_file_path already exists
"""
if not n2_file_path:
n2_file_path = "n2.html"
n2_file_path = pth.abspath(n2_file_path)
if not overwrite and pth.exists(n2_file_path):
raise FastFileExistsError(
"N2-diagram file %s not written because it already exists. "
"Use overwrite=True to bypass." % n2_file_path,
n2_file_path,
)
make_parent_dir(n2_file_path)
conf = FASTOADProblemConfigurator(configuration_file_path)
conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
problem = conf.get_problem()
problem.setup()
problem.final_setup()
om.n2(problem, outfile=n2_file_path, show_browser=False)
if InteractiveShell.initialized():
clear_output()
_LOGGER.info("N2 diagram written in %s", pth.abspath(n2_file_path))
return n2_file_path
def list_modules(
source_path: Union[List[str], str] = None,
out: Union[IO, str] = None,
overwrite: bool = False,
verbose: bool = False,
force_text_output: bool = False,
):
"""
Writes list of available systems.
If source_path is given and if it defines paths where there are registered systems,
they will be listed too.
:param source_path: either a configuration file path, folder path, or list of folder path
:param out: the output stream or a path for the output file (None means sys.stdout)
:param overwrite: if True and out is a file path, the file will be written even if one already
exists
:param verbose: if True, shows detailed information for each system
if False, shows only identifier and path of each system
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:return: path of generated file, or None if no file was generated.
:raise FastFileExistsError: if `overwrite==False` and `out` is a file path and the file exists
"""
if out is None:
out = sys.stdout
if isinstance(source_path, str):
if pth.isfile(source_path):
conf = FASTOADProblemConfigurator(source_path)
conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
# As the problem has been configured,
# BundleLoader now knows additional registered systems
elif pth.isdir(source_path):
RegisterOpenMDAOSystem.explore_folder(source_path)
else:
raise FileNotFoundError("Could not find %s" % source_path)
elif isinstance(source_path, Iterable):
for folder_path in source_path:
if not pth.isdir(folder_path):
_LOGGER.warning("SKIPPED %s: folder does not exist.",
folder_path)
else:
RegisterOpenMDAOSystem.explore_folder(folder_path)
elif source_path is not None:
raise RuntimeError("Unexpected type for source_path")
if verbose:
cell_list = _get_detailed_system_list()
else:
cell_list = _get_simple_system_list()
if isinstance(out, str):
out = pth.abspath(out)
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
else:
if (out == sys.stdout and InteractiveShell.initialized()
and not force_text_output and not verbose):
display(HTML(tabulate(cell_list, tablefmt="html")))
return None
out_file = out
out_file.write(tabulate(cell_list, tablefmt="grid"))
out_file.write("\n")
if isinstance(out, str):
out_file.close()
_LOGGER.info("System list written in %s", out)
return out
return None
def list_variables(
configuration_file_path: str,
out: Union[IO, str] = None,
overwrite: bool = False,
force_text_output: bool = False,
tablefmt: str = "grid",
):
"""
Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.
List is generally written as text. It can be displayed as a scrollable table view if:
- function is used in an interactive IPython shell
- out == sys.stdout
- force_text_output == False
:param configuration_file_path:
:param out: the output stream or a path for the output file (None means sys.stdout)
:param overwrite: if True and out parameter is a file path, the file will be written even if one
already exists
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:param tablefmt: The formatting of the requested table. Options are the same as those available
to the tabulate package. See tabulate.tabulate_formats for a complete list.
:raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
exists
"""
if out is None:
out = sys.stdout
conf = FASTOADProblemConfigurator(configuration_file_path)
conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
problem = conf.get_problem()
problem.setup()
# Extracting inputs and outputs
variables = VariableList.from_problem(problem)
variables.sort(key=lambda var: var.name)
input_variables = VariableList([var for var in variables if var.is_input])
output_variables = VariableList([var for var in variables if not var.is_input])
for var in input_variables:
var.metadata["I/O"] = "IN"
for var in output_variables:
var.metadata["I/O"] = "OUT"
variables_df = (
(input_variables + output_variables)
.to_dataframe()[["name", "I/O", "desc"]]
.rename(columns={"name": "NAME", "desc": "DESCRIPTION"})
)
if isinstance(out, str):
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
else:
if out == sys.stdout and InteractiveShell.initialized() and not force_text_output:
display(HTML(variables_df.to_html(index=False)))
return
# Here we continue with text output
out_file = out
# For a terminal output, we limit width of NAME column
variables_df["NAME"] = variables_df["NAME"].apply(lambda s: "\n".join(tw.wrap(s, 50)))
# In any case, let's break descriptions that are too long
variables_df["DESCRIPTION"] = variables_df["DESCRIPTION"].apply(
lambda s: "\n".join(tw.wrap(s, 100,))
)
out_file.write(
tabulate(variables_df, headers=variables_df.columns, showindex=False, tablefmt=tablefmt)
)
out_file.write("\n")
if isinstance(out, str):
out_file.close()
_LOGGER.info("Output list written in %s", out_file)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from IPython import InteractiveShell
inter = InteractiveShell()
inter.get_ipython().run_line_magic('matplotlib', 'inline')
pd.options.display.max_columns = 50
amzn = pd.read_csv('data/amzn_stock.csv',
index_col='Date',
parse_dates=['Date'])
print(amzn.head())
amzn_daily_return = amzn.Close.pct_change()
print(amzn_daily_return.head())
amzn_daily_return = amzn_daily_return.dropna()
print(amzn_daily_return.hist(bins=20))
mean = amzn_daily_return.mean()
std = amzn_daily_return.std()
abs_z_score = amzn_daily_return.sub(mean).abs().div(std)
pcts = [abs_z_score.lt(i).mean() for i in range(1, 4)]
print('{:.3f} fall within 1 standard deviation. '
'{:.3f} within 2 and {:.3f} within 3'.format(*pcts))
def test_return_normality(stock_data):
close = stock_data['Close']
daily_return = close.pct_change().dropna()
from mpl_toolkits.basemap import cm as bacm
except:
try:
from matplotlib.toolkits.basemap import cm as bacm
except:
print "Cannot load basemap colormaps (bacm)"
# Coping with the IPython API change from v0.10 to v0.11
try:
import IPython.ipapi # IPython 0.10 and earlier
ip = IPython.ipapi.get()
def_Magic = ip.expose_magic
IPY_LEGACY = True
except:
from IPython import InteractiveShell # IPython 0.11 and later
ip = InteractiveShell.instance()
def_Magic = ip.define_magic
IPY_LEGACY = False
def ip_Exec(self, cmd):
if IPY_LEGACY:
self.api.ex(cmd)
else:
ip.ex(cmd)
def grads_(self, arg=' '):
ip_Exec(self, "from grads import GrADS; ga=GrADS(%s)" % arg)
def _start_ipython():
"""Start a global IPython shell, which we need for IPython-specific syntax.
"""
def xsys(self, cmd):
"""Replace the default system call with a capturing one for doctest.
"""
# We use getoutput, but we need to strip it because pexpect captures
# the trailing newline differently from commands.getoutput
print(self.getoutput(cmd, split=False, depth=1).rstrip(), end="", file=sys.stdout)
sys.stdout.flush()
def _showtraceback(self, etype, evalue, stb):
"""Print the traceback purely on stdout for doctest to capture it.
"""
print(self.InteractiveTB.stb2text(stb), file=sys.stdout)
global get_ipython
# This function should only ever run once!
if hasattr(_start_ipython, "already_called"):
return
_start_ipython.already_called = True
# Store certain global objects that IPython modifies
_displayhook = sys.displayhook
_excepthook = sys.excepthook
_main = sys.modules.get("__main__")
# Create custom argv and namespaces for our IPython to be test-friendly
config = tools.default_config()
config.TerminalInteractiveShell.simple_prompt = True
# Create and initialize our test-friendly IPython instance.
shell = InteractiveShell.instance(config=config)
# A few more tweaks needed for playing nicely with doctests...
# remove history file
shell.tempfiles.append(config.HistoryManager.hist_file)
# These traps are normally only active for interactive use, set them
# permanently since we'll be mocking interactive sessions.
shell.builtin_trap.activate()
# Modify the IPython system call with one that uses getoutput, so that we
# can capture subcommands and print them to Python's stdout, otherwise the
# doctest machinery would miss them.
shell.system = types.MethodType(xsys, shell)
shell._showtraceback = types.MethodType(_showtraceback, shell)
# IPython is ready, now clean up some global state...
# Deactivate the various python system hooks added by ipython for
# interactive convenience so we don't confuse the doctest system
sys.modules["__main__"] = _main
sys.displayhook = _displayhook
sys.excepthook = _excepthook
# So that ipython magics and aliases can be doctested (they work by making
# a call into a global _ip object). Also make the top-level get_ipython
# now return this without recursively calling here again.
_ip = shell
get_ipython = _ip.get_ipython
builtin_mod._ip = _ip
builtin_mod.ip = _ip
builtin_mod.get_ipython = get_ipython
# Override paging, so we don't require user interaction during the tests.
def nopage(strng, start=0, screen_lines=0, pager_cmd=None):
if isinstance(strng, dict):
strng = strng.get("text/plain", "")
print(strng)
page.orig_page = page.pager_page
page.pager_page = nopage
return _ip
from pathlib import Path
from typing import Optional
import nbconvert
import nbformat
from IPython import InteractiveShell
from IPython.core.magics import CodeMagics
from black import format_str, FileMode
from lektor import build_programs
from lektor.db import Attachment
from lektor.pluginsystem import Plugin
from nbconvert.preprocessors import ExecutePreprocessor
log = logging.getLogger(__name__)
IPYTHON_SHELL = InteractiveShell()
_BLACKIFY = partial(format_str, mode=FileMode(line_length=79))
_already_built = set() # hack: prevent duplicate builds after clean
PLUGIN_KEY = "JUPYTER_PREPROCESS"
config = {
"url.source": None,
"metadata.blackify": True,
"metadata.execute": True,
# todo figure out how jupyter does these things and play together with it
"metadata.allow_errors": False,
"metadata.full_traceback": True,
"cell.source": "\n\n```{language}\n{cell.source}\n```",
# TODO figure out, why node.data[text/plain] is correct (no quotes around key!1?!?)
"node.execute_result": "```text\n[result]\n{node.data[text/plain]}\n```",
"node.stream": "```text\n[{node.name}]\n{node.text}\n```",
def load_ipython_extension(ipython: InteractiveShell) -> None:
ipython.register_magics(_PexEnvironmentBootstrapper)
import pandas as pd
import numpy as np
from IPython import InteractiveShell
inter = InteractiveShell()
college = pd.read_csv('data/college.csv', index_col='INSTNM')
cn = 'Texas A & M University-College Station'
print(college.loc[cn, 'UGDS_WHITE'])
print(college.at[cn, 'UGDS_WHITE'])
print(inter.get_ipython().run_line_magic('timeit', "college.loc[cn, 'UGDS_WHITE']"))
print(inter.get_ipython().run_line_magic('timeit', "college.at[cn, 'UGDS_WHITE']"))
row_num = college.index.get_loc(cn)
col_num = college.columns.get_loc('UGDS_WHITE')
print(row_num, col_num)
print(inter.get_ipython().run_line_magic('timeit', 'college.iloc[row_num, col_num]'))
print(inter.get_ipython().run_line_magic('timeit', 'college.iat[row_num, col_num]'))
print(inter.get_ipython().run_line_magic('timeit', 'college.iloc[5, col_num]'))
print(inter.get_ipython().run_line_magic('timeit', 'college.iat[5, col_num]'))
state = college['STABBR']
print(state.iat[1000])
print(state.at['Stanford University'])
from IPython import get_ipython
from IPython import InteractiveShell
shell = InteractiveShell.instance()
ip = get_ipython()
ip.run_cell("%time None")
# CPU times: user 2 µs, sys: 0 ns, total: 2 µs
# Wall time: 3.58 µs
from altair.vega.v5 import Vega
DATA_RECORDS = [
{"amount": 28, "category": "A"},
{"amount": 55, "category": "B"},
{"amount": 43, "category": "C"},
{"amount": 91, "category": "D"},
{"amount": 81, "category": "E"},
{"amount": 53, "category": "F"},
{"amount": 19, "category": "G"},
{"amount": 87, "category": "H"},
]
if IPYTHON_AVAILABLE:
_ipshell = InteractiveShell.instance()
_ipshell.run_cell("%load_ext altair")
_ipshell.run_cell(
"""
import pandas as pd
table = pd.DataFrame.from_records({})
the_data = table
""".format(
DATA_RECORDS
)
)
VEGA_SPEC = {
"$schema": "https://vega.github.io/schema/vega/v5.json",
"axes": [
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from IPython import InteractiveShell
inter = InteractiveShell()
pd.options.display.max_columns = 50
movie = pd.read_csv('data/movie.csv', index_col='movie_title')
c1 = movie['title_year'] >= 2010
c2 = movie['title_year'].isnull()
criteria = c1 | c2
print(movie.mask(criteria).head())
movie_mask = movie.mask(criteria).dropna(how='all')
print(movie_mask.head())
movie_boolean = movie[movie['title_year'] < 2010]
print(movie_boolean.head())
print(movie_mask.equals(movie_boolean))
print(movie_mask.shape == movie_boolean.shape)
print(movie_mask.dtypes == movie_boolean.dtypes)
from pandas.testing import assert_frame_equal
assert_frame_equal(movie_boolean, movie_mask, check_dtype=False)
print(inter.get_ipython().run_line_magic(
'timeit', "movie.mask(criteria).dropna(how='all')"))
print(inter.get_ipython().run_line_magic('timeit',
"movie[movie['title_year'] < 2010]"))
import pandas as pd
import numpy as np
from IPython import InteractiveShell
inter = InteractiveShell()
pd.options.display.max_columns = 50
college = pd.read_csv('data/college.csv')
college2 = college.set_index('STABBR')
print(college2.index.is_monotonic)
college3 = college2.sort_index()
print(college3.index.is_monotonic)
print(inter.get_ipython().run_line_magic('timeit',
"college[college['STABBR'] == 'TX']"))
print(inter.get_ipython().run_line_magic('timeit', "college2.loc['TX']"))
print(inter.get_ipython().run_line_magic('timeit', "college3.loc['TX']"))
college_unique = college.set_index('INSTNM')
print(college_unique.index.is_unique)
college[college['INSTNM'] == 'Stanford University']
print(college_unique.loc['Stanford University'])
print(inter.get_ipython().run_line_magic(
'timeit', "college[college['INSTNM'] == 'Stanford University']"))
print(inter.get_ipython().run_line_magic(
'timeit', "college_unique.loc['Stanford University']"))
college.index = college['CITY'] + ', ' + college['STABBR']
college = college.sort_index()
print(college.head())
class JupyterShell:
"""Helper class for managing shell interpreters."""
_instance = None
def __new__(cls, *__, **___):
raise RuntimeError(
f'Use static get_instance method in instantiate {cls}.')
def __init__(self, user_ns=None):
self.shell = None
self.user_ns = user_ns or {}
self._tmpdir = self.tmpdir = None
self.stdout = STDOUT
self.stderr = STDERR
def update_user_ns(self, **kwargs):
"""Add a varaible to the user namespace."""
self.user_ns.update(**kwargs)
@staticmethod
def get_instance():
if JupyterShell._instance is None:
obj = object.__new__(JupyterShell)
obj.__init__()
obj.new()
JupyterShell._instance = obj
return JupyterShell._instance
def new(self):
"""Create a new shell."""
self._tmpdir = tempfile.TemporaryDirectory()
self.tmpdir = os.path.realpath(self._tmpdir.name)
self.update_user_ns(Shell=self)
self.shell = InteractiveShell(user_ns=self.user_ns)
def reset(self):
"""Reset the shell."""
if self._tmpdir:
self._tmpdir.cleanup()
self._tmpdir = self.tmpdir = None
self.shell.reset()
self.user_ns = dict(Shell=self)
self.new()
def cleanup(self):
if self._tmpdir:
self._tmpdir.cleanup()
self._tmpdir = self.tmpdir = None
def get_cell_count(self):
"""Return the last cell number."""
try:
return max(self.user_ns.get('Out').keys())
except (AttributeError, ValueError):
return None
def run_cell(self, code, silent=False):
"""Run a cell."""
if not self.shell:
self.new()
with closing(StringIO()) as _stdout, closing(StringIO()) as _stderr:
with redirect_stdout(_stdout), redirect_stderr(_stderr):
result = self.shell.run_cell(f'{code}\n',
store_history=not silent,
silent=silent)
stdout = _stdout.getvalue() # pylint: disable=no-member
stderr = _stdout.getvalue() # pylint: disable=no-member
if result is not None:
pattern = r'^Out\s*\[{}\]'.format(self.get_cell_count())
stdout = '\n'.join(pop_until_match(stdout.split('\n'), pattern))
return result, stdout, stderr
@contextmanager
def chdir(self, newdir=None): # pylint: disable=invalid-name, no-self-argument
"""Change working directory."""
newdir = newdir or self.tmpdir
curdir = os.getcwd()
if newdir != curdir:
os.chdir(os.path.realpath(os.path.expanduser(newdir)))
try:
yield
finally:
if newdir != curdir:
os.chdir(curdir)
@staticmethod
def copy(source, destination):
"""Copy a file."""
if isinstance(source, (list, tuple)):
source = os.path.join(*source)
if isinstance(destination, (list, tuple)):
destination = os.path.join(*destination)
shutil.copy(source, destination)
def list_variables(
configuration_file_path: str,
out: Union[IO, str] = sys.stdout,
overwrite: bool = False,
force_text_output: bool = False,
):
"""
Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.
List is generally written as text. It can be displayed as a scrollable table view if:
- function is used in an interactive IPython shell
- out == sys.stdout
- force_text_output == False
:param configuration_file_path:
:param out: the output stream or a path for the output file
:param overwrite: if True and out parameter is a file path, the file will be written even if one
already exists
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
exists
"""
conf = FASTOADProblemConfigurator(configuration_file_path)
problem = conf.get_problem()
problem.setup()
# Extracting inputs and outputs
variables = VariableList.from_problem(problem, get_promoted_names=False)
variables.sort(key=lambda var: var.name)
input_variables = VariableList([var for var in variables if var.is_input])
output_variables = VariableList(
[var for var in variables if not var.is_input])
if isinstance(out, str):
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
table_width = MAX_TABLE_WIDTH
else:
if out == sys.stdout and InteractiveShell.initialized(
) and not force_text_output:
# Here we display the variable list as VariableViewer in a notebook
for var in input_variables:
var.metadata["I/O"] = "IN"
for var in output_variables:
var.metadata["I/O"] = "OUT"
df = ((input_variables + output_variables).to_dataframe()[[
"I/O", "name", "desc"
]].rename(columns={
"name": "Name",
"desc": "Description"
}))
display(HTML(df.to_html()))
return
# Here we continue with text output
out_file = out
table_width = min(get_terminal_size().columns, MAX_TABLE_WIDTH) - 1
pd.set_option("display.max_colwidth", 1000)
max_name_length = np.max([
len(name)
for name in input_variables.names() + output_variables.names()
])
description_text_width = table_width - max_name_length - 2
def _write_variables(out_f, variables):
"""Writes variables and their description as a pandas DataFrame"""
df = variables.to_dataframe()
# Create a new Series where description are wrapped on several lines if needed.
# Each line becomes an element of the Series
df["desc"] = [
"\n".join(tw.wrap(s, description_text_width)) for s in df["desc"]
]
new_desc = df.desc.str.split("\n", expand=True).stack()
# Create a Series for name that will match new_desc Series. Variable name will be in front of
# first line of description. An empty string will be in front of other lines.
new_name = [
df.name.loc[i] if j == 0 else "" for i, j in new_desc.index
]
# Create the DataFrame that will be displayed
new_df = pd.DataFrame({"NAME": new_name, "DESCRIPTION": new_desc})
out_f.write(
new_df.to_string(
index=False,
columns=["NAME", "DESCRIPTION"],
justify="center",
formatters={ # Formatters are needed for enforcing left justification
"NAME": ("{:%s}" % max_name_length).format,
"DESCRIPTION": ("{:%s}" % description_text_width).format,
},
)
)
out_file.write("\n")
def _write_text_with_line(txt: str, line_length: int):
""" Writes a line of given length with provided text inside """
out_file.write("-" + txt + "-" * (line_length - 1 - len(txt)) + "\n")
# Inputs
_write_text_with_line(" INPUTS OF THE PROBLEM ", table_width)
_write_variables(out_file, input_variables)
# Outputs
out_file.write("\n")
_write_text_with_line(" OUTPUTS OF THE PROBLEM ", table_width)
_write_variables(out_file, output_variables)
_write_text_with_line("", table_width)
if isinstance(out, str):
out_file.close()
_LOGGER.info("Output list written in %s", out_file)
def new(self):
"""Create a new shell."""
self._tmpdir = tempfile.TemporaryDirectory()
self.tmpdir = os.path.realpath(self._tmpdir.name)
self.update_user_ns(Shell=self)
self.shell = InteractiveShell(user_ns=self.user_ns)
def list_modules(
configuration_file_path: str = None,
out: Union[IO, str] = None,
overwrite: bool = False,
verbose: bool = False,
force_text_output: bool = False,
):
"""
Writes list of available systems.
If configuration_file_path is given and if it defines paths where there are registered systems,
they will be listed too.
:param configuration_file_path:
:param out: the output stream or a path for the output file (None means sys.stdout)
:param overwrite: if True and out is a file path, the file will be written even if one already
exists
:param verbose: if True, shows detailed information for each system
if False, shows only identifier and path of each system
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:raise FastFileExistsError: if overwrite==False and out is a file path and the file exists
"""
if out is None:
out = sys.stdout
if configuration_file_path:
conf = FASTOADProblemConfigurator(configuration_file_path)
conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
conf.load(configuration_file_path)
# As the problem has been configured, BundleLoader now knows additional registered systems
if verbose:
cell_list = _get_detailed_system_list()
else:
cell_list = _get_simple_system_list()
if isinstance(out, str):
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
else:
if (
out == sys.stdout
and InteractiveShell.initialized()
and not force_text_output
and not verbose
):
display(HTML(tabulate(cell_list, tablefmt="html")))
return
out_file = out
out_file.write(tabulate(cell_list, tablefmt="grid"))
out_file.write("\n")
if isinstance(out, str):
out_file.close()
_LOGGER.info("System list written in %s", out_file)
import pandas as pd
import numpy as np
from IPython import InteractiveShell
inter = InteractiveShell()
pd.options.display.max_columns = 50
college = pd.read_csv('data/college.csv')
print(college[college['STABBR'] == 'TX'].head())
college2 = college.set_index('STABBR')
print(college2.loc['TX'].head())
print(inter.get_ipython().run_line_magic('timeit',
"college[college['STABBR'] == 'TX']"))
print(inter.get_ipython().run_line_magic('timeit', "college2.loc['TX']"))
print(inter.get_ipython().run_line_magic(
'timeit', "college2 = college.set_index('STABBR')"))
states = ['TX', 'CA', 'NY']
print(college[college['STABBR'].isin(states)])
print(college2.loc[states].head())
