def _exception_handler(self,
                           i_shell: InteractiveShell,
                           etype,
                           value,
                           tb: TracebackType,
                           tb_offset=None):
        i_shell.showtraceback((etype, value, tb), tb_offset=tb_offset)

        self._notify_of_exception(i_shell, etype, value, tb, tb_offset)
        if self.failure_audio:
            play_sound(self.failure_audio)
        if self.scroll_to_exceptions:
            scroll_to_current_cell(self.scroll_to_exceptions)
    def __init__(self, exec_lines=None):

        self.cout = StringIO()

        if exec_lines is None:
            exec_lines = []

        # Create config object for IPython
        config = Config()
        config.HistoryManager.hist_file = ':memory:'
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = 'NoColor'

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
        profname = 'auto_profile_sphinx_build'
        pdir = os.path.join(tmp_profile_dir, profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize global ipython, but don't start its mainloop.
        # This will persist across different EmbededSphinxShell instances.
        IP = InteractiveShell.instance(config=config, profile_dir=profile)
        atexit.register(self.cleanup)

        sys.stdout = self.cout
        sys.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        #from IPython.utils.io import Tee
        #sys.stdout = Tee(self.cout, channel='stdout') # dbg
        #sys.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ''
        self.output = ''
        self.tmp_profile_dir = tmp_profile_dir

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # Optionally, provide more detailed information to shell.
        # this is assigned by the SetUp method of IPythonDirective
        # to point at itself.
        #
        # So, you can access handy things at self.directive.state
        self.directive = None

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # Prepopulate the namespace.
        for line in exec_lines:
            self.process_input_line(line, store_history=False)
    def __init__(self, exec_lines=None):

        self.cout = StringIO()

        if exec_lines is None:
            exec_lines = []

        # Create config object for IPython
        config = Config()
        config.HistoryManager.hist_file = ":memory:"
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = "NoColor"

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix="profile_")
        profname = "auto_profile_sphinx_build"
        pdir = os.path.join(tmp_profile_dir, profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize global ipython, but don't start its mainloop.
        # This will persist across different EmbededSphinxShell instances.
        IP = InteractiveShell.instance(config=config, profile_dir=profile)
        atexit.register(self.cleanup)

        sys.stdout = self.cout
        sys.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        # from IPython.utils.io import Tee
        # sys.stdout = Tee(self.cout, channel='stdout') # dbg
        # sys.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ""
        self.output = ""
        self.tmp_profile_dir = tmp_profile_dir

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # Optionally, provide more detailed information to shell.
        # this is assigned by the SetUp method of IPythonDirective
        # to point at itself.
        #
        # So, you can access handy things at self.directive.state
        self.directive = None

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # Prepopulate the namespace.
        for line in exec_lines:
            self.process_input_line(line, store_history=False)
 def spark_magic(self, option=""):
     ip = InteractiveShell.instance()
     ip.register_magics(SparkexMagicsForTests)
     code = " from pyspark.sql import SparkSession" \
            " \n SparkSession.builder.appName('abc')"
     result = ip.run_cell_magic("spark", option, code)
     return result
Exemple #5
0
    def __init__(self, exec_lines=None, state=None):

        self.cout = DecodingStringIO("")

        if exec_lines is None:
            exec_lines = []

        self.state = state

        # Create config object for IPython
        config = Config()
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = "NoColor"

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix="profile_")
        profname = "auto_profile_sphinx_build"
        pdir = os.path.join(tmp_profile_dir, profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize global ipython, but don't start its mainloop.
        # This will persist across different EmbededSphinxShell instances.
        IP = InteractiveShell.instance(config=config, profile_dir=profile)

        # io.stdout redirect must be done after instantiating InteractiveShell
        io.stdout = self.cout
        io.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        # from IPython.utils.io import Tee
        # io.stdout = Tee(self.cout, channel='stdout') # dbg
        # io.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ""
        self.output = ""

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # Optionally, provide more detailed information to shell.
        self.directive = None

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # Prepopulate the namespace.
        for line in exec_lines:
            self.process_input_line(line, store_history=False)
    def __init__(self, exec_lines=None,state=None):

        self.cout = DecodingStringIO(u'')

        if exec_lines is None:
            exec_lines = []

        self.state = state

        # Create config object for IPython
        config = Config()
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = 'NoColor'

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
        profname = 'auto_profile_sphinx_build'
        pdir = os.path.join(tmp_profile_dir,profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize global ipython, but don't start its mainloop.
        # This will persist across different EmbededSphinxShell instances.
        IP = InteractiveShell.instance(config=config, profile_dir=profile)

        # io.stdout redirect must be done after instantiating InteractiveShell
        io.stdout = self.cout
        io.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        #from IPython.utils.io import Tee
        #io.stdout = Tee(self.cout, channel='stdout') # dbg
        #io.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ''
        self.output = ''

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # Optionally, provide more detailed information to shell.
        self.directive = None

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # Prepopulate the namespace.
        for line in exec_lines:
            self.process_input_line(line, store_history=False)
Exemple #7
0
    def __init__(self):

        # self.cout = cStringIO.StringIO()
        self.cout = io.StringIO()

        # Create config object for IPython
        config = Config()
        config.Global.display_banner = False
        config.Global.exec_lines = [
            'import numpy as np', 'from pylab import *'
        ]
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = 'NoColor'

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
        profname = 'auto_profile_sphinx_build'
        pdir = os.path.join(tmp_profile_dir, profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize ipython, but don't start its mainloop
        IP = InteractiveShell.instance(config=config, profile_dir=profile)

        # io.stdout redirect must be done *after* instantiating InteractiveShell
        #io.stdout = self.cout
        sys.stdout = self.cout
        #io.stderr = self.cout
        io.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        #from IPython.utils.io import Tee
        #io.stdout = Tee(self.cout, channel='stdout') # dbg
        #io.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ''
        self.output = ''

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False
Exemple #8
0
    def __init__(self):

        self.cout = cStringIO.StringIO()

        # Create config object for IPython
        config = Config()
        config.Global.display_banner = False
        config.Global.exec_lines = ['import numpy as np',
                                    'from pylab import *'
                                    ]
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = 'NoColor'
        config.InteractiveShell.cache_size = 0

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
        profname = 'auto_profile_sphinx_build'
        pdir = os.path.join(tmp_profile_dir, profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize ipython, but don't start its mainloop
        IP = InteractiveShell.instance(config=config, profile_dir=profile)

        # io.stdout redirect must be done *after* instantiating
        # InteractiveShell
        io.stdout = self.cout
        io.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        # from IPython.utils.io import Tee
        # io.stdout = Tee(self.cout, channel='stdout') # dbg
        # io.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ''
        self.output = ''

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False
Exemple #9
0
def __install():
    log = logging.getLogger('tpython')
    log.info('setting up twisted reactor in ipython loop')
    
    from twisted.internet import _threadedselect
    _threadedselect.install()
    
    from twisted.internet import reactor
    from collections import deque
    from IPython.lib import inputhook
    from IPython import InteractiveShell
    
    q = deque()
    
    def reactor_wake(twisted_loop_next, q=q):
        q.append(twisted_loop_next)
    
    def reactor_work(*_args):
        if q:
            while len(q):
                q.popleft()()
        return 0
    
    def reactor_start(*_args):
        log.info('starting twisted reactor in ipython')
        reactor.interleave(reactor_wake)  # @UndefinedVariable
        inputhook.set_inputhook(reactor_work)
    
    def reactor_stop():
        if reactor.threadpool:  # @UndefinedVariable
            log.info('stopping twisted threads')
            reactor.threadpool.stop()  # @UndefinedVariable
        log.info('shutting down twisted reactor')
        reactor._mainLoopShutdown()  # @UndefinedVariable

    ip = InteractiveShell.instance()

    ask_exit = ip.ask_exit
    def ipython_exit():
        reactor_stop()
        return ask_exit()

    ip.ask_exit = ipython_exit

    reactor_start()

    return reactor
Exemple #10
0
    def __init__(self):

        self.cout = cStringIO.StringIO()
        Term.cout = self.cout
        Term.cerr = self.cout

        # For debugging, so we can see normal output, use this:
        # from IPython.utils.io import Tee
        # Term.cout = Tee(self.cout, channel='stdout') # dbg
        # Term.cerr = Tee(self.cout, channel='stderr') # dbg

        # Create config object for IPython
        config = Config()
        config.Global.display_banner = False
        config.Global.exec_lines = [
            "import numpy as np", "from pylab import *"
        ]
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = "NoColor"

        # Create and initialize ipython, but don't start its mainloop
        IP = InteractiveShell.instance(config=config)

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ""
        self.output = ""

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # we need bookmark the current dir first so we can save
        # relative to it
        self.process_input_line("bookmark ipy_basedir")
        self.cout.seek(0)
        self.cout.truncate(0)
    def __init__(self):

        self.cout = io.StringIO()
        Term.cout = self.cout
        Term.cerr = self.cout

        # For debugging, so we can see normal output, use this:
        # from IPython.utils.io import Tee
        #Term.cout = Tee(self.cout, channel='stdout') # dbg
        #Term.cerr = Tee(self.cout, channel='stderr') # dbg

        # Create config object for IPython
        config = Config()
        config.Global.display_banner = False
        config.Global.exec_lines = ['import numpy as np',
                                    'from pylab import *'
                                    ]
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = 'NoColor'

        # Create and initialize ipython, but don't start its mainloop
        IP = InteractiveShell.instance(config=config)

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns
                                    
        self.input = ''
        self.output = ''

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # we need bookmark the current dir first so we can save
        # relative to it
        self.process_input_line('bookmark ipy_basedir')
        self.cout.seek(0)
        self.cout.truncate(0)
Exemple #12
0
def write_n2(configuration_file_path: str,
             n2_file_path: str = None,
             overwrite: bool = False):
    """
    Write the N2 diagram of the problem in file n2.html

    :param configuration_file_path:
    :param n2_file_path: if None, will default to `n2.html`
    :param overwrite:
    :return: path of generated file.
    :raise FastFileExistsError: if overwrite==False and n2_file_path already exists
    """

    if not n2_file_path:
        n2_file_path = "n2.html"
    n2_file_path = pth.abspath(n2_file_path)

    if not overwrite and pth.exists(n2_file_path):
        raise FastFileExistsError(
            "N2-diagram file %s not written because it already exists. "
            "Use overwrite=True to bypass." % n2_file_path,
            n2_file_path,
        )

    make_parent_dir(n2_file_path)
    conf = FASTOADProblemConfigurator(configuration_file_path)
    conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
    problem = conf.get_problem()
    problem.setup()
    problem.final_setup()

    om.n2(problem, outfile=n2_file_path, show_browser=False)
    if InteractiveShell.initialized():
        clear_output()
    _LOGGER.info("N2 diagram written in %s", pth.abspath(n2_file_path))
    return n2_file_path
Exemple #13
0
def list_modules(
    source_path: Union[List[str], str] = None,
    out: Union[IO, str] = None,
    overwrite: bool = False,
    verbose: bool = False,
    force_text_output: bool = False,
):
    """
    Writes list of available systems.
    If source_path is given and if it defines paths where there are registered systems,
    they will be listed too.

    :param source_path: either a configuration file path, folder path, or list of folder path
    :param out: the output stream or a path for the output file (None means sys.stdout)
    :param overwrite: if True and out is a file path, the file will be written even if one already
                      exists
    :param verbose: if True, shows detailed information for each system
                    if False, shows only identifier and path of each system
    :param force_text_output: if True, list will be written as text, even if command is used in an
                              interactive IPython shell (Jupyter notebook). Has no effect in other
                              shells or if out parameter is not sys.stdout
    :return: path of generated file, or None if no file was generated.
    :raise FastFileExistsError: if `overwrite==False` and `out` is a file path and the file exists
    """
    if out is None:
        out = sys.stdout

    if isinstance(source_path, str):
        if pth.isfile(source_path):
            conf = FASTOADProblemConfigurator(source_path)
            conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
            # As the problem has been configured,
            # BundleLoader now knows additional registered systems
        elif pth.isdir(source_path):
            RegisterOpenMDAOSystem.explore_folder(source_path)
        else:
            raise FileNotFoundError("Could not find %s" % source_path)
    elif isinstance(source_path, Iterable):
        for folder_path in source_path:
            if not pth.isdir(folder_path):
                _LOGGER.warning("SKIPPED %s: folder does not exist.",
                                folder_path)
            else:
                RegisterOpenMDAOSystem.explore_folder(folder_path)
    elif source_path is not None:
        raise RuntimeError("Unexpected type for source_path")

    if verbose:
        cell_list = _get_detailed_system_list()
    else:
        cell_list = _get_simple_system_list()

    if isinstance(out, str):
        out = pth.abspath(out)
        if not overwrite and pth.exists(out):
            raise FastFileExistsError(
                "File %s not written because it already exists. "
                "Use overwrite=True to bypass." % out,
                out,
            )

        make_parent_dir(out)
        out_file = open(out, "w")
    else:
        if (out == sys.stdout and InteractiveShell.initialized()
                and not force_text_output and not verbose):
            display(HTML(tabulate(cell_list, tablefmt="html")))
            return None

        out_file = out

    out_file.write(tabulate(cell_list, tablefmt="grid"))
    out_file.write("\n")

    if isinstance(out, str):
        out_file.close()
        _LOGGER.info("System list written in %s", out)
        return out

    return None
Exemple #14
0
def list_variables(
    configuration_file_path: str,
    out: Union[IO, str] = None,
    overwrite: bool = False,
    force_text_output: bool = False,
    tablefmt: str = "grid",
):
    """
    Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.

    List is generally written as text. It can be displayed as a scrollable table view if:
    - function is used in an interactive IPython shell
    - out == sys.stdout
    - force_text_output == False

    :param configuration_file_path:
    :param out: the output stream or a path for the output file (None means sys.stdout)
    :param overwrite: if True and out parameter is a file path, the file will be written even if one
                      already exists
    :param force_text_output: if True, list will be written as text, even if command is used in an
                              interactive IPython shell (Jupyter notebook). Has no effect in other
                              shells or if out parameter is not sys.stdout
    :param tablefmt: The formatting of the requested table. Options are the same as those available
                     to the tabulate package. See tabulate.tabulate_formats for a complete list.
    :raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
                                exists
    """
    if out is None:
        out = sys.stdout

    conf = FASTOADProblemConfigurator(configuration_file_path)
    conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
    problem = conf.get_problem()
    problem.setup()

    # Extracting inputs and outputs
    variables = VariableList.from_problem(problem)
    variables.sort(key=lambda var: var.name)
    input_variables = VariableList([var for var in variables if var.is_input])
    output_variables = VariableList([var for var in variables if not var.is_input])

    for var in input_variables:
        var.metadata["I/O"] = "IN"
    for var in output_variables:
        var.metadata["I/O"] = "OUT"

    variables_df = (
        (input_variables + output_variables)
        .to_dataframe()[["name", "I/O", "desc"]]
        .rename(columns={"name": "NAME", "desc": "DESCRIPTION"})
    )

    if isinstance(out, str):
        if not overwrite and pth.exists(out):
            raise FastFileExistsError(
                "File %s not written because it already exists. "
                "Use overwrite=True to bypass." % out,
                out,
            )
        make_parent_dir(out)
        out_file = open(out, "w")
    else:
        if out == sys.stdout and InteractiveShell.initialized() and not force_text_output:
            display(HTML(variables_df.to_html(index=False)))
            return

        # Here we continue with text output
        out_file = out

        # For a terminal output, we limit width of NAME column
        variables_df["NAME"] = variables_df["NAME"].apply(lambda s: "\n".join(tw.wrap(s, 50)))

    # In any case, let's break descriptions that are too long
    variables_df["DESCRIPTION"] = variables_df["DESCRIPTION"].apply(
        lambda s: "\n".join(tw.wrap(s, 100,))
    )

    out_file.write(
        tabulate(variables_df, headers=variables_df.columns, showindex=False, tablefmt=tablefmt)
    )
    out_file.write("\n")

    if isinstance(out, str):
        out_file.close()
        _LOGGER.info("Output list written in %s", out_file)
Exemple #15
0
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from IPython import InteractiveShell

inter = InteractiveShell()
inter.get_ipython().run_line_magic('matplotlib', 'inline')
pd.options.display.max_columns = 50

amzn = pd.read_csv('data/amzn_stock.csv',
                   index_col='Date',
                   parse_dates=['Date'])
print(amzn.head())

amzn_daily_return = amzn.Close.pct_change()
print(amzn_daily_return.head())

amzn_daily_return = amzn_daily_return.dropna()
print(amzn_daily_return.hist(bins=20))

mean = amzn_daily_return.mean()
std = amzn_daily_return.std()
abs_z_score = amzn_daily_return.sub(mean).abs().div(std)
pcts = [abs_z_score.lt(i).mean() for i in range(1, 4)]
print('{:.3f} fall within 1 standard deviation. '
      '{:.3f} within 2 and {:.3f} within 3'.format(*pcts))


def test_return_normality(stock_data):
    close = stock_data['Close']
    daily_return = close.pct_change().dropna()
Exemple #16
0
        from mpl_toolkits.basemap import cm as bacm
    except:
        try:
            from matplotlib.toolkits.basemap import cm as bacm
        except:
            print "Cannot load basemap colormaps (bacm)"

# Coping with the IPython API change from v0.10 to v0.11
try:
    import IPython.ipapi  # IPython 0.10 and earlier
    ip = IPython.ipapi.get()
    def_Magic = ip.expose_magic
    IPY_LEGACY = True
except:
    from IPython import InteractiveShell  # IPython 0.11 and later
    ip = InteractiveShell.instance()
    def_Magic = ip.define_magic
    IPY_LEGACY = False


def ip_Exec(self, cmd):
    if IPY_LEGACY:
        self.api.ex(cmd)
    else:
        ip.ex(cmd)


def grads_(self, arg=' '):
    ip_Exec(self, "from grads import GrADS; ga=GrADS(%s)" % arg)

def _start_ipython():
    """Start a global IPython shell, which we need for IPython-specific syntax.
    """

    def xsys(self, cmd):
        """Replace the default system call with a capturing one for doctest.
        """
        # We use getoutput, but we need to strip it because pexpect captures
        # the trailing newline differently from commands.getoutput
        print(self.getoutput(cmd, split=False, depth=1).rstrip(), end="", file=sys.stdout)
        sys.stdout.flush()

    def _showtraceback(self, etype, evalue, stb):
        """Print the traceback purely on stdout for doctest to capture it.
        """
        print(self.InteractiveTB.stb2text(stb), file=sys.stdout)

    global get_ipython

    # This function should only ever run once!
    if hasattr(_start_ipython, "already_called"):
        return
    _start_ipython.already_called = True

    # Store certain global objects that IPython modifies
    _displayhook = sys.displayhook
    _excepthook = sys.excepthook
    _main = sys.modules.get("__main__")

    # Create custom argv and namespaces for our IPython to be test-friendly
    config = tools.default_config()
    config.TerminalInteractiveShell.simple_prompt = True

    # Create and initialize our test-friendly IPython instance.
    shell = InteractiveShell.instance(config=config)

    # A few more tweaks needed for playing nicely with doctests...

    # remove history file
    shell.tempfiles.append(config.HistoryManager.hist_file)

    # These traps are normally only active for interactive use, set them
    # permanently since we'll be mocking interactive sessions.
    shell.builtin_trap.activate()

    # Modify the IPython system call with one that uses getoutput, so that we
    # can capture subcommands and print them to Python's stdout, otherwise the
    # doctest machinery would miss them.
    shell.system = types.MethodType(xsys, shell)

    shell._showtraceback = types.MethodType(_showtraceback, shell)

    # IPython is ready, now clean up some global state...

    # Deactivate the various python system hooks added by ipython for
    # interactive convenience so we don't confuse the doctest system
    sys.modules["__main__"] = _main
    sys.displayhook = _displayhook
    sys.excepthook = _excepthook

    # So that ipython magics and aliases can be doctested (they work by making
    # a call into a global _ip object).  Also make the top-level get_ipython
    # now return this without recursively calling here again.
    _ip = shell
    get_ipython = _ip.get_ipython
    builtin_mod._ip = _ip
    builtin_mod.ip = _ip
    builtin_mod.get_ipython = get_ipython

    # Override paging, so we don't require user interaction during the tests.
    def nopage(strng, start=0, screen_lines=0, pager_cmd=None):
        if isinstance(strng, dict):
            strng = strng.get("text/plain", "")
        print(strng)

    page.orig_page = page.pager_page
    page.pager_page = nopage

    return _ip
from pathlib import Path
from typing import Optional

import nbconvert
import nbformat
from IPython import InteractiveShell
from IPython.core.magics import CodeMagics
from black import format_str, FileMode
from lektor import build_programs
from lektor.db import Attachment
from lektor.pluginsystem import Plugin
from nbconvert.preprocessors import ExecutePreprocessor

log = logging.getLogger(__name__)

IPYTHON_SHELL = InteractiveShell()
_BLACKIFY = partial(format_str, mode=FileMode(line_length=79))
_already_built = set()  # hack: prevent duplicate builds after clean

PLUGIN_KEY = "JUPYTER_PREPROCESS"
config = {
    "url.source": None,
    "metadata.blackify": True,
    "metadata.execute": True,
    # todo figure out how jupyter does these things and play together with it
    "metadata.allow_errors": False,
    "metadata.full_traceback": True,
    "cell.source": "\n\n```{language}\n{cell.source}\n```",
    # TODO figure out, why node.data[text/plain] is correct (no quotes around key!1?!?)
    "node.execute_result": "```text\n[result]\n{node.data[text/plain]}\n```",
    "node.stream": "```text\n[{node.name}]\n{node.text}\n```",
def load_ipython_extension(ipython: InteractiveShell) -> None:
    ipython.register_magics(_PexEnvironmentBootstrapper)
Exemple #20
0
import pandas as pd
import numpy as np
from IPython import InteractiveShell

inter = InteractiveShell()

college = pd.read_csv('data/college.csv', index_col='INSTNM')
cn = 'Texas A & M University-College Station'
print(college.loc[cn, 'UGDS_WHITE'])
print(college.at[cn, 'UGDS_WHITE'])
print(inter.get_ipython().run_line_magic('timeit', "college.loc[cn, 'UGDS_WHITE']"))
print(inter.get_ipython().run_line_magic('timeit', "college.at[cn, 'UGDS_WHITE']"))

row_num = college.index.get_loc(cn)
col_num = college.columns.get_loc('UGDS_WHITE')
print(row_num, col_num)

print(inter.get_ipython().run_line_magic('timeit', 'college.iloc[row_num, col_num]'))
print(inter.get_ipython().run_line_magic('timeit', 'college.iat[row_num, col_num]'))
print(inter.get_ipython().run_line_magic('timeit', 'college.iloc[5, col_num]'))
print(inter.get_ipython().run_line_magic('timeit', 'college.iat[5, col_num]'))

state = college['STABBR']
print(state.iat[1000])
print(state.at['Stanford University'])
Exemple #21
0
from IPython import get_ipython
from IPython import InteractiveShell

shell = InteractiveShell.instance()

ip = get_ipython()
ip.run_cell("%time None")
# CPU times: user 2 µs, sys: 0 ns, total: 2 µs
# Wall time: 3.58 µs
from altair.vega.v5 import Vega


DATA_RECORDS = [
    {"amount": 28, "category": "A"},
    {"amount": 55, "category": "B"},
    {"amount": 43, "category": "C"},
    {"amount": 91, "category": "D"},
    {"amount": 81, "category": "E"},
    {"amount": 53, "category": "F"},
    {"amount": 19, "category": "G"},
    {"amount": 87, "category": "H"},
]

if IPYTHON_AVAILABLE:
    _ipshell = InteractiveShell.instance()
    _ipshell.run_cell("%load_ext altair")
    _ipshell.run_cell(
        """
import pandas as pd
table = pd.DataFrame.from_records({})
the_data = table
""".format(
            DATA_RECORDS
        )
    )


VEGA_SPEC = {
    "$schema": "https://vega.github.io/schema/vega/v5.json",
    "axes": [
Exemple #23
0
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from IPython import InteractiveShell

inter = InteractiveShell()
pd.options.display.max_columns = 50

movie = pd.read_csv('data/movie.csv', index_col='movie_title')
c1 = movie['title_year'] >= 2010
c2 = movie['title_year'].isnull()
criteria = c1 | c2
print(movie.mask(criteria).head())

movie_mask = movie.mask(criteria).dropna(how='all')
print(movie_mask.head())

movie_boolean = movie[movie['title_year'] < 2010]
print(movie_boolean.head())
print(movie_mask.equals(movie_boolean))

print(movie_mask.shape == movie_boolean.shape)
print(movie_mask.dtypes == movie_boolean.dtypes)

from pandas.testing import assert_frame_equal
assert_frame_equal(movie_boolean, movie_mask, check_dtype=False)

print(inter.get_ipython().run_line_magic(
    'timeit', "movie.mask(criteria).dropna(how='all')"))
print(inter.get_ipython().run_line_magic('timeit',
                                         "movie[movie['title_year'] < 2010]"))
import pandas as pd
import numpy as np
from IPython import InteractiveShell

inter = InteractiveShell()
pd.options.display.max_columns = 50

college = pd.read_csv('data/college.csv')
college2 = college.set_index('STABBR')
print(college2.index.is_monotonic)

college3 = college2.sort_index()
print(college3.index.is_monotonic)
print(inter.get_ipython().run_line_magic('timeit',
                                         "college[college['STABBR'] == 'TX']"))
print(inter.get_ipython().run_line_magic('timeit', "college2.loc['TX']"))
print(inter.get_ipython().run_line_magic('timeit', "college3.loc['TX']"))

college_unique = college.set_index('INSTNM')
print(college_unique.index.is_unique)

college[college['INSTNM'] == 'Stanford University']
print(college_unique.loc['Stanford University'])
print(inter.get_ipython().run_line_magic(
    'timeit', "college[college['INSTNM'] == 'Stanford University']"))
print(inter.get_ipython().run_line_magic(
    'timeit', "college_unique.loc['Stanford University']"))

college.index = college['CITY'] + ', ' + college['STABBR']
college = college.sort_index()
print(college.head())
Exemple #25
0
class JupyterShell:
    """Helper class for managing shell interpreters."""

    _instance = None

    def __new__(cls, *__, **___):
        raise RuntimeError(
            f'Use static get_instance method in instantiate {cls}.')

    def __init__(self, user_ns=None):
        self.shell = None
        self.user_ns = user_ns or {}
        self._tmpdir = self.tmpdir = None
        self.stdout = STDOUT
        self.stderr = STDERR

    def update_user_ns(self, **kwargs):
        """Add a varaible to the user namespace."""
        self.user_ns.update(**kwargs)

    @staticmethod
    def get_instance():
        if JupyterShell._instance is None:
            obj = object.__new__(JupyterShell)
            obj.__init__()
            obj.new()
            JupyterShell._instance = obj
        return JupyterShell._instance

    def new(self):
        """Create a new shell."""
        self._tmpdir = tempfile.TemporaryDirectory()
        self.tmpdir = os.path.realpath(self._tmpdir.name)
        self.update_user_ns(Shell=self)
        self.shell = InteractiveShell(user_ns=self.user_ns)

    def reset(self):
        """Reset the shell."""
        if self._tmpdir:
            self._tmpdir.cleanup()
            self._tmpdir = self.tmpdir = None
        self.shell.reset()
        self.user_ns = dict(Shell=self)
        self.new()

    def cleanup(self):
        if self._tmpdir:
            self._tmpdir.cleanup()
            self._tmpdir = self.tmpdir = None

    def get_cell_count(self):
        """Return the last cell number."""
        try:
            return max(self.user_ns.get('Out').keys())
        except (AttributeError, ValueError):
            return None

    def run_cell(self, code, silent=False):
        """Run a cell."""
        if not self.shell:
            self.new()

        with closing(StringIO()) as _stdout, closing(StringIO()) as _stderr:
            with redirect_stdout(_stdout), redirect_stderr(_stderr):
                result = self.shell.run_cell(f'{code}\n',
                                             store_history=not silent,
                                             silent=silent)

            stdout = _stdout.getvalue()  # pylint: disable=no-member
            stderr = _stdout.getvalue()  # pylint: disable=no-member

        if result is not None:
            pattern = r'^Out\s*\[{}\]'.format(self.get_cell_count())
            stdout = '\n'.join(pop_until_match(stdout.split('\n'), pattern))
        return result, stdout, stderr

    @contextmanager
    def chdir(self, newdir=None):  # pylint: disable=invalid-name, no-self-argument
        """Change working directory."""
        newdir = newdir or self.tmpdir
        curdir = os.getcwd()
        if newdir != curdir:
            os.chdir(os.path.realpath(os.path.expanduser(newdir)))
        try:
            yield
        finally:
            if newdir != curdir:
                os.chdir(curdir)

    @staticmethod
    def copy(source, destination):
        """Copy a file."""
        if isinstance(source, (list, tuple)):
            source = os.path.join(*source)
        if isinstance(destination, (list, tuple)):
            destination = os.path.join(*destination)
        shutil.copy(source, destination)
Exemple #26
0
def list_variables(
    configuration_file_path: str,
    out: Union[IO, str] = sys.stdout,
    overwrite: bool = False,
    force_text_output: bool = False,
):
    """
    Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.

    List is generally written as text. It can be displayed as a scrollable table view if:
    - function is used in an interactive IPython shell
    - out == sys.stdout
    - force_text_output == False

    :param configuration_file_path:
    :param out: the output stream or a path for the output file
    :param overwrite: if True and out parameter is a file path, the file will be written even if one
                      already exists
    :param force_text_output: if True, list will be written as text, even if command is used in an
                              interactive IPython shell (Jupyter notebook). Has no effect in other
                              shells or if out parameter is not sys.stdout
    :raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
                                exists
    """
    conf = FASTOADProblemConfigurator(configuration_file_path)
    problem = conf.get_problem()
    problem.setup()

    # Extracting inputs and outputs
    variables = VariableList.from_problem(problem, get_promoted_names=False)
    variables.sort(key=lambda var: var.name)
    input_variables = VariableList([var for var in variables if var.is_input])
    output_variables = VariableList(
        [var for var in variables if not var.is_input])

    if isinstance(out, str):
        if not overwrite and pth.exists(out):
            raise FastFileExistsError(
                "File %s not written because it already exists. "
                "Use overwrite=True to bypass." % out,
                out,
            )
        make_parent_dir(out)
        out_file = open(out, "w")
        table_width = MAX_TABLE_WIDTH
    else:
        if out == sys.stdout and InteractiveShell.initialized(
        ) and not force_text_output:
            # Here we display the variable list as VariableViewer in a notebook
            for var in input_variables:
                var.metadata["I/O"] = "IN"
            for var in output_variables:
                var.metadata["I/O"] = "OUT"

            df = ((input_variables + output_variables).to_dataframe()[[
                "I/O", "name", "desc"
            ]].rename(columns={
                "name": "Name",
                "desc": "Description"
            }))
            display(HTML(df.to_html()))
            return

        # Here we continue with text output
        out_file = out
        table_width = min(get_terminal_size().columns, MAX_TABLE_WIDTH) - 1

    pd.set_option("display.max_colwidth", 1000)
    max_name_length = np.max([
        len(name)
        for name in input_variables.names() + output_variables.names()
    ])
    description_text_width = table_width - max_name_length - 2

    def _write_variables(out_f, variables):
        """Writes variables and their description as a pandas DataFrame"""
        df = variables.to_dataframe()

        # Create a new Series where description are wrapped on several lines if needed.
        # Each line becomes an element of the Series
        df["desc"] = [
            "\n".join(tw.wrap(s, description_text_width)) for s in df["desc"]
        ]
        new_desc = df.desc.str.split("\n", expand=True).stack()

        # Create a Series for name that will match new_desc Series. Variable name will be in front of
        # first line of description. An empty string will be in front of other lines.
        new_name = [
            df.name.loc[i] if j == 0 else "" for i, j in new_desc.index
        ]

        # Create the DataFrame that will be displayed
        new_df = pd.DataFrame({"NAME": new_name, "DESCRIPTION": new_desc})

        out_f.write(
            new_df.to_string(
                index=False,
                columns=["NAME", "DESCRIPTION"],
                justify="center",
                formatters={  # Formatters are needed for enforcing left justification
                    "NAME": ("{:%s}" % max_name_length).format,
                    "DESCRIPTION": ("{:%s}" % description_text_width).format,
                },
            )
        )
        out_file.write("\n")

    def _write_text_with_line(txt: str, line_length: int):
        """ Writes a line of given length with provided text inside """
        out_file.write("-" + txt + "-" * (line_length - 1 - len(txt)) + "\n")

    # Inputs
    _write_text_with_line(" INPUTS OF THE PROBLEM ", table_width)
    _write_variables(out_file, input_variables)

    # Outputs
    out_file.write("\n")
    _write_text_with_line(" OUTPUTS OF THE PROBLEM ", table_width)
    _write_variables(out_file, output_variables)
    _write_text_with_line("", table_width)

    if isinstance(out, str):
        out_file.close()
        _LOGGER.info("Output list written in %s", out_file)
Exemple #27
0
 def new(self):
     """Create a new shell."""
     self._tmpdir = tempfile.TemporaryDirectory()
     self.tmpdir = os.path.realpath(self._tmpdir.name)
     self.update_user_ns(Shell=self)
     self.shell = InteractiveShell(user_ns=self.user_ns)
Exemple #28
0
def list_modules(
    configuration_file_path: str = None,
    out: Union[IO, str] = None,
    overwrite: bool = False,
    verbose: bool = False,
    force_text_output: bool = False,
):
    """
    Writes list of available systems.
    If configuration_file_path is given and if it defines paths where there are registered systems,
    they will be listed too.

    :param configuration_file_path:
    :param out: the output stream or a path for the output file (None means sys.stdout)
    :param overwrite: if True and out is a file path, the file will be written even if one already
                      exists
    :param verbose: if True, shows detailed information for each system
                    if False, shows only identifier and path of each system
    :param force_text_output: if True, list will be written as text, even if command is used in an
                              interactive IPython shell (Jupyter notebook). Has no effect in other
                              shells or if out parameter is not sys.stdout
   :raise FastFileExistsError: if overwrite==False and out is a file path and the file exists
    """
    if out is None:
        out = sys.stdout

    if configuration_file_path:
        conf = FASTOADProblemConfigurator(configuration_file_path)
        conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
        conf.load(configuration_file_path)
    # As the problem has been configured, BundleLoader now knows additional registered systems

    if verbose:
        cell_list = _get_detailed_system_list()
    else:
        cell_list = _get_simple_system_list()

    if isinstance(out, str):
        if not overwrite and pth.exists(out):
            raise FastFileExistsError(
                "File %s not written because it already exists. "
                "Use overwrite=True to bypass." % out,
                out,
            )

        make_parent_dir(out)
        out_file = open(out, "w")
    else:
        if (
            out == sys.stdout
            and InteractiveShell.initialized()
            and not force_text_output
            and not verbose
        ):
            display(HTML(tabulate(cell_list, tablefmt="html")))
            return

        out_file = out

    out_file.write(tabulate(cell_list, tablefmt="grid"))
    out_file.write("\n")

    if isinstance(out, str):
        out_file.close()
        _LOGGER.info("System list written in %s", out_file)
import pandas as pd
import numpy as np
from IPython import InteractiveShell

inter = InteractiveShell()
pd.options.display.max_columns = 50

college = pd.read_csv('data/college.csv')
print(college[college['STABBR'] == 'TX'].head())

college2 = college.set_index('STABBR')
print(college2.loc['TX'].head())

print(inter.get_ipython().run_line_magic('timeit',
                                         "college[college['STABBR'] == 'TX']"))
print(inter.get_ipython().run_line_magic('timeit', "college2.loc['TX']"))
print(inter.get_ipython().run_line_magic(
    'timeit', "college2 = college.set_index('STABBR')"))

states = ['TX', 'CA', 'NY']
print(college[college['STABBR'].isin(states)])
print(college2.loc[states].head())