def open(self, data=None, metadata=None):
"""Open the frontend-side version of this comm"""
if data is None:
data = self._open_data
self._closed = False
get_ipython().comm_manager.register_comm(self)
self._publish_msg('comm_open', data, metadata, target_name=self.target_name)
def __init__(self, target_name='', data=None, **kwargs):
if target_name:
kwargs['target_name'] = target_name
super(Comm, self).__init__(**kwargs)
get_ipython().comm_manager.register_comm(self)
if self.primary:
# I am primary, open my peer.
self.open(data)
def close(self, data=None, metadata=None):
"""Close the frontend-side version of this comm"""
if self._closed:
# only close once
return
if data is None:
data = self._close_data
self._publish_msg('comm_close', data, metadata)
get_ipython().comm_manager.unregister_comm(self)
self._closed = True
def __init__(self, value=None, **kwargs):
self.log.info("Created a new Function widget.")
self.on_msg(self._handle_invoke_msg)
self.shell = get_ipython()
self.serializer = Serializer()
super(Function, self).__init__(**kwargs)
def enable_notebook(d3_url=None):
"""Enable the automatic display of figures in the IPython Notebook.
This function should be used with the inline Matplotlib backend
that ships with IPython that can be enabled with `%pylab inline`
or `%matplotlib inline`. This works by adding an HTML formatter
for Figure objects; the existing SVG/PNG formatters will remain
enabled.
Parameters
----------
d3_url : string (optional)
if specified, then find the d3 library at the provided URL
See Also
--------
disable_notebook : undo this operation
display_d3 : display a single figure in the notebook
"""
try:
from IPython.core.getipython import get_ipython
from matplotlib.figure import Figure
except ImportError:
raise ImportError('This feature requires IPython 1.0+ and Matplotlib')
ip = get_ipython()
formatter = ip.display_formatter.formatters['text/html']
formatter.for_type(Figure, lambda fig: fig_to_d3(fig, d3_url))
def assess_gc_fit(reffile=None, gcname='Glassy_Carbon'):
ip = get_ipython()
if reffile is None:
reffile = ip.user_ns['_loaders'][0].get_subpath('config/GC_data_nm.dat')
refcurve = Curve.new_from_file(reffile)
f = plt.figure()
f.add_subplot(1, 1, 1)
rads = {}
for fsn in sorted([h.fsn for h in ip.user_ns['_headers']['processed'] if h.title == gcname]):
try:
ex = load_exposure(fsn, raw=False, processed=True)
except:
continue
rads[ex.header.fsn] = ex.radial_average(refcurve.q)
del ex
qmin = max([r.sanitize().q.min() for r in rads.values()])
qmax = min([r.sanitize().q.max() for r in rads.values()])
refcurve.trim(qmin, qmax).loglog('o', mfc='none', ms=10)
for r in sorted(rads):
rads[r].loglog('.', label='#{:d}'.format(r))
plt.axis('tight')
plt.legend(loc='best', numpoints=1)
plt.xlabel('q (nm$^{-1}$)')
plt.ylabel('$d\Sigma/d\Omega$ (cm$^{-1}$ sr$^{-1}$)')
plt.grid(True, which='both')
plt.draw()
def enable_notebook(**kwargs):
"""Enable the automatic display of figures in the IPython Notebook.
This function should be used with the inline Matplotlib backend
that ships with IPython that can be enabled with `%pylab inline`
or `%matplotlib inline`. This works by adding an HTML formatter
for Figure objects; the existing SVG/PNG formatters will remain
enabled.
Parameters
----------
**kwargs :
all keyword parameters are passed through to :func:`fig_to_html`
See Also
--------
:func:`disable_notebook` : undo the action of enable_notebook
:func:`display` : embed figure within the IPython notebook
:func:`show` : launch a local server and show a figure in a browser
"""
try:
from IPython.core.getipython import get_ipython
from matplotlib.figure import Figure
except ImportError:
raise ImportError('This feature requires IPython 1.0+ and Matplotlib')
ip = get_ipython()
formatter = ip.display_formatter.formatters['text/html']
formatter.for_type(Figure,
lambda fig, kwds=kwargs: fig_to_html(fig, **kwds))
def doit(line):
"""
Run *doit* with `task_creators` from all interactive variables
(IPython's global namespace).
Examples:
>>> %doit --help ## Show help for options and arguments.
>>> def task_foo():
return {'actions': ['echo hi IPython'],
'verbosity': 2}
>>> %doit list ## List any tasks discovered.
foo
>>> %doit ## Run any tasks.
. foo
hi IPython
"""
ip = get_ipython()
# Override db-files location inside ipython-profile dir,
# which is certainly writable.
prof_dir = ip.profile_dir.location
opt_vals = {'dep_file': os.path.join(prof_dir, 'db', '.doit.db')}
commander = DoitMain(ModuleTaskLoader(ip.user_module),
extra_config={'GLOBAL': opt_vals})
commander.run(line.split())
def setUp(self):
self.notary = sign.NotebookNotary(
secret=b'secret',
profile_dir=get_ipython().profile_dir
)
with self.fopen(u'test3.ipynb', u'r') as f:
self.nb = read(f, u'json')
def __init__(self, value=None, **kwargs):
self.log.info("Created a new DataFrame widget.")
self.on_msg(self._handle_state_msg)
self.shell = get_ipython()
self.serializer = Serializer()
super(DataFrame, self).__init__(**kwargs)
def page(strng, start=0, screen_lines=0, pager_cmd=None):
"""Print a string, piping through a pager.
This version ignores the screen_lines and pager_cmd arguments and uses
IPython's payload system instead.
Parameters
----------
strng : str or mime-dict
Text to page, or a mime-type keyed dict of already formatted data.
start : int
Starting line at which to place the display.
"""
# Some routines may auto-compute start offsets incorrectly and pass a
# negative value. Offset to 0 for robustness.
start = max(0, start)
shell = get_ipython()
if isinstance(strng, dict):
data = strng
else:
data = {'text/plain' : strng}
payload = dict(
source='page',
data=data,
start=start,
screen_lines=screen_lines,
)
shell.payload_manager.write_payload(payload)
def assess_sample_stability(end_cutoff=3):
ip = get_ipython()
rowavg = ip.user_ns['_rowavg']
tab = [['Sample name', 'Distance', 'Slope of autocorrelation function', 'Stability']]
plt.figure()
for sn in sorted(rowavg):
for dist in sorted(rowavg[sn]):
rowavg_rescaled = rowavg[sn][dist] / rowavg[sn][dist].mean()
rowavg_std = rowavg_rescaled.std() * np.ones_like(rowavg_rescaled)
try:
A, B, stat = nonlinear_leastsquares(np.arange(len(rowavg_rescaled)), rowavg_rescaled, rowavg_std,
lambda x, a, b: a * x + b, [0, 0])
problematic = (A.val > A.err * 3)
except TypeError:
A = 'N/A'
problematic = 2
tab.append([sn, '%.2f' % dist, A, ["\u2713", "\u2718\u2718\u2718\u2718\u2718", '\u274e'][problematic]])
plt.errorbar(np.arange(len(rowavg_rescaled)), rowavg_rescaled,
label=sn + ' %.2f mm' % dist) # ,diags_std[1:])
plt.xlabel('Separation in time (FSN units)')
plt.ylabel('Average discrepancy between curves')
plt.legend(loc='best')
tab = ipy_table.IpyTable(tab)
tab.apply_theme('basic')
display(tab)
plt.show()
def __init__(self, **kwargs):
self.log.info("Created a new IpywProxy widget.")
self.on_msg(self._handle_custom_event_msg)
self.shell = get_ipython()
super(IpywProxy, self).__init__(**kwargs)
def update(self, *args):
"""
Call the interact function and update the output widget with
the result of the function call.
Parameters
----------
*args : ignored
Required for this method to be used as traitlets callback.
"""
self.kwargs = {}
if self.manual:
self.manual_button.disabled = True
try:
show_inline_matplotlib_plots()
with self.out:
if self.clear_output:
clear_output(wait=True)
for widget in self.kwargs_widgets:
value = widget.get_interact_value()
self.kwargs[widget._kwarg] = value
self.result = self.f(**self.kwargs)
show_inline_matplotlib_plots()
if self.auto_display and self.result is not None:
display(self.result)
except Exception as e:
ip = get_ipython()
if ip is None:
self.log.warn("Exception in interact callback: %s", e, exc_info=True)
else:
ip.showtraceback()
finally:
if self.manual:
self.manual_button.disabled = False
def warn_format_error(method, self, *args, **kwargs):
"""decorator for warning on failed format call"""
try:
r = method(self, *args, **kwargs)
except NotImplementedError:
# don't warn on NotImplementedErrors
return None
except Exception:
exc_info = sys.exc_info()
ip = get_ipython()
if ip is not None:
ip.showtraceback(exc_info)
else:
traceback.print_exception(*exc_info)
return None
if r is None or isinstance(r, self._return_type) or \
(isinstance(r, tuple) and r and isinstance(r[0], self._return_type)):
return r
else:
warnings.warn(
"%s formatter returned invalid type %s (expected %s) for object: %s" %
(self.format_type, type(r),
self._return_type, _safe_repr(args[0])),
FormatterWarning
)
def assess_transmission():
ip = get_ipython()
tab = [
['Sample name', 'Distance', 'Transmission', 'Linear absorption coefficient (1/cm)', 'Absorption length (cm)']]
for sample in sorted(ip.user_ns['_headers_sample']):
for dist in sorted(ip.user_ns['_headers_sample'][sample]):
transms_seen = []
for h in ip.user_ns['_headers_sample'][sample][dist]:
if float(h.transmission) not in transms_seen:
transms_seen.append(float(h.transmission))
transm = h.transmission
thickness = h.thickness
try:
mu = (-transm.log()) / thickness
except ZeroDivisionError:
mu = 'Infinite'
invmu = 0
else:
try:
invmu = (1 / mu).tostring(extra_digits=2)
except ZeroDivisionError:
invmu = 'Infinite'
mu = mu.tostring(extra_digits=2)
tab.append([sample, dist, transm.tostring(extra_digits=2), mu, invmu])
tab = ipy_table.IpyTable(tab)
tab.apply_theme('basic')
display(tab)
def load_headers(fsns: List[int]):
"""Load header files
"""
ip = get_ipython()
ip.user_ns["_headers"] = {}
for type_ in ["raw", "processed"]:
print("Loading %d headers (%s)" % (len(fsns), type_), flush=True)
processed = type_ == "processed"
headers = []
for f in fsns:
for l in [l_ for l_ in ip.user_ns["_loaders"] if l_.processed == processed]:
try:
headers.append(l.loadheader(f))
break
except FileNotFoundError:
continue
allsamplenames = {h.title for h in headers}
if not headers:
print('NO HEADERS READ FOR TYPE "%s"' % type_)
else:
print("%d headers (%s) out of %d have been loaded successfully." % (len(headers), type_, len(fsns)))
print("Read FSN range:", min([h.fsn for h in headers]), "to", max([h.fsn for h in headers]))
print("Samples covered by these headers:")
print(" " + "\n ".join(sorted(allsamplenames)), flush=True)
if processed:
ip.user_ns["allsamplenames"] = allsamplenames
ip.user_ns["_headers"][type_] = headers
def test_update_display():
ip = get_ipython()
with mock.patch.object(ip.display_pub, 'publish') as pub:
with nt.assert_raises(TypeError):
display.update_display('x')
display.update_display('x', display_id='1')
display.update_display('y', display_id='2')
args, kwargs = pub.call_args_list[0]
nt.assert_equal(args, ())
nt.assert_equal(kwargs, {
'data': {
'text/plain': repr('x')
},
'metadata': {},
'transient': {
'display_id': '1',
},
'update': True,
})
args, kwargs = pub.call_args_list[1]
nt.assert_equal(args, ())
nt.assert_equal(kwargs, {
'data': {
'text/plain': repr('y')
},
'metadata': {},
'transient': {
'display_id': '2',
},
'update': True,
})
def enable_ipython(self, **kwargs):
"""
Enable plotting in the iPython notebook.
Once enabled, all lightning plots will be automatically produced
within the iPython notebook. They will also be available on
your lightning server within the current session.
"""
# inspired by code powering similar functionality in mpld3
# https://github.com/jakevdp/mpld3/blob/master/mpld3/_display.py#L357
from IPython.core.getipython import get_ipython
from IPython.display import display, Javascript, HTML
self.ipython_enabled = True
self.set_size('medium')
ip = get_ipython()
formatter = ip.display_formatter.formatters['text/html']
if self.local_enabled:
from lightning.visualization import VisualizationLocal
js = VisualizationLocal.load_embed()
display(HTML("<script>" + js + "</script>"))
if not self.quiet:
print('Running local mode, some functionality limited.\n')
formatter.for_type(VisualizationLocal, lambda viz, kwds=kwargs: viz.get_html())
else:
formatter.for_type(Visualization, lambda viz, kwds=kwargs: viz.get_html())
r = requests.get(self.get_ipython_markup_link(), auth=self.auth)
display(Javascript(r.text))
def get_ipython_notebook_path():
"""Return the path to the current notebook file
This is a horrible hack, but there doesn't seem to be another way to do it.
"""
# Partly from: http://stackoverflow.com/a/13055551/1144479
from IPython.core.getipython import get_ipython
import requests
import os
from IPython.lib import kernel
cf = kernel.get_connection_file()
kernel_id = cf.split('-', 1)[1].split('.')[0]
r = requests.get('http://127.0.0.1:8888/api/sessions')
r.json()
notebook_path = None
for s in requests.get('http://127.0.0.1:8888/api/sessions').json():
if s['kernel']['id'] == kernel_id:
notebook_path = os.path.join(get_ipython().starting_dir, s['notebook']['path'], s['notebook']['name'])
break
return notebook_path
def cut(self, line, cell):
from IPython.core.getipython import get_ipython
from sys import stdout
args = parse_argstring(self.cut, line)
max_lines = args.lines
max_chars = args.chars
counters = dict(chars=0, lines=0)
def write(string):
counters["lines"] += string.count("\n")
counters["chars"] += len(string)
if counters["lines"] >= max_lines:
raise IOError("Too many lines")
elif counters["chars"] >= max_chars:
raise IOError("Too many characters")
else:
old_write(string)
try:
old_write, stdout.write = stdout.write, write
ipython = get_ipython()
ipython.run_cell(cell)
finally:
del stdout.write
def matplotlib_png(self, line):
ip = get_ipython()
ip.run_cell("""
%config InlineBackend.figure_format = 'png'
%config InlineBackend.rc = {'figure.figsize': (12.0, 6.0), 'figure.facecolor':(1.0, 1.0, 1.0)}
%matplotlib inline
""")
def include(self, line, cell):
"""
include section of the files in scpy2 folder
"""
import json
ip = get_ipython()
args = parse_argstring(self.include, line)
language = args.language
filepath = args.path
section = args.section
run = args.run
first_line = "%%include " + line
section_text = get_section(filepath, section)
if run:
ip.run_cell(section_text)
text = json.dumps(unicode(first_line) + u"\n" + section_text.decode("utf8"))
code = """%%javascript
(function(pattern, text){{
var cells = IPython.notebook.get_cells();
for (var i = 0; i < cells.length; i++) {{
var cell = cells[i];
if (cell.get_text().indexOf(pattern) == 0){{
cell.set_text(text);
}}
}}
}})({0}, {1});
""".format(json.dumps(first_line), text)
ip.run_cell(code)
from IPython import display
display.clear_output()
def __enter__(self):
from IPython.core.getipython import get_ipython
from IPython.core.displaypub import CapturingDisplayPublisher
self.sys_stdout = sys.stdout
self.sys_stderr = sys.stderr
if self.display:
self.shell = get_ipython()
if self.shell is None:
self.save_display_pub = None
self.display = False
stdout = stderr = outputs = None
if self.stdout:
stdout = sys.stdout = StringIO()
if self.stderr:
stderr = sys.stderr = StringIO()
if self.display:
self.save_display_pub = self.shell.display_pub
self.shell.display_pub = CapturingDisplayPublisher()
outputs = self.shell.display_pub.outputs
return CapturedIO(stdout, stderr, outputs)
def post_mortem_excepthook(type, value, tb):
"""
For post mortem exception handling, print a banner and enable post
mortem debugging.
"""
clear_post_mortem()
if IS_IPYTHON:
from IPython.core.getipython import get_ipython
ipython_shell = get_ipython()
ipython_shell.showtraceback((type, value, tb))
p = pdb.Pdb(ipython_shell.colors)
else:
traceback.print_exception(type, value, tb, file=sys.stderr)
p = pdb.Pdb()
if not type == SyntaxError:
# wait for stderr to print (stderr.flush does not work in this case)
time.sleep(0.1)
_print('*' * 40)
_print('Entering post mortem debugging...')
_print('*' * 40)
# add ability to move between frames
p.send_initial_notification = False
p.reset()
frame = tb.tb_frame
prev = frame
while frame.f_back:
prev = frame
frame = frame.f_back
frame = prev
# wait for stdout to print
time.sleep(0.1)
p.interaction(frame, tb)
def assess_instrumental_background(Wx=20, Wy=20, emptyname='Empty_Beam', maskname='mask.mat'):
ip = get_ipython()
for dist in ip.user_ns['_headers_sample'][emptyname]:
data = ip.user_ns['_data2d'][emptyname][dist]
assert isinstance(data, Exposure)
intensity = data.intensity / data.header.exposuretime
mask = load_mask(maskname).astype(np.bool)
m = np.zeros_like(intensity)
print(' Mean intensity per pixel:', intensity[mask].mean(), 'cps')
print(' STD intensity per pixel:', intensity[mask == 1].std(), 'cps')
print(' Total intensity:', intensity[mask == 1].sum(), 'cps')
for row in range(m.shape[0]):
for col in range(m.shape[1]):
m[row, col] = intensity[max(row - Wy, 0):min(row + Wy, m.shape[0] - 1),
max(col - Wx, 0):min(col + Wx, m.shape[1] - 1)][
mask[max(row - Wy, 0):min(row + Wy, m.shape[0] - 1),
max(col - Wx, 0):
min(col + Wx, m.shape[1] - 1)] == 1].mean()
plt.figure()
plt.subplot(1, 2, 1)
plt.imshow(intensity, norm=LogNorm())
plt.subplot(1, 2, 2)
plt.imshow(m)
plt.tight_layout()
plt.suptitle('Empty beam, {} mm'.format(dist))
def test_image_mimes():
fmt = get_ipython().display_formatter.format
for format in display.Image._ACCEPTABLE_EMBEDDINGS:
mime = display.Image._MIMETYPES[format]
img = display.Image(b'garbage', format=format)
data, metadata = fmt(img)
nt.assert_equal(sorted(data), sorted([mime, 'text/plain']))
def __init__(self, **kwargs):
self.log.info("Created a new Function widget.")
self.on_msg(self._handle_custom_event_msg)
self.shell = get_ipython()
self.serializer = Serializer()
super(Function, self).__init__(**kwargs)
def test_display_handle():
ip = get_ipython()
handle = display.DisplayHandle()
nt.assert_is_instance(handle.display_id, str)
handle = display.DisplayHandle('my-id')
nt.assert_equal(handle.display_id, 'my-id')
with mock.patch.object(ip.display_pub, 'publish') as pub:
handle.display('x')
handle.update('y')
args, kwargs = pub.call_args_list[0]
nt.assert_equal(args, ())
nt.assert_equal(kwargs, {
'data': {
'text/plain': repr('x')
},
'metadata': {},
'transient': {
'display_id': handle.display_id,
}
})
args, kwargs = pub.call_args_list[1]
nt.assert_equal(args, ())
nt.assert_equal(kwargs, {
'data': {
'text/plain': repr('y')
},
'metadata': {},
'transient': {
'display_id': handle.display_id,
},
'update': True,
})
def assess_flux_stability(samplename='Glassy_Carbon'):
ip = get_ipython()
f = plt.figure()
ax1 = f.add_subplot(1, 1, 1)
plt.xlabel('Date of exposure')
plt.ylabel('Beam flux (photon/sec), continuous lines')
ax2 = plt.twinx()
plt.ylabel('Vacuum pressure (mbar), dotted lines')
plt.title('Beam flux stability')
samplenames = sorted([sn_ for sn_ in ip.user_ns['_headers_sample'] if samplename in sn_])
linestyles = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
lines = []
for sn, ls in zip(samplenames, linestyles):
print(sn)
heds = ip.user_ns['_headers_sample'][sn]
allheds = []
for k in heds.keys():
allheds.extend(heds[k])
allheds = sorted(allheds, key=lambda x: x.fsn)
flux = np.array([float(h.flux) for h in allheds])
dates = [h.date for h in allheds]
lines.extend(ax1.plot(dates, flux, ls + 'o', label='Flux (%s)' % sn))
vacuums = np.array([float(h.vacuum) for h in allheds])
lines.extend(ax2.plot(dates, vacuums, ls + 's', label='Vacuum (%s)' % sn, lw=2))
print(' Measurement duration: %.2f h' % ((dates[-1] - dates[0]).total_seconds() / 3600.))
print(' Mean flux: ', flux.mean(), '+/-', flux.std(), 'photons/sec')
print(' RMS variation of flux: ', flux.std() / flux.mean() * 100, '%')
print(' P-P variation of flux: ', flux.ptp() / flux.mean() * 100, '%')
ax1.legend(lines, [l.get_label() for l in lines], loc='best')
plt.show()
def suppress_warnings(line: str = None, cell: str = None, local_ns: dict = None):
"""Suppress all stderr output produced by function call.
NOTE: The output is still present in the DOM, but not visible.
"""
_ = line # line
_ = local_ns # ignore
shell = get_ipython()
code = cell
last_command = None
code_lines = cell.splitlines()
if not re.search(r"^(\s)+", code_lines[-1]):
# is not part of a block, evaluate separately
last_command = code_lines.pop()
code = '\n'.join(code_lines)
# evaluate the whole script except the last line
shell.ex(code)
try:
# try to evaluate and return last command
ret = shell.ev(last_command)
except SyntaxError:
# if this one throws too, then its users error
ret = shell.ex(last_command)
# suppress warnings produced by the execution
script = """
$(element)
.parents('.output')
.find('.output_stderr')
.css('display', 'none');
"""
execute_with_requirements(script, required=[])
return ret
def runcell(cellname, filename):
"""
Run a code cell from an editor as a file.
Currently looks for code in an `ipython` property called `cell_code`.
This property must be set by the editor prior to calling this function.
This function deletes the contents of `cell_code` upon completion.
Parameters
----------
cellname : str
Used as a reference in the history log of which
cell was run with the fuction. This variable is not used.
filename : str
Needed to allow for proper traceback links.
"""
try:
filename = filename.decode('utf-8')
except (UnicodeError, TypeError, AttributeError):
# UnicodeError, TypeError --> eventually raised in Python 2
# AttributeError --> systematically raised in Python 3
pass
ipython_shell = get_ipython()
namespace = _get_globals()
namespace['__file__'] = filename
try:
cell_code = ipython_shell.cell_code
except AttributeError:
_print("--Run Cell Error--\n"
"Please use only through Spyder's Editor; "
"shouldn't be called manually from the console")
return
# Trigger `post_execute` to exit the additional pre-execution.
# See Spyder PR #7310.
ipython_shell.events.trigger('post_execute')
ipython_shell.run_cell(cell_code)
namespace.pop('__file__')
del ipython_shell.cell_code
def enable_notebook(local=False, **kwargs):
"""Enable the automatic display of visualizations in the IPython Notebook.
Parameters
----------
local : boolean (optional, default=False)
if True, then copy the d3 & LDAvis libraries to a location visible to
the notebook server, and source them from there. See Notes below.
**kwargs :
all keyword parameters are passed through to :func:`prepared_data_to_html`
Notes
-----
Known issues: using ``local=True`` may not work correctly in certain cases:
- In IPython < 2.0, ``local=True`` may fail if the current working
directory is changed within the notebook (e.g. with the %cd command).
- In IPython 2.0+, ``local=True`` may fail if a url prefix is added
(e.g. by setting NotebookApp.base_url).
See Also
--------
:func:`disable_notebook` : undo the action of enable_notebook
:func:`display` : embed visualization within the IPython notebook
:func:`show` : launch a local server and show a visualization in a browser
"""
try:
from IPython.core.getipython import get_ipython
except ImportError:
raise ImportError('This feature requires IPython 1.0+')
if local:
if 'ldavis_url' in kwargs or 'd3_url' in kwargs:
warnings.warn("enable_notebook: specified urls are ignored when local=True")
kwargs['d3_url'], kwargs['ldavis_url'], kwargs['ldavis_css_url'] = write_ipynb_local_js()
ip = get_ipython()
formatter = ip.display_formatter.formatters['text/html']
formatter.for_type(PreparedData,
lambda data, kwds=kwargs: prepared_data_to_html(data, **kwds))
def provide_files(line: str):
args = parse_argstring(provide_files, line)
items = []
ipy = get_ipython()
url_prefix = os.path.relpath(os.getcwd(), ipy.starting_dir) + os.path.sep
print(url_prefix)
global __file_provider_functions
global __instances, __instantiated_experiments
for exp_instance, exp_config in zip(__instances, __instantiated_experiments):
dfs = __file_provider_functions[args.file_provider](exp_instance, exp_config, args.args)
if isinstance(dfs, DownloadFile):
items.append(Output())
with items[-1]:
display(DownloadFileLink(dfs.path, file_name=dfs.file_name, link_text=dfs.link_text,
url_prefix=url_prefix))
else:
items.append(VBox([DownloadFileLink(df.path, file_name=df.file_name, link_text=df.link_text,
url_prefix=url_prefix) for df in dfs]))
if len(items) > 1:
tabs = Tab(children=items)
for i, exp in enumerate(__instantiated_experiments):
if args.tab_title:
if (args.tab_title[0] == args.tab_title[-1]) and args.tab_title.startswith(("'", '"')):
selectors = args.tab_title[1:-1]
else:
selectors = args.tab_title
selectors = selectors.split(' ')
values = [reduce(lambda a, b: a[b], [exp['params'], *selector.split('.')]) for selector in selectors]
tabs.set_title(i, ' '.join(map(str, values)))
else:
tabs.set_title(i, '...' + exp['name'][-15:])
display(tabs)
elif len(items) == 1:
return items[0]
else:
warnings.warn('No files loaded')
def nb_setup(autoreload=True, paths=None):
"""Use in first cell of nb for setting up autoreload, paths, ...
By default, nb_setup() loads and set autoreload and adds a path to a directory named 'src'
at the same level as where the notebook directory is located.
By default, nb_setup assumes the following file structure:
project_directory
| --- notebooks
| | --- current_nb.ipynb
| | --- ...
|
|--- src
| | --- scripts_to_import.py
| | --- ...
|
|--- data
| |
| | ...
"""
# Add paths. Default is 'src' if it exists
if paths is None:
p = Path('../src').resolve().absolute()
if p.is_dir():
paths = [str(p)]
else:
paths=[]
if paths:
for p in paths:
sys.path.insert(1, str(p))
print(f"Added following paths: {','.join(paths)}")
# Setup auto reload
if autoreload:
ipshell = get_ipython()
ipshell.run_line_magic('load_ext', 'autoreload')
ipshell.run_line_magic('autoreload', '2')
print('Set autoreload mode')
def test_display_handle():
ip = get_ipython()
handle = display.DisplayHandle()
nt.assert_is_instance(handle.display_id, str)
handle = display.DisplayHandle("my-id")
nt.assert_equal(handle.display_id, "my-id")
with mock.patch.object(ip.display_pub, "publish") as pub:
handle.display("x")
handle.update("y")
args, kwargs = pub.call_args_list[0]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs,
{
"data": {
"text/plain": repr("x")
},
"metadata": {},
"transient": {
"display_id": handle.display_id
},
},
)
args, kwargs = pub.call_args_list[1]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs,
{
"data": {
"text/plain": repr("y")
},
"metadata": {},
"transient": {
"display_id": handle.display_id
},
"update": True,
},
)
def test_select_figure_formats_kwargs():
ip = get_ipython()
kwargs = dict(quality=10, bbox_inches='tight')
pt.select_figure_formats(ip, 'png', **kwargs)
formatter = ip.display_formatter.formatters['image/png']
f = formatter.lookup_by_type(Figure)
cell = f.keywords
expected = kwargs
expected["base64"] = True
expected["fmt"] = "png"
assert cell == expected
# check that the formatter doesn't raise
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot([1,2,3])
plt.draw()
formatter.enabled = True
png = formatter(fig)
assert isinstance(png, str)
png_bytes = a2b_base64(png)
assert png_bytes.startswith(_PNG)
def create_new_cell(contents):
"""
Creates new cell in jupyter notebook.
Args:
contents: contents of cell.
"""
import sys
from IPython.core.getipython import get_ipython
if 'ipykernel' not in sys.modules:
raise EnvironmentError('The magic functions are only usable in a '
'Jupyter notebook.')
shell = get_ipython()
payload = dict(
source='set_next_input',
text=contents,
replace=False,
)
shell.payload_manager.write_payload(payload, single=False)
def __exit__(self, exc_type, exc_val, exc_tb):
"""
Reset the namespace.
"""
ipython_shell = get_ipython()
ipython_shell.kernel._running_namespace = None
if self._previous_filename:
self.ns_globals['__file__'] = self._previous_filename
elif '__file__' in self.ns_globals:
self.ns_globals.pop('__file__')
if not self.current_namespace:
self.context_globals.update(self.ns_globals)
if self.context_locals and self.ns_locals:
self.context_locals.update(self.ns_locals)
if self._previous_main:
sys.modules['__main__'] = self._previous_main
elif '__main__' in sys.modules and self._reset_main:
del sys.modules['__main__']
if self.filename in linecache.cache and os.path.exists(self.filename):
linecache.cache.pop(self.filename)
def __enter__(self):
"""
Prepare the namespace.
"""
# Save previous __file__
if self.ns_globals is None:
if self.current_namespace:
self.ns_globals, self.ns_locals = _get_globals_locals()
if '__file__' in self.ns_globals:
self._previous_filename = self.ns_globals['__file__']
self.ns_globals['__file__'] = self.filename
else:
ipython_shell = get_ipython()
main_mod = ipython_shell.new_main_mod(
self.filename, '__main__')
self.ns_globals = main_mod.__dict__
self.ns_locals = None
# Needed to allow pickle to reference main
if '__main__' in sys.modules:
self._previous_main = sys.modules['__main__']
sys.modules['__main__'] = main_mod
self._reset_main = True
if (self._file_code is not None
and not PY2
and isinstance(self._file_code, bytes)):
try:
self._file_code = self._file_code.decode()
except UnicodeDecodeError:
# Setting the cache is not supported for non utf-8 files
self._file_code = None
if self._file_code is not None:
# '\n' is used instead of the native line endings. (see linecache)
# mtime is set to None to avoid a cache update.
linecache.cache[self.filename] = (
len(self._file_code), None,
[line + '\n' for line in self._file_code.splitlines()],
self.filename)
return self.ns_globals, self.ns_locals
def enter_debugger(filename, continue_if_has_breakpoints, code_format):
"""Enter debugger. Code format should be a format that accept filename."""
kernel = get_ipython().kernel
recursive = kernel.is_debugging()
if recursive:
parent_debugger = kernel._pdb_obj
sys.settrace(None)
globals = parent_debugger.curframe.f_globals
locals = parent_debugger.curframe_locals
# Create child debugger
debugger = SpyderPdb(
completekey=parent_debugger.completekey,
stdin=parent_debugger.stdin, stdout=parent_debugger.stdout)
debugger.use_rawinput = parent_debugger.use_rawinput
debugger.prompt = "(%s) " % parent_debugger.prompt.strip()
else:
debugger = SpyderPdb()
filename = debugger.canonic(filename)
debugger._wait_for_mainpyfile = True
debugger.mainpyfile = filename
debugger.continue_if_has_breakpoints = continue_if_has_breakpoints
debugger._user_requested_quit = False
if os.name == 'nt':
filename = filename.replace('\\', '/')
code = code_format.format(repr(filename))
if recursive:
# Enter recursive debugger
sys.call_tracing(debugger.run, (code, globals, locals))
# Reset parent debugger
sys.settrace(parent_debugger.trace_dispatch)
parent_debugger.lastcmd = debugger.lastcmd
kernel._register_pdb_session(parent_debugger)
else:
# The breakpoint might not be in the cell
debugger.run(code)
def __enter__(self):
"""
Prepare the namespace.
"""
# Save previous __file__
if self.namespace is None:
if self.current_namespace:
self.namespace = _get_globals()
else:
ipython_shell = get_ipython()
main_mod = ipython_shell.new_main_mod(
self.filename, '__main__')
self.namespace = main_mod.__dict__
# Needed to allow pickle to reference main
if '__main__' in sys.modules:
self._previous_main = sys.modules['__main__']
sys.modules['__main__'] = main_mod
self._reset_main = True
if '__file__' in self.namespace:
self._previous_filename = self.namespace['__file__']
self.namespace['__file__'] = self.filename
return self.namespace
def attach_exception_hook():
"""Injects async exception hook into the sys.excepthook."""
try:
# detect whether we're running in IPython
__IPYTHON__
except NameError:
global is_attached, original_hook
if is_attached:
sys.stderr.write(
"Warning: async exception hook was already attached.\n")
return
original_hook = sys.excepthook
sys.excepthook = async_exception_hook
is_attached = True
else:
# override ipython's exception handler if in a shell.
# we need to do this because ipython overrides sys.excepthook
# so just the else block doesn't cover that case.
from IPython.core.getipython import get_ipython
shell = get_ipython()
shell.set_custom_exc((BaseException, ),
ipython_custom_exception_handler)
def autonotify(self, line):
# Record options
args = parse_argstring(self.autonotify, line)
self.options["body"] = args.message.lstrip("\'\"").rstrip("\'\"")
self.options['autonotify_after'] = args.after
self.options['autonotify_output'] = args.output
### Register events
ip = get_ipython()
# Remove events if they're already registered
# This is necessary because jupyter makes a new instance everytime
pre, post = self.__class__._events
if pre and pre in ip.events.callbacks['pre_run_cell']:
ip.events.callbacks['pre_run_cell'].remove(pre)
if post and post in ip.events.callbacks['post_run_cell']:
ip.events.callbacks['post_run_cell'].remove(post)
# Register new events
ip.events.register('pre_run_cell', self.pre_run_cell)
ip.events.register('post_run_cell', self.post_run_cell)
self.__class__._events = self.pre_run_cell, self.post_run_cell
def post_mortem_excepthook(type, value, tb):
"""
For post mortem exception handling, print a banner and enable post
mortem debugging.
"""
ipython_shell = get_ipython()
ipython_shell.showtraceback((type, value, tb))
p = pdb.Pdb(ipython_shell.colors)
if not type == SyntaxError:
# wait for stderr to print (stderr.flush does not work in this case)
time.sleep(0.1)
_print('*' * 40)
_print('Entering post mortem debugging...')
_print('*' * 40)
# add ability to move between frames
p.send_initial_notification = False
p.reset()
frame = tb.tb_next.tb_frame
# wait for stdout to print
time.sleep(0.1)
p.interaction(frame, tb)
def call_f(name=None, old=None, new=None):
container.kwargs = {}
for widget in kwargs_widgets:
value = widget.value
container.kwargs[widget._kwarg] = value
if co:
clear_output(wait=True)
if manual:
manual_button.disabled = True
try:
container.result = f(**container.kwargs)
except Exception as e:
ip = get_ipython()
if ip is None:
container.log.warn("Exception in interact callback: %s",
e,
exc_info=True)
else:
ip.showtraceback()
finally:
if manual:
manual_button.disabled = False
def runOrcSco(orcName, scoName):
"""Run an orc and sco stored in the user namespace.
One can store an orc in the user namespace with the %%orc magic, and
a sco with the %%sco magic as well.
"""
if slots[0] == None:
slots[0] = ctcsound.Csound()
cs = slots[0]
ip = get_ipython()
orc = ip.user_ns["__orc"][orcName]
ret = cs.compileOrc(orc)
if ret != ctcsound.CSOUND_SUCCESS:
return 'Error in orchestra'
sco = ip.user_ns["__sco"][scoName]
ret = cs.readScore(sco)
if ret != ctcsound.CSOUND_SUCCESS:
return 'Error in score'
cs.start()
cs.perform()
cs.reset()
return 'OK'
def _get_notebook_name(verbose=False) -> Optional[str]:
"""
Return the full path of the jupyter notebook.
See https://github.com/jupyter/notebook/issues/1000
In some situations (e.g. running on the command line via nbconvert),
the notebook name is not available. We return None in those cases.
"""
# kernel_id = re.search('kernel-(.*).json',
# ipykernel.connect.get_connection_file()).group(1)
try:
ipy = get_ipython()
info = ipy.ev("DWS_JUPYTER_INFO")
return info.notebook_path
except Exception as e:
if verbose:
print("DWS Jupyter extension was not loaded: %s" % e)
try:
connection_file = ipykernel.connect.get_connection_file()
mo = re.search('kernel-(.*).json', connection_file)
if mo is not None:
kernel_id = mo.group(1)
servers = list_running_servers()
for ss in servers:
response = requests.get(urljoin(ss['url'], 'api/sessions'),
params={'token': ss.get('token', '')})
for nn in json.loads(response.text):
if nn['kernel']['id'] == kernel_id:
relative_path = nn['notebook']['path']
return join(ss['notebook_dir'], relative_path)
print("Did not find a matching notebook server for %s" %
connection_file)
return None
except Exception as e:
if verbose:
print("Unable to use notebook API to access session info: %s" % e)
# all our atempts failed
return None
def debugfile(filename=None,
args=None,
wdir=None,
post_mortem=False,
current_namespace=False):
"""
Debug filename
args: command line arguments (string)
wdir: working directory
post_mortem: boolean, included for compatiblity with runfile
"""
# Tell IPython to hide this frame (>7.16)
__tracebackhide__ = True
if filename is None:
filename = get_current_file_name()
if filename is None:
return
shell = get_ipython()
if shell.is_debugging():
# Recursive
code = ("runfile({}".format(repr(normalise_filename(filename))) +
", args=%r, wdir=%r, current_namespace=%r)" %
(args, wdir, current_namespace))
shell.pdb_session.enter_recursive_debugger(
code,
filename,
True,
)
else:
debugger = get_new_debugger(filename, True)
_exec_file(filename=debugger.canonic(filename),
args=args,
wdir=wdir,
current_namespace=current_namespace,
exec_fun=debugger.run,
stack_depth=1)
def notify_collect(self, line='', cell=None, local_ns=None):
args = magic_arguments.parse_argstring(self.notify_collect, line)
self.__validate_args(args) #validate params
original_out = execute_cell_get_output(get_ipython(), line, cell,
local_ns)
attachment = None
if original_out is not None:
if hasattr(original_out, "to_html"):
attachment = Attachment(type="table",
content=self.build_final_html(
args.body.replace("\"", ""),
original_out.to_html(
index=False,
bold_rows=True,
justify="left")))
elif original_out.__class__.__name__ == 'AxesSubplot':
image_uuid = uuid.uuid1()
original_out.figure.savefig('/tmp/{0}.png'.format(image_uuid))
attachment = Attachment(type="image",
content={
'path':
'/tmp/{0}.png'.format(image_uuid),
'id':
image_uuid,
'message':
args.body.replace("\"", "")
})
else:
attachment = Attachment(type="text", content=str(original_out))
mail_keys = args.kind + '_' + args.destination.replace(
"\"", "") + '_' + args.header.replace("\"", "")
if mail_keys in self.collector:
self.collector[mail_keys].append(attachment)
else:
self.collector[mail_keys] = []
self.collector[mail_keys].append(attachment)
return original_out
def test_display_handle():
ip = get_ipython()
handle = display.DisplayHandle()
if sys.version_info < (3, ):
nt.assert_is_instance(handle.display_id, unicode)
else:
nt.assert_is_instance(handle.display_id, str)
handle = display.DisplayHandle('my-id')
nt.assert_equal(handle.display_id, 'my-id')
with mock.patch.object(ip.display_pub, 'publish') as pub:
handle.display('x')
handle.update('y')
args, kwargs = pub.call_args_list[0]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs, {
'data': {
'text/plain': repr('x')
},
'metadata': {},
'transient': {
'display_id': handle.display_id,
}
})
args, kwargs = pub.call_args_list[1]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs, {
'data': {
'text/plain': repr('y')
},
'metadata': {},
'transient': {
'display_id': handle.display_id,
},
'update': True,
})
def interaction(self, frame, traceback):
"""
Called when a user interaction is required.
If this is from sigint, break on the upper frame.
If the frame is in spydercustomize.py, quit.
Notifies spyder and print current code.
"""
if self._pdb_breaking:
self._pdb_breaking = False
if frame and frame.f_back:
return self.interaction(frame.f_back, traceback)
if (frame is not None
and "spydercustomize.py" in frame.f_code.co_filename
and "exec_code" == frame.f_code.co_name):
self.onecmd('exit')
else:
self.setup(frame, traceback)
if get_ipython().kernel._pdb_print_code:
self.print_stack_entry(self.stack[self.curindex])
self._cmdloop()
self.forget()
def call_f(self, *args):
self.kwargs = {}
if self.manual:
manual_button.disabled = True
try:
for widget in self.kwargs_widgets:
value = widget.value
self.kwargs[widget._kwarg] = value
with self.out:
if self.clear_output:
clear_output(wait=True)
self.result = self.f(**self.kwargs)
if self.result is not None:
display(self.result)
except Exception as e:
ip = get_ipython()
if ip is None:
self.log.warn("Exception in interact callback: %s", e, exc_info=True)
else:
ip.showtraceback()
finally:
if self.manual:
manual_button.disabled = False
def do_debug(self, arg):
"""
Debug code
Enter a recursive debugger that steps through the code
argument (which is an arbitrary expression or statement to be
executed in the current environment).
"""
try:
super(SpyderPdb, self).do_debug(arg)
except Exception:
if PY2:
t, v = sys.exc_info()[:2]
if type(t) == type(''):
exc_type_name = t
else:
exc_type_name = t.__name__
print >> self.stdout, '***', exc_type_name + ':', v
else:
exc_info = sys.exc_info()[:2]
self.error(
traceback.format_exception_only(*exc_info)[-1].strip())
get_ipython().pdb_session = self
def pd_stack_arrays_in_html(self):
"""
When displaying df's as html, display arrays within cells as vertically stacked values (inspired by bq web UI)
- http://ipython.readthedocs.io/en/stable/api/generated/IPython.core.formatters.html
"""
ipy = get_ipython()
if ipy:
ipy.display_formatter.formatters['text/html'].for_type(
pd.DataFrame,
lambda df: df.apply(axis=0,
func=lambda col: col.
apply(self._format_html_df_cell)).to_html(
escape=False, # Allow html in cells
))
# TODO 'text/plain' for pd.DataFrame
# - TODO Have to self.foo(...) instead of returning... [what did I mean by this a long time ago?]
# ipy.display_formatter.formatters['text/plain'].for_type(pd.DataFrame, lambda df, self, cycle:
# df.apply(axis=0, func=lambda col:
# col.apply(lambda x:
# x if not isinstance(x, list)
# else '' if len(x) == 0
# else pd.Series(x).to_string(index=False)
# )
# ).to_string()
# )
# pd.Series doesn't have _repr_html_ or .to_html, like pd.DataFrame does
# - TODO 'text/plain' for pd.Series
# - I briefly tried and nothing happened, so I went with 'text/html' instead; do more research...
ipy.display_formatter.formatters['text/html'].for_type(
pd.Series,
lambda s:
# Use <div> instad of <pre>, since <pre> might bring along a lot of style baggage
'<div style="white-space: pre">%s</div>' %
(s.apply(self._format_any)))
def test_update_display():
ip = get_ipython()
with mock.patch.object(ip.display_pub, "publish") as pub:
with nt.assert_raises(TypeError):
display.update_display("x")
display.update_display("x", display_id="1")
display.update_display("y", display_id="2")
args, kwargs = pub.call_args_list[0]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs,
{
"data": {
"text/plain": repr("x")
},
"metadata": {},
"transient": {
"display_id": "1"
},
"update": True,
},
)
args, kwargs = pub.call_args_list[1]
nt.assert_equal(args, ())
nt.assert_equal(
kwargs,
{
"data": {
"text/plain": repr("y")
},
"metadata": {},
"transient": {
"display_id": "2"
},
"update": True,
},
)
def __enter__(self):
from IPython.core.getipython import get_ipython
from IPython.core.displaypub import CapturingDisplayPublisher
self.sys_stdout = sys.stdout
self.sys_stderr = sys.stderr
if self.display:
self.shell = get_ipython()
if self.shell is None:
self.save_display_pub = None
self.display = False
stdout = stderr = outputs = None
if self.stdout:
stdout = sys.stdout = StringIO()
if self.stderr:
stderr = sys.stderr = StringIO()
if self.display:
self.save_display_pub = self.shell.display_pub
self.shell.display_pub = CapturingDisplayPublisher()
outputs = self.shell.display_pub.outputs
return CapturedIO(stdout, stderr, outputs)
def warn_format_error(method, self, *args, **kwargs):
"""decorator for warning on failed format call"""
try:
r = method(self, *args, **kwargs)
except NotImplementedError:
# don't warn on NotImplementedErrors
return None
except Exception:
exc_info = sys.exc_info()
ip = get_ipython()
if ip is not None:
ip.showtraceback(exc_info)
else:
traceback.print_exception(*exc_info)
return None
if r is None or isinstance(r, self._return_type) or \
(isinstance(r, tuple) and r and isinstance(r[0], self._return_type)):
return r
else:
warnings.warn(
"%s formatter returned invalid type %s (expected %s) for object: %s" % \
(self.format_type, type(r), self._return_type, _safe_repr(args[0])),
FormatterWarning
)
def set_mpl_inline_bbox_inches(self, bbox_inches):
"""
Set inline print figure bbox inches.
The change is done by updating the 'print_figure_kwargs' config dict.
"""
from IPython.core.getipython import get_ipython
config = get_ipython().kernel.config
inline_config = (config['InlineBackend']
if 'InlineBackend' in config else {})
print_figure_kwargs = (inline_config['print_figure_kwargs'] if
'print_figure_kwargs' in inline_config else {})
bbox_inches_dict = {'bbox_inches': 'tight' if bbox_inches else None}
print_figure_kwargs.update(bbox_inches_dict)
# This seems to be necessary for newer versions of Traitlets because
# print_figure_kwargs doesn't return a dict.
if isinstance(print_figure_kwargs, LazyConfigValue):
figure_kwargs_dict = print_figure_kwargs.to_dict().get('update')
if figure_kwargs_dict:
print_figure_kwargs = figure_kwargs_dict
self._set_config_option('InlineBackend.print_figure_kwargs',
print_figure_kwargs)
def start(self):
if not self.debugpy_initialized:
tmp_dir = get_tmp_directory()
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
host, port = self.debugpy_client.get_host_port()
code = 'import debugpy;'
code += 'debugpy.listen(("' + host + '",' + port + '))'
content = {'code': code, 'silent': True}
self.session.send(self.shell_socket, 'execute_request', content,
None, (self.shell_socket.getsockopt(ROUTING_ID)))
ident, msg = self.session.recv(self.shell_socket, mode=0)
self.debugpy_initialized = msg['content']['status'] == 'ok'
# Don't remove leading empty lines when debugging so the breakpoints are correctly positioned
cleanup_transforms = get_ipython(
).input_transformer_manager.cleanup_transforms
if leading_empty_lines in cleanup_transforms:
index = cleanup_transforms.index(leading_empty_lines)
self._removed_cleanup[index] = cleanup_transforms.pop(index)
self.debugpy_client.connect_tcp_socket()
return self.debugpy_initialized
