def parse_args(args):
""" Given arg string 'CMD files... target', return ([files], target) """
tup = args.split(None, 1)
if len(tup) == 1:
raise UsageError("Expected arguments for " + tup[0])
tup2 = shlex.split(tup[1])
flist, trg = mglob.expand(tup2[0:-1]), tup2[-1]
if not flist:
raise UsageError("No files found:" + str(tup2[0:-1]))
return flist, trg
def magic_lprun(self, parameter_s=''):
""" Execute a statement under the line-by-line profiler from the
line_profiler module.
Usage:
%lprun -f func1 -f func2 <statement>
The given statement (which doesn't require quote marks) is run via the
LineProfiler. Profiling is enabled for the functions specified by the -f
options. The statistics will be shown side-by-side with the code through the
pager once the statement has completed.
Options:
-f <function>: LineProfiler only profiles functions and methods it is told
to profile. This option tells the profiler about these functions. Multiple
-f options may be used. The argument may be any expression that gives
a Python function or method object. However, one must be careful to avoid
spaces that may confuse the option parser. Additionally, functions defined
in the interpreter at the In[] prompt or via %run currently cannot be
displayed. Write these functions out to a separate file and import them.
One or more -f options are required to get any useful results.
-D <filename>: dump the raw statistics out to a pickle file on disk. The
usual extension for this is ".lprof". These statistics may be viewed later
by running line_profiler.py as a script.
-T <filename>: dump the text-formatted statistics with the code side-by-side
out to a text file.
-r: return the LineProfiler object after it has completed profiling.
"""
# Local import to avoid hard dependency.
from IPython.genutils import page
from IPython.ipstruct import Struct
from IPython.ipapi import UsageError
# Escape quote markers.
opts_def = Struct(D=[''], T=[''], f=[])
parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")
opts, arg_str = self.parse_options(parameter_s, 'rf:D:T:', list_all=True)
opts.merge(opts_def)
global_ns = self.shell.user_global_ns
local_ns = self.shell.user_ns
# Get the requested functions.
funcs = []
for name in opts.f:
try:
funcs.append(eval(name, global_ns, local_ns))
except Exception, e:
raise UsageError('Could not find function %r.\n%s: %s' %
(name, e.__class__.__name__, e))
def irm(ip,arg):
""" irm path[s]...
Remove file[s] or dir[s] path. Dirs are deleted recursively.
"""
try:
paths = mglob.expand(arg.split(None,1)[1])
except IndexError:
raise UsageError("%irm paths...")
import distutils.dir_util
for p in paths:
print "rm",p
if os.path.isdir(p):
distutils.dir_util.remove_tree(p, verbose = 1)
else:
os.remove(p)
def mprun(self, parameter_s='', cell=None):
""" Execute a statement under the line-by-line memory profiler from the
memory_profiler module.
Usage, in line mode:
%mprun -f func1 -f func2 <statement>
Usage, in cell mode:
%%mprun -f func1 -f func2 [statement]
code...
code...
In cell mode, the additional code lines are appended to the (possibly
empty) statement in the first line. Cell mode allows you to easily
profile multiline blocks without having to put them in a separate
function.
The given statement (which doesn't require quote marks) is run via the
LineProfiler. Profiling is enabled for the functions specified by the -f
options. The statistics will be shown side-by-side with the code through
the pager once the statement has completed.
Options:
-f <function>: LineProfiler only profiles functions and methods it is told
to profile. This option tells the profiler about these functions. Multiple
-f options may be used. The argument may be any expression that gives
a Python function or method object. However, one must be careful to avoid
spaces that may confuse the option parser. Additionally, functions defined
in the interpreter at the In[] prompt or via %run currently cannot be
displayed. Write these functions out to a separate file and import them.
One or more -f options are required to get any useful results.
-T <filename>: dump the text-formatted statistics with the code
side-by-side out to a text file.
-r: return the LineProfiler object after it has completed profiling.
-c: If present, add the memory usage of any children process to the report.
"""
from io import StringIO
from memory_profiler import show_results, LineProfiler
# Local imports to avoid hard dependency.
from distutils.version import LooseVersion
import IPython
ipython_version = LooseVersion(IPython.__version__)
if ipython_version < '0.11':
from IPython.genutils import page
from IPython.ipstruct import Struct
from IPython.ipapi import UsageError
else:
from IPython.core.page import page
from IPython.utils.ipstruct import Struct
from IPython.core.error import UsageError
# Escape quote markers.
opts_def = Struct(T=[''], f=[])
parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")
opts, arg_str = self.parse_options(parameter_s, 'rf:T:c',
list_all=True)
opts.merge(opts_def)
global_ns = self.shell.user_global_ns
local_ns = self.shell.user_ns
if cell is not None:
arg_str += '\n' + cell
# Get the requested functions.
funcs = []
for name in opts.f:
try:
funcs.append(eval(name, global_ns, local_ns))
except Exception as e:
raise UsageError('Could not find function %r.\n%s: %s' % (name,
e.__class__.__name__,
e))
include_children = 'c' in opts
profile = LineProfiler(include_children=include_children)
for func in funcs:
profile(func)
# Add the profiler to the builtins for @profile.
if 'profile' in builtins.__dict__:
had_profile = True
old_profile = builtins.__dict__['profile']
else:
had_profile = False
old_profile = None
builtins.__dict__['profile'] = profile
try:
profile.runctx(arg_str, global_ns, local_ns)
message = ''
except SystemExit:
message = "*** SystemExit exception caught in code being profiled."
except KeyboardInterrupt:
message = ("*** KeyboardInterrupt exception caught in code being "
"profiled.")
finally:
if had_profile:
builtins.__dict__['profile'] = old_profile
# Trap text output.
stdout_trap = StringIO()
show_results(profile, stdout_trap)
output = stdout_trap.getvalue()
output = output.rstrip()
if ipython_version < '0.11':
page(output, screen_lines=self.shell.rc.screen_length)
else:
page(output)
print(message, )
text_file = opts.T[0]
if text_file:
with open(text_file, 'w') as pfile:
pfile.write(output)
print('\n*** Profile printout saved to text file %s. %s' % (
text_file,
message))
return_value = None
if 'r' in opts:
return_value = profile
return return_value
def magic_store(self, parameter_s=''):
"""Lightweight persistence for python variables.
Example:
ville@badger[~]|1> A = ['hello',10,'world']\\
ville@badger[~]|2> %store A\\
ville@badger[~]|3> Exit
(IPython session is closed and started again...)
ville@badger:~$ ipython -p pysh\\
ville@badger[~]|1> print A
['hello', 10, 'world']
Usage:
%store - Show list of all variables and their current values\\
%store <var> - Store the *current* value of the variable to disk\\
%store -d <var> - Remove the variable and its value from storage\\
%store -z - Remove all variables from storage\\
%store -r - Refresh all variables from store (delete current vals)\\
%store foo >a.txt - Store value of foo to new file a.txt\\
%store foo >>a.txt - Append value of foo to file a.txt\\
It should be noted that if you change the value of a variable, you
need to %store it again if you want to persist the new value.
Note also that the variables will need to be pickleable; most basic
python types can be safely %stored.
Also aliases can be %store'd across sessions.
"""
opts, argsl = self.parse_options(parameter_s, 'drz', mode='string')
args = argsl.split(None, 1)
ip = self.getapi()
db = ip.db
# delete
if opts.has_key('d'):
try:
todel = args[0]
except IndexError:
raise UsageError('You must provide the variable to forget')
else:
try:
del db['autorestore/' + todel]
except:
raise UsageError("Can't delete variable '%s'" % todel)
# reset
elif opts.has_key('z'):
for k in db.keys('autorestore/*'):
del db[k]
elif opts.has_key('r'):
refresh_variables(ip)
# run without arguments -> list variables & values
elif not args:
vars = self.db.keys('autorestore/*')
vars.sort()
if vars:
size = max(map(len, vars))
else:
size = 0
print 'Stored variables and their in-db values:'
fmt = '%-' + str(size) + 's -> %s'
get = db.get
for var in vars:
justkey = os.path.basename(var)
# print 30 first characters from every var
print fmt % (justkey, repr(get(var, '<unavailable>'))[:50])
# default action - store the variable
else:
# %store foo >file.txt or >>file.txt
if len(args) > 1 and args[1].startswith('>'):
fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
if args[1].startswith('>>'):
fil = open(fnam, 'a')
else:
fil = open(fnam, 'w')
obj = ip.ev(args[0])
print "Writing '%s' (%s) to file '%s'." % (
args[0], obj.__class__.__name__, fnam)
if not isinstance(obj, basestring):
from pprint import pprint
pprint(obj, fil)
else:
fil.write(obj)
if not obj.endswith('\n'):
fil.write('\n')
fil.close()
return
# %store foo
try:
obj = ip.user_ns[args[0]]
except KeyError:
# it might be an alias
if args[0] in self.alias_table:
staliases = db.get('stored_aliases', {})
staliases[args[0]] = self.alias_table[args[0]]
db['stored_aliases'] = staliases
print "Alias stored:", args[0], self.alias_table[args[0]]
return
else:
raise UsageError("Unknown variable '%s'" % args[0])
else:
if isinstance(inspect.getmodule(obj), FakeModule):
print textwrap.dedent("""\
Warning:%s is %s
Proper storage of interactively declared classes (or instances
of those classes) is not possible! Only instances
of classes in real modules on file system can be %%store'd.
""" % (args[0], obj))
return
#pickled = pickle.dumps(obj)
self.db['autorestore/' + args[0]] = obj
print "Stored '%s' (%s)" % (args[0], obj.__class__.__name__)
def openf(self, parameter_s=''):
"""Change the current working directory.
This command automatically maintains an internal list of directories
you visit during your IPython session, in the variable _sh. The
command %dhist shows this history nicely formatted. You can also
do 'cd -<tab>' to see directory history conveniently.
Usage:
openFile 'dir': changes to directory 'dir'.
openFile -: changes to the last visited directory.
openFile -<n>: changes to the n-th directory in the directory history.
openFile --foo: change to directory that matches 'foo' in history
openFile -b <bookmark_name>: jump to a bookmark set by %bookmark
(note: cd <bookmark_name> is enough if there is no
directory <bookmark_name>, but a bookmark with the name exists.)
'cd -b <tab>' allows you to tab-complete bookmark names.
Options:
-q: quiet. Do not print the working directory after the cd command is
executed. By default IPython's cd command does print this directory,
since the default prompts do not display path information.
Note that !cd doesn't work for this purpose because the shell where
!command runs is immediately discarded after executing 'command'."""
parameter_s = parameter_s.strip()
#bkms = self.shell.persist.get("bookmarks",{})
oldcwd = os.getcwd()
numcd = re.match(r'(-)(\d+)$', parameter_s)
# jump in directory history by number
if numcd:
nn = int(numcd.group(2))
try:
ps = ip.ev('_sh[%d]' % nn)
except IndexError:
print 'The requested directory does not exist in history.'
return
else:
opts = {}
# elif parameter_s.startswith('--'):
# ps = None
# fallback = None
# pat = parameter_s[2:]
# dh = self.shell.user_ns['_sh']
# # first search only by basename (last component)
# for ent in reversed(dh):
# if pat in os.path.basename(ent) and os.path.isdir(ent):
# ps = ent
# break
#
# if fallback is None and pat in ent and os.path.isdir(ent):
# fallback = ent
#
# # if we have no last part match, pick the first full path match
# if ps is None:
# ps = fallback
#
# if ps is None:
# print "No matching entry in directory history"
# return
# else:
# opts = {}
else:
# turn all non-space-escaping backslashes to slashes,
# for c:\windows\directory\names\
parameter_s = re.sub(r'\\(?! )', '/', parameter_s)
opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
# jump to previous
if ps == '-':
try:
ps = ip.ev('_sh[-2]' % nn)
except IndexError:
raise UsageError('%cd -: No previous directory to change to.')
# # jump to bookmark if needed
# else:
# if not os.path.exists(ps) or opts.has_key('b'):
# bkms = self.db.get('bookmarks', {})
#
# if bkms.has_key(ps):
# target = bkms[ps]
# print '(bookmark:%s) -> %s' % (ps,target)
# ps = target
# else:
# if opts.has_key('b'):
# raise UsageError("Bookmark '%s' not found. "
# "Use '%%bookmark -l' to see your bookmarks." % ps)
# at this point ps should point to the target dir
if ps:
ip.ex('openFile("%s", f=1)' % ps)
def magic_lprun(self, parameter_s=''):
""" Execute a statement under the line-by-line profiler from the
line_profiler module.
Usage:
%lprun -f func1 -f func2 <statement>
The given statement (which doesn't require quote marks) is run via the
LineProfiler. Profiling is enabled for the functions specified by the -f
options. The statistics will be shown side-by-side with the code through the
pager once the statement has completed.
Options:
-f <function>: LineProfiler only profiles functions and methods it is told
to profile. This option tells the profiler about these functions. Multiple
-f options may be used. The argument may be any expression that gives
a Python function or method object. However, one must be careful to avoid
spaces that may confuse the option parser. Additionally, functions defined
in the interpreter at the In[] prompt or via %run currently cannot be
displayed. Write these functions out to a separate file and import them.
-m <module>: Get all the functions/methods in a module
One or more -f or -m options are required to get any useful results.
-D <filename>: dump the raw statistics out to a pickle file on disk. The
usual extension for this is ".lprof". These statistics may be viewed later
by running line_profiler.py as a script.
-T <filename>: dump the text-formatted statistics with the code side-by-side
out to a text file.
-r: return the LineProfiler object after it has completed profiling.
-s: strip out all entries from the print-out that have zeros.
"""
# Local imports to avoid hard dependency.
from distutils.version import LooseVersion
import IPython
ipython_version = LooseVersion(IPython.__version__)
if ipython_version < '0.11':
from IPython.genutils import page
from IPython.ipstruct import Struct
from IPython.ipapi import UsageError
else:
from IPython.core.page import page
from IPython.utils.ipstruct import Struct
from IPython.core.error import UsageError
# Escape quote markers.
opts_def = Struct(D=[''], T=[''], f=[], m=[])
parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")
opts, arg_str = self.parse_options(parameter_s,
'rsf:m:D:T:',
list_all=True)
opts.merge(opts_def)
global_ns = self.shell.user_global_ns
local_ns = self.shell.user_ns
# Get the requested functions.
funcs = []
for name in opts.f:
try:
funcs.append(eval(name, global_ns, local_ns))
except Exception as e:
raise UsageError('Could not find function %r.\n%s: %s' %
(name, e.__class__.__name__, e))
profile = LineProfiler(*funcs)
# Get the modules, too
for modname in opts.m:
try:
mod = __import__(modname, fromlist=[''])
profile.add_module(mod)
except Exception as e:
raise UsageError('Could not find module %r.\n%s: %s' %
(modname, e.__class__.__name__, e))
# Add the profiler to the builtins for @profile.
if PY3:
import builtins
else:
import __builtin__ as builtins
if 'profile' in builtins.__dict__:
had_profile = True
old_profile = builtins.__dict__['profile']
else:
had_profile = False
old_profile = None
builtins.__dict__['profile'] = profile
try:
try:
profile.runctx(arg_str, global_ns, local_ns)
message = ''
except SystemExit:
message = """*** SystemExit exception caught in code being profiled."""
except KeyboardInterrupt:
message = ("*** KeyboardInterrupt exception caught in code being "
"profiled.")
finally:
if had_profile:
builtins.__dict__['profile'] = old_profile
# Trap text output.
stdout_trap = StringIO()
profile.print_stats(stdout_trap, stripzeros='s' in opts)
output = stdout_trap.getvalue()
output = output.rstrip()
if ipython_version < '0.11':
page(output, screen_lines=self.shell.rc.screen_length)
else:
page(output)
print(message, end="")
dump_file = opts.D[0]
if dump_file:
profile.dump_stats(dump_file)
print('\n*** Profile stats pickled to file %r. %s' %
(dump_file, message))
text_file = opts.T[0]
if text_file:
pfile = open(text_file, 'w')
pfile.write(output)
pfile.close()
print('\n*** Profile printout saved to text file %r. %s' %
(text_file, message))
return_value = None
if 'r' in opts:
return_value = profile
return return_value
