def getblocks_from_history(object, lstrip=False):# gettype=False):
"""extract code blocks from a code object using stored history"""
import readline, inspect, types
lbuf = readline.get_current_history_length()
code = [readline.get_history_item(i)+'\n' for i in range(1,lbuf)]
lnum = 0
codeblocks = []
#objtypes = []
while lnum < len(code):#-1:
if code[lnum].lstrip().startswith('def '):
block = inspect.getblock(code[lnum:])
lnum += len(block)
if block[0].lstrip().startswith('def %s(' % object.func_name):
if lstrip: block[0] = block[0].lstrip()
codeblocks.append(block)
# obtypes.append(types.FunctionType)
elif 'lambda ' in code[lnum]:
block = inspect.getblock(code[lnum:])
lnum += len(block)
lhs,rhs = block[0].split('lambda ',1)[-1].split(":", 1) #FIXME: bad
try: #FIXME: unsafe
_ = eval("lambda %s : %s" % (lhs, rhs), globals(), locals())
except: pass
if _.func_code.co_code == object.func_code.co_code:
if lstrip: block[0] = block[0].lstrip()
codeblocks.append(block)
# obtypes.append('<lambda>')
else:
lnum +=1
#if gettype: return codeblocks, objtypes
return codeblocks #XXX: danger... gets methods and closures w/o containers
def getblocks(object, lstrip=False):# gettype=False):
"""extract code blocks from a code object using stored history"""
import readline, inspect #, types
lbuf = readline.get_current_history_length()
code = [readline.get_history_item(i)+'\n' for i in range(1,lbuf)]
lnum = 0
codeblocks = []
#objtypes = []
try:
if PYTHON3:
fname = object.__name__
ocode = object.__code__
else:
fname = object.func_name
ocode = object.func_code
cname = ''
except AttributeError:
fname = ''
ocode = lambda :'__this_is_a_big_dummy_object__'
ocode.co_code = '__this_is_a_big_dummy_co_code__'
#try: inspect.getmro(object) #XXX: ensure that it's a class
if hasattr(object, '__name__'): cname = object.__name__ # class
else: cname = object.__class__.__name__ # instance
while lnum < len(code):#-1:
if fname and code[lnum].lstrip().startswith('def '):
# functions and methods
block = inspect.getblock(code[lnum:])
lnum += len(block)
if block[0].lstrip().startswith('def %s(' % fname):
if lstrip: block[0] = block[0].lstrip()
codeblocks.append(block)
# obtypes.append(types.FunctionType)
elif cname and code[lnum].lstrip().startswith('class '):
# classes and instances
block = inspect.getblock(code[lnum:])
lnum += len(block)
_cname = ('class %s(' % cname, 'class %s:' % cname)
if block[0].lstrip().startswith(_cname):
if lstrip: block[0] = block[0].lstrip()
codeblocks.append(block)
elif fname and 'lambda ' in code[lnum]:
# lambdas
block = inspect.getblock(code[lnum:])
lnum += len(block)
lhs,rhs = block[0].split('lambda ',1)[-1].split(":", 1) #FIXME: bad
try: #FIXME: unsafe
_ = eval("lambda %s : %s" % (lhs, rhs), globals(), locals())
except: _ = lambda : "__this_is_a_big_dummy_function__"
if PYTHON3: _ = _.__code__
else: _ = _.func_code
if _.co_code == ocode.co_code:
if lstrip: block[0] = block[0].lstrip()
codeblocks.append(block)
# obtypes.append('<lambda>')
#XXX: would be nice to grab constructor for instance, but yikes.
else:
lnum +=1
#if gettype: return codeblocks, objtypes
return codeblocks #XXX: danger... gets methods and closures w/o containers
def __prepare__(meta, name, bases):
bare_cls = meta("<Bare>", bases, SettingsDict(options=None, bare_cls=None))
frame = sys._getframe(1)
filename = inspect.getsourcefile(frame)
with tokenize.open(filename) as file:
lines = file.readlines()[frame.f_lineno - 1 :]
source = "".join(inspect.getblock(lines))
source = textwrap.dedent(source.expandtabs(tabsize=8))
cls_node = ast.parse(source).body[0]
for node in reversed(list(ast.iter_child_nodes(cls_node))):
if isinstance(node, ast.ClassDef) and node.name == "Meta":
cf_mask = sum(
getattr(__future__, feature).compiler_flag
for feature in __future__.all_feature_names
)
code = compile(
ast.Module(body=[node], type_ignores=[]),
filename="<meta>",
mode="exec",
flags=frame.f_code.co_flags & cf_mask,
dont_inherit=True,
)
globals = frame.f_globals
locals = {}
exec(code, globals, locals)
meta = locals["Meta"]
break
else:
meta = getattr(bare_cls, "Meta", None)
options = Options(meta)
bare_cls._options = options
return SettingsDict(options=options, bare_cls=bare_cls)
def getsourcelines(object):
"""Return a list of source lines and starting line number for an object.
Parameters
----------
object : a python object
The argument may be a module, class, method, function, traceback, frame,
or code object. The source code is returned as a string with the lines
corresponding to the object and the line number indicates where in the
original source file the first line of code was found. An IOError is
raised if the source code cannot be retrieved."""
lines, lnum = inspect.findsource(object)
if inspect.ismodule(object):
lnum = 0
ss = ''
for x in lines:
ss += x
else:
lines = inspect.getblock(lines[lnum:])
lnum = lnum + 1
ss = ''
for x in lines:
ss += x
return ss, 0
def get_annotated_lines(self):
"""Helper function that returns lines with extra information."""
lines = [Line(idx + 1, x) for idx, x in enumerate(self.sourcelines)]
# find function definition and mark lines
if hasattr(self.code, 'co_firstlineno'):
lineno = self.code.co_firstlineno - 1
while lineno > 0:
if _funcdef_re.match(lines[lineno].code):
break
lineno -= 1
try:
offset = len(inspect.getblock([x.code + '\n' for x
in lines[lineno:]]))
except TokenError:
offset = 0
for line in lines[lineno:lineno + offset]:
line.in_frame = True
# mark current line
try:
lines[self.lineno - 1].current = True
except IndexError:
pass
return lines
def get_annotated_lines(self):
"""Helper function that returns lines with extra information."""
lines = [Line(idx + 1, x) for idx, x in enumerate(self.sourcelines)]
# find function definition and mark lines
if hasattr(self.code, "co_firstlineno"):
lineno = self.code.co_firstlineno - 1
while lineno > 0:
if _funcdef_re.match(lines[lineno].code):
break
lineno -= 1
try:
offset = len(
inspect.getblock([x.code + "\n" for x in lines[lineno:]]))
except TokenError:
offset = 0
for line in lines[lineno:lineno + offset]:
line.in_frame = True
# mark current line
try:
lines[self.lineno - 1].current = True
except IndexError:
pass
return lines
def show_results(prof, stream=None, precision=3):
if stream is None:
stream = sys.stdout
template = '{0:>6} {1:>12} {2:>12} {3:<}'
for code in prof.code_map:
lines = prof.code_map[code]
if not lines:
# .. measurements are empty ..
continue
filename = code.co_filename
if filename.endswith((".pyc", ".pyo")):
filename = filename[:-1]
stream.write('Filename: ' + filename + '\n\n')
if not os.path.exists(filename):
stream.write('ERROR: Could not find file ' + filename + '\n')
if filename.startswith("ipython-input") or filename.startswith("<ipython-input"):
print(
"NOTE: %mprun can only be used on functions defined in "
"physical files, and not in the IPython environment."
)
continue
all_lines = linecache.getlines(filename)
sub_lines = inspect.getblock(all_lines[code.co_firstlineno - 1 :])
linenos = range(code.co_firstlineno, code.co_firstlineno + len(sub_lines))
lines_normalized = {}
header = template.format('Line #', 'Mem usage', 'Increment', 'Line Contents')
stream.write(header + '\n')
stream.write('=' * len(header) + '\n')
# move everything one frame up
keys = sorted(lines.keys())
k_old = keys[0] - 1
lines_normalized[keys[0] - 1] = lines[keys[0]]
for i in range(1, len(lines_normalized[keys[0] - 1])):
lines_normalized[keys[0] - 1][i] = -1.0
k = keys.pop(0)
while keys:
lines_normalized[k] = lines[keys[0]]
for i in range(len(lines_normalized[k_old]), len(lines_normalized[k])):
lines_normalized[k][i] = -1.0
k_old = k
k = keys.pop(0)
first_line = sorted(lines_normalized.keys())[0]
mem_old = max(lines_normalized[first_line])
precision = int(precision)
template_mem = '{{0:{0}.{1}'.format(precision + 6, precision) + 'f} MB'
for i, l in enumerate(linenos):
mem = ''
inc = ''
if l in lines_normalized:
mem = max(lines_normalized[l])
inc = mem - mem_old
mem_old = mem
mem = template_mem.format(mem)
inc = template_mem.format(inc)
stream.write(template.format(l, mem, inc, sub_lines[i]))
stream.write('\n\n')
def load_data(self, profdatafile):
"""Load line profiler data saved by kernprof.py module"""
# lstats has the following layout :
# lstats.timings =
# {(filename1, line_no1, function_name1):
# [(line_no1, hits1, total_time1),
# (line_no2, hits2, total_time2)],
# (filename2, line_no2, function_name2):
# [(line_no1, hits1, total_time1),
# (line_no2, hits2, total_time2),
# (line_no3, hits3, total_time3)]}
# lstats.unit = time_factor
with open(profdatafile, 'rb') as fid:
lstats = cPickle.load(fid)
# First pass to group by filename
self.stats = dict()
linecache.checkcache()
for func_info, stats in lstats.timings.iteritems():
# func_info is a tuple containing (filename, line, function anme)
filename, start_line_no = func_info[:2]
filename = filename.decode('utf8')
# Read code
start_line_no -= 1 # include the @profile decorator
all_lines = linecache.getlines(filename)
block_lines = inspect.getblock(all_lines[start_line_no:])
# Loop on each line of code
func_stats = []
func_total_time = 0.0
next_stat_line = 0
for line_no, code_line in enumerate(block_lines):
line_no += start_line_no + 1 # Lines start at 1
code_line = code_line.rstrip('\n').decode('utf8')
if (next_stat_line >= len(stats)
or line_no != stats[next_stat_line][0]):
# Line didn't run
hits, line_total_time, time_per_hit = None, None, None
else:
# Compute line times
hits, line_total_time = stats[next_stat_line][1:]
line_total_time *= lstats.unit
time_per_hit = line_total_time / hits
func_total_time += line_total_time
next_stat_line += 1
func_stats.append(
[line_no, code_line, line_total_time, time_per_hit,
hits])
# Compute percent time
for line in func_stats:
line_total_time = line[2]
if line_total_time is None:
line.append(None)
else:
line.append(line_total_time / func_total_time)
# Fill dict
self.stats[func_info] = [func_stats, func_total_time]
def _extract_source(lines, lineno):
r"""
Given a list of lines or a multiline string and a starting lineno,
_extract_source returns [source_lines]. [source_lines] is the smallest
indentation block starting at lineno.
INPUT:
- ``lines`` - string or list of strings
- ``lineno`` - positive integer
EXAMPLES::
sage: from sagenb.misc.sageinspect import _extract_source
sage: s2 = "#hello\n\n class f():\n pass\n\n#goodbye"
sage: _extract_source(s2, 3)
[' class f():\n', ' pass\n']
"""
if lineno < 1:
raise ValueError, "Line numbering starts at 1! (tried to extract line %s)" % lineno
lineno -= 1
if isinstance(lines, str):
lines = lines.splitlines(True) # true keeps the '\n'
if len(lines) > 0:
# Fixes an issue with getblock
lines[-1] += '\n'
return inspect.getblock(lines[lineno:])
def getsourcelines(obj):
(lines, lineno) = inspect.findsource(obj)
if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
return (lines, 1)
if inspect.ismodule(obj):
return (lines, 1)
return (inspect.getblock(lines[lineno:]), lineno + 1)
def render_source(self):
"""Render the sourcecode."""
lines = [Line(idx + 1, x) for idx, x in enumerate(self.sourcelines)]
# find function definition and mark lines
if hasattr(self.code, 'co_firstlineno'):
lineno = self.code.co_firstlineno - 1
while lineno > 0:
if _funcdef_re.match(lines[lineno].code):
break
lineno -= 1
try:
offset = len(inspect.getblock([x.code + '\n' for x
in lines[lineno:]]))
except TokenError:
offset = 0
for line in lines[lineno:lineno + offset]:
line.in_frame = True
# mark current line
try:
lines[self.lineno - 1].current = True
except IndexError:
pass
return render_template('source.html', frame=self, lines=lines)
def show_results(prof, stream=None):
if stream is None:
stream = sys.stdout
template = '%6s %12s %-s'
header = template % ('Line #', 'Mem usage', 'Line Contents')
stream.write(header + '\n')
stream.write('=' * len(header) + '\n')
for code in prof.code_map:
lines = prof.code_map[code]
filename = code.co_filename
if (filename.endswith(".pyc") or
filename.endswith(".pyo")):
filename = filename[:-1]
all_lines = linecache.getlines(filename)
sub_lines = inspect.getblock(all_lines[code.co_firstlineno-1:])
linenos = range(code.co_firstlineno, code.co_firstlineno + len(sub_lines))
lines_normalized = {}
# move everything one frame up
keys = lines.keys()
keys.sort()
lines_normalized[code.co_firstlineno+1] = lines[keys[0]]
while len(keys) > 1:
v = keys.pop(0)
lines_normalized[v] = lines[keys[0]]
for l in linenos:
mem = ''
if lines_normalized.has_key(l):
mem = '%5.2f MB' % max(lines_normalized.get(l))
line = linecache.getline(filename, l)
stream.write(template % (l, mem, line))
def getsourcelines(obj):
(lines, lineno) = inspect.findsource(obj)
if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
return (lines, 1)
if inspect.ismodule(obj):
return (lines, 1)
return (inspect.getblock(lines[lineno:]), lineno + 1)
def getsourcelines(object):
"""Return a list of source lines and starting line number for an object.
Parameters
----------
object : a python object
The argument may be a module, class, method, function, traceback, frame,
or code object. The source code is returned as a string with the lines
corresponding to the object and the line number indicates where in the
original source file the first line of code was found. An IOError is
raised if the source code cannot be retrieved."""
lines, lnum = inspect.findsource(object)
if inspect.ismodule(object):
lnum = 0
ss = ''
for x in lines:
ss += x
else:
lines = inspect.getblock(lines[lnum:])
lnum = lnum + 1
ss = ''
for x in lines:
ss += x
return ss, 0
def get_func_real_firstlineno(func):
start_lineno = 0
lines = linecache.getlines(func.__code__.co_filename)
cls = _findclass(func)
if cls:
lines, lnum = findsource(cls)
lines = getblock(lines[lnum:])
start_lineno = lnum
# referenced from inspect _findclass
pat = re.compile(r'^(\s*)def\s*' + func.__name__ + r'\b')
candidates = []
for i in range(len(lines)):
match = pat.match(lines[i])
if match:
# if it's at toplevel, it's already the best one
if lines[i][0] == 'd':
return (i + start_lineno)
# else add whitespace to candidate list
candidates.append((match.group(1), i))
if candidates:
# this will sort by whitespace, and by line number,
# less whitespace first
candidates.sort()
return start_lineno + candidates[0][1]
else:
raise OSError('could not find function definition')
def render_source(self):
"""Render the sourcecode."""
lines = [Line(idx + 1, x) for idx, x in enumerate(self.sourcelines)]
# find function definition and mark lines
if hasattr(self.code, 'co_firstlineno'):
lineno = self.code.co_firstlineno - 1
while lineno > 0:
if _funcdef_re.match(lines[lineno].code):
break
lineno -= 1
try:
offset = len(
inspect.getblock([x.code + '\n' for x in lines[lineno:]]))
except TokenError:
offset = 0
for line in lines[lineno:lineno + offset]:
line.in_frame = True
# mark current line
try:
lines[self.lineno - 1].current = True
except IndexError:
pass
return render_template('source.html', frame=self, lines=lines)
def _get_source(function):
"""
Custom parser because built-in of Python fails sometimes.
This function parses student-solution sourcecode.
"""
name = function.__name__
file = getsourcefile(function)
# Get lines
linecache.checkcache(file)
module = getmodule(function, file)
lines = linecache.getlines(file, module.__dict__)
# Parse lines
pat = re.compile("(def {}\()".format(name)) # Begin of function pattern.
for lnum, line in enumerate(lines):
if pat.match(line):
break
src = getblock(lines[lnum:])
src = "".join(src)
return src
def load_data(self, profdatafile):
"""Load line profiler data saved by kernprof module"""
# lstats has the following layout :
# lstats.timings =
# {(filename1, line_no1, function_name1):
# [(line_no1, hits1, total_time1),
# (line_no2, hits2, total_time2)],
# (filename2, line_no2, function_name2):
# [(line_no1, hits1, total_time1),
# (line_no2, hits2, total_time2),
# (line_no3, hits3, total_time3)]}
# lstats.unit = time_factor
with open(profdatafile, 'rb') as fid:
lstats = pickle.load(fid)
# First pass to group by filename
self.stats = dict()
linecache.checkcache()
for func_info, stats in lstats.timings.items():
# func_info is a tuple containing (filename, line, function anme)
filename, start_line_no = func_info[:2]
# Read code
start_line_no -= 1 # include the @profile decorator
all_lines = linecache.getlines(filename)
block_lines = inspect.getblock(all_lines[start_line_no:])
# Loop on each line of code
func_stats = []
func_total_time = 0.0
next_stat_line = 0
for line_no, code_line in enumerate(block_lines):
line_no += start_line_no + 1 # Lines start at 1
code_line = code_line.rstrip('\n')
if (next_stat_line >= len(stats)
or line_no != stats[next_stat_line][0]):
# Line didn't run
hits, line_total_time, time_per_hit = None, None, None
else:
# Compute line times
hits, line_total_time = stats[next_stat_line][1:]
line_total_time *= lstats.unit
time_per_hit = line_total_time / hits
func_total_time += line_total_time
next_stat_line += 1
func_stats.append(
[line_no, code_line, line_total_time, time_per_hit,
hits])
# Compute percent time
for line in func_stats:
line_total_time = line[2]
if line_total_time is None:
line.append(None)
else:
line.append(line_total_time / func_total_time)
# Fill dict
self.stats[func_info] = [func_stats, func_total_time]
def getsourcefallback(cls):
""" Fallback for getting the source of interactively defined classes (typically in ipython)
This is basically just a patched version of the inspect module, in which
we get the code by calling inspect.findsource on an *instancemethod* of
a class for which inspect.findsource fails.
"""
for attr in cls.__dict__:
if inspect.ismethod(getattr(cls, attr)):
imethod = getattr(cls, attr)
break
else:
raise AttributeError(
"Cannot get this class' source; it does not appear to have any methods")
### This part is derived from inspect.findsource ###
module = inspect.getmodule(cls)
file = inspect.getfile(imethod)
lines = linecache.getlines(file, module.__dict__)
name = cls.__name__
pat = re.compile(r'^(\s*)class\s*'+name+r'\b')
# AMVMOD: find the encoding (necessary for python 2 only)
#if PY2:
# with open(file, 'rb') as infile:
# encoding = detect_encoding(infile.readline)[0]
# make some effort to find the best matching class definition:
# use the one with the least indentation, which is the one
# that's most probably not inside a function definition.
candidates = []
toplevel = False
for i in range(len(lines)):
match = pat.match(lines[i])
if match:
# if it's at toplevel, it's already the best one
if lines[i][0] == 'c':
flines, flnum = lines, i
toplevel = True
break
# else add whitespace to candidate list
candidates.append((match.group(1), i))
if candidates and not toplevel:
# this will sort by whitespace, and by line number,
# less whitespace first
candidates.sort()
flines, flnum = lines, candidates[0][1]
elif not candidates and not toplevel:
raise IOError('could not find class definition')
### end modified inspect.findsource ###
# this is what inspect.getsourcelines does
glines = inspect.getblock(flines[flnum:])
# And this is what inspect.getsource does
if False: #if PY2:
return ("".join(glines)).decode(encoding)
else:
return "".join(glines)
def show_func(filename, start_lineno, func_name, timings, unit, stream=None, stripzeros=False):
""" Show results for a single function.
"""
if stream is None:
stream = sys.stdout
template = '%6s %9s %12s %8s %8s %-s'
d = {}
total_time = 0.0
linenos = []
for lineno, nhits, time in timings:
total_time += time
linenos.append(lineno)
if stripzeros and total_time == 0:
return
stream.write("Total time: %g s\n" % (total_time * unit))
if os.path.exists(filename) or filename.startswith("<ipython-input-"):
stream.write("File: %s\n" % filename)
stream.write("Function: %s at line %s\n" % (func_name, start_lineno))
if os.path.exists(filename):
# Clear the cache to ensure that we get up-to-date results.
linecache.clearcache()
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno-1:])
else:
stream.write("\n")
stream.write("Could not find file %s\n" % filename)
stream.write("Are you sure you are running this program from the same directory\n")
stream.write("that you ran the profiler from?\n")
stream.write("Continuing without the function's contents.\n")
# Fake empty lines so we can see the timings, if not the code.
if linenos:
nlines = max(linenos) - min(min(linenos), start_lineno) + 1
else:
nlines = 1
sublines = [''] * nlines
for lineno, nhits, time in timings:
d[lineno] = (nhits, time, '%5.1f' % (float(time) / nhits if nhits else 0),
'%5.1f' % (100*time / total_time if total_time else 0))
linenos = range(start_lineno, start_lineno + len(sublines))
empty = ('', '', '', '')
header = template % ('Line #', 'Hits', 'Time', 'Per Hit', '% Time',
'Line Contents')
stream.write("\n")
stream.write(header)
stream.write("\n")
stream.write('=' * len(header))
stream.write("\n")
if total_time > 0:
for lineno, line in zip(linenos, sublines):
nhits, time, per_hit, percent = d.get(lineno, empty)
txt = template % (lineno, nhits, time, per_hit, percent,
line.rstrip('\n').rstrip('\r'))
stream.write(txt)
stream.write("\n")
stream.write("\n")
def show_results(prof, stream=None):
if stream is None:
stream = sys.stdout
template = '{0:>6} {1:>12} {2:>12} {3:<}'
for code in prof.code_map:
lines = prof.code_map[code]
if not lines:
# .. measurements are empty ..
continue
filename = code.co_filename
if filename.endswith((".pyc", ".pyo")):
filename = filename[:-1]
stream.write('Filename: ' + filename + '\n\n')
if not os.path.exists(filename):
stream.write('ERROR: Could not find file ' + filename + '\n')
if filename.startswith("ipython-input"):
print("NOTE: %mprun can only be used on functions defined in "
"physical files, and not in the IPython environment.")
continue
all_lines = linecache.getlines(filename)
sub_lines = inspect.getblock(all_lines[code.co_firstlineno - 1:])
linenos = range(code.co_firstlineno, code.co_firstlineno +
len(sub_lines))
lines_normalized = {}
header = template.format('Line #', 'Mem usage', 'Increment',
'Line Contents')
stream.write(header + '\n')
stream.write('=' * len(header) + '\n')
# move everything one frame up
keys = sorted(lines.keys())
k_old = keys[0] - 1
lines_normalized[keys[0] - 1] = lines[keys[0]]
for i in range(1, len(lines_normalized[keys[0] - 1])):
lines_normalized[keys[0] - 1][i] = -1.
k = keys.pop(0)
while keys:
lines_normalized[k] = lines[keys[0]]
for i in range(len(lines_normalized[k_old]),
len(lines_normalized[k])):
lines_normalized[k][i] = -1.
k_old = k
k = keys.pop(0)
first_line = sorted(lines_normalized.keys())[0]
mem_old = max(lines_normalized[first_line])
for i, l in enumerate(linenos):
mem = ''
inc = ''
if l in lines_normalized:
mem = max(lines_normalized[l])
inc = mem - mem_old
mem_old = mem
mem = '{0:9.2f} MB'.format(mem)
inc = '{0:9.2f} MB'.format(inc)
stream.write(template.format(l, mem, inc, sub_lines[i]))
stream.write('\n\n')
def getsourcelines(self, obj):
lines, lineno = inspect.findsource(obj)
if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
# must be a module frame: do not try to cut a block out of it
return lines, 1
elif inspect.ismodule(obj):
return lines, 1
return inspect.getblock(lines[lineno:]), lineno + 1
def getsourcelines(self, obj):
lines, lineno = inspect.findsource(obj)
if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
# must be a module frame: do not try to cut a block out of it
return lines, 1
elif inspect.ismodule(obj):
return lines, 1
return inspect.getblock(lines[lineno:]), lineno + 1
def get_source_from_func(filename,first_line_num):
"see python's inspect.py"
import inspect
lines = inspect.linecache.getlines(filename)
line_num = first_line_num - 1
while line_num > 0:
if func_definition.match(lines[line_num]):
break
lnum = lnum - 1
return '\n'.join(inspect.getblock(lines[line_num:]))
def get_source_code(func, func_runtime_name, firstlineno):
lines = linecache.getlines(func.__code__.co_filename)
code_lines = getblock(lines[firstlineno:])
# repalce function name
code_lines[0] = code_lines[0].replace(func.__name__, func_runtime_name, 1)
i_indent = code_lines[0].index("def")
# code indentation
code_lines = [line[i_indent:] for line in code_lines]
code = ''.join(code_lines)
return code
def get_func_code(func):
""" Attempts to retrieve a reliable function code hash.
The reason we don't use inspect.getsource is that it caches the
source, whereas we want this to be modified on the fly when the
function is modified.
Returns
-------
func_code: string
The function code
source_file: string
The path to the file in which the function is defined.
first_line: int
The first line of the code in the source file.
Notes
------
This function does a bit more magic than inspect, and is thus
more robust.
"""
source_file = None
try:
code = func.__code__
source_file = code.co_filename
if not os.path.exists(source_file):
# Use inspect for lambda functions and functions defined in an
# interactive shell, or in doctests
source_code = ''.join(inspect.getsourcelines(func)[0])
line_no = 1
if source_file.startswith('<doctest '):
source_file, line_no = re.match(
'\<doctest (.*\.rst)\[(.*)\]\>',
source_file).groups()
line_no = int(line_no)
source_file = '<doctest %s>' % source_file
return source_code, source_file, line_no
# Try to retrieve the source code.
with open_py_source(source_file) as source_file_obj:
first_line = code.co_firstlineno
# All the lines after the function definition:
source_lines = list(islice(source_file_obj, first_line - 1, None))
return ''.join(inspect.getblock(source_lines)), source_file, first_line
except:
# If the source code fails, we use the hash. This is fragile and
# might change from one session to another.
if hasattr(func, '__code__'):
# Python 3.X
return str(func.__code__.__hash__()), source_file, -1
else:
# Weird objects like numpy ufunc don't have __code__
# This is fragile, as quite often the id of the object is
# in the repr, so it might not persist across sessions,
# however it will work for ufuncs.
return repr(func), source_file, -1
def get_func_code(func):
""" Attempts to retrieve a reliable function code hash.
The reason we don't use inspect.getsource is that it caches the
source, whereas we want this to be modified on the fly when the
function is modified.
Returns
-------
func_code: string
The function code
source_file: string
The path to the file in which the function is defined.
first_line: int
The first line of the code in the source file.
Notes
------
This function does a bit more magic than inspect, and is thus
more robust.
"""
source_file = None
try:
code = func.__code__
source_file = code.co_filename
if not os.path.exists(source_file):
# Use inspect for lambda functions and functions defined in an
# interactive shell, or in doctests
source_code = ''.join(inspect.getsourcelines(func)[0])
line_no = 1
if source_file.startswith('<doctest '):
source_file, line_no = re.match(
'\<doctest (.*\.rst)\[(.*)\]\>',
source_file).groups()
line_no = int(line_no)
source_file = '<doctest %s>' % source_file
return source_code, source_file, line_no
# Try to retrieve the source code.
with open_py_source(source_file) as source_file_obj:
first_line = code.co_firstlineno
# All the lines after the function definition:
source_lines = list(islice(source_file_obj, first_line - 1, None))
return ''.join(inspect.getblock(source_lines)), source_file, first_line
except:
# If the source code fails, we use the hash. This is fragile and
# might change from one session to another.
if hasattr(func, '__code__'):
# Python 3.X
return str(func.__code__.__hash__()), source_file, -1
else:
# Weird objects like numpy ufunc don't have __code__
# This is fragile, as quite often the id of the object is
# in the repr, so it might not persist across sessions,
# however it will work for ufuncs.
return repr(func), source_file, -1
def show_func(filename, start_lineno, func_name, timings, unit, stream=None, stripzeros=False):
""" Show results for a single function.
"""
if stream is None:
stream = sys.stdout
template = '%6s %9s %12s %8s %8s %-s'
d = {}
total_time = 0.0
linenos = []
for lineno, nhits, time in timings:
total_time += time
linenos.append(lineno)
if stripzeros and total_time == 0:
return
stream.write("Total time: %g s\n" % (total_time * unit))
if os.path.exists(filename) or filename.startswith("<ipython-input-"):
stream.write("File: %s\n" % filename)
stream.write("Function: %s at line %s\n" % (func_name, start_lineno))
if os.path.exists(filename):
# Clear the cache to ensure that we get up-to-date results.
linecache.clearcache()
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno-1:])
else:
stream.write("\n")
stream.write("Could not find file %s\n" % filename)
stream.write("Are you sure you are running this program from the same directory\n")
stream.write("that you ran the profiler from?\n")
stream.write("Continuing without the function's contents.\n")
# Fake empty lines so we can see the timings, if not the code.
nlines = max(linenos) - min(min(linenos), start_lineno) + 1
sublines = [''] * nlines
for lineno, nhits, time in timings:
d[lineno] = (nhits, time, '%5.1f' % (float(time) / nhits),
'%5.1f' % (100*time / total_time))
linenos = range(start_lineno, start_lineno + len(sublines))
empty = ('', '', '', '')
header = template % ('Line #', 'Hits', 'Time', 'Per Hit', '% Time',
'Line Contents')
stream.write("\n")
stream.write(header)
stream.write("\n")
stream.write('=' * len(header))
stream.write("\n")
for lineno, line in zip(linenos, sublines):
nhits, time, per_hit, percent = d.get(lineno, empty)
txt = template % (lineno, nhits, time, per_hit, percent,
line.rstrip('\n').rstrip('\r'))
stream.write(txt)
stream.write("\n")
stream.write("\n")
def __init__(self, *args, **kwds):
self._long_opts = kwds.get('long_opts', {})
self._short_opts = kwds.get('short_opts', {})
f = self.__filter
c = self.__convert
p = self.__pattern
classes = dict(
(c(t), getattr(self, n)) for (n, t) in [
(n, p.findall(n)) for n in filter(f, dir(self))
]
)
names = dict((v.__name__, k) for (k, v) in classes.items())
cls = self.__class__
classname = cls.__name__
filename = inspect.getsourcefile(cls)
lines = linecache.getlines(filename)
lines_len = len(lines)
prefix = 'class %s(' % classname
found = False
for i in range(lines_len):
line = lines[i]
if prefix in line:
found = i
break
if not found:
raise IOError('could not find source code for class')
block = inspect.getblock(lines[found:])
text = ''.join(block)
inner = self.__inner_classes_pattern
order = [
(i, names[n[0]]) for (i, n) in enumerate(inner.findall(text))
]
instances = {
name: cls(self, name)
for (cls, name) in [
(classes[name], name)
for (_, name) in order
]
}
self._invariants = instances
self._invariant_names = names
self._invariant_order = order
self._invariant_classes = classes
self._invariants_processed = list()
def __init__(self, *args, **kwds):
self._long_opts = kwds.get('long_opts', {})
self._short_opts = kwds.get('short_opts', {})
f = self.__filter
c = self.__convert
p = self.__pattern
classes = dict(
(c(t), getattr(self, n)) for (n, t) in [
(n, p.findall(n)) for n in filter(f, dir(self))
]
)
names = dict((v.__name__, k) for (k, v) in classes.items())
cls = self.__class__
classname = cls.__name__
filename = inspect.getsourcefile(cls)
lines = linecache.getlines(filename)
lines_len = len(lines)
prefix = 'class %s(' % classname
found = False
for i in range(lines_len):
line = lines[i]
if prefix in line:
found = i
break
if not found:
raise IOError('could not find source code for class')
block = inspect.getblock(lines[found:])
text = ''.join(block)
inner = self.__inner_classes_pattern
order = [
(i, names[n[0]]) for (i, n) in enumerate(inner.findall(text))
]
instances = {
name: cls(self, name)
for (cls, name) in [
(classes[name], name)
for (_, name) in order
]
}
self._invariants = instances
self._invariant_names = names
self._invariant_order = order
self._invariant_classes = classes
self._invariants_processed = list()
def inspect_generator(g):
sourcecode = open(g.gi_code.co_filename).readlines()
gline = g.gi_code.co_firstlineno
generator_code = inspect.getblock(sourcecode[gline-1:])
output = "Generator %r from %r\n" % (g.gi_code.co_name, g.gi_code.co_filename)
output += "".join("%4s: %s" % (idx+gline, line) for idx, line in enumerate(generator_code))
output += "Local variables:\n"
output += "".join("%s = %r\n" % (key,value) for key,value in g.gi_frame.f_locals.items())
return output
def getsourcelines(pyObject):
"""Return a list of source lines and starting line number for an object.
The argument may be a module, class, method, function, traceback, frame,
or code object. The source code is returned as a list of the lines
corresponding to the object and the line number indicates where in the
original source file the first line of code was found. An IOError is
raised if the source code cannot be retrieved."""
lines, lnum = findsource(pyObject)
if ismodule(pyObject): return lines, 0
else: return getblock(lines[lnum:]), lnum + 1
def show_func(filename, start_lineno, func_name, timings, unit):
""" Show results for a single function.
"""
d = {}
total_time = 0.0
linenos = []
for lineno, nhits, time in timings:
total_time += time
linenos.append(lineno)
if total_time == 0:
return False
retval = {}
retval['Total time'] = "%g s" % (total_time * unit)
if os.path.exists(filename) or filename.startswith("<ipython-input-"):
retval['File'] = filename
retval['Function'] = '%s at line %s' % (func_name, start_lineno)
if os.path.exists(filename):
# Clear the cache to ensure that we get up-to-date results.
linecache.clearcache()
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno-1:])
else:
# Fake empty lines so we can see the timings, if not the code.
nlines = max(linenos) - min(min(linenos), start_lineno) + 1
sublines = [''] * nlines
for lineno, nhits, time in timings:
d[lineno] = (nhits, time, '%5.1f' % (float(time) / nhits),
'%5.1f' % (100*time / total_time))
linenos = range(start_lineno, start_lineno + len(sublines))
empty = ('', '', '', '')
lines = []
for lineno, line in zip(linenos, sublines):
nhits, time, per_hit, percent = d.get(lineno, empty)
line = {
'Line #': lineno,
'Hits': nhits,
'Time': time,
'Per Hit': per_hit,
'% Time': percent,
'Line Contents': line.rstrip('\n').rstrip('\r')
}
lines.append(line)
retval['lines'] = lines
return retval
def show_results(prof, stream=None):
if stream is None:
stream = sys.stdout
template = '{0:>6} {1:>12} {2:>10} {3:<}'
header = template.format('Line #', 'Mem usage', 'Increment', 'Line Contents')
stream.write(header + '\n')
stream.write('=' * len(header) + '\n')
for code in prof.code_map:
lines = prof.code_map[code]
if not lines:
# .. measurements are empty ..
continue
filename = code.co_filename
if (filename.endswith(".pyc") or
filename.endswith(".pyo")):
filename = filename[:-1]
all_lines = linecache.getlines(filename)
sub_lines = inspect.getblock(all_lines[code.co_firstlineno-1:])
linenos = range(code.co_firstlineno, code.co_firstlineno + len(sub_lines))
lines_normalized = {}
# move everything one frame up
keys = sorted(lines.keys())
k_old = keys[0] - 1
lines_normalized[keys[0] - 1] = lines[keys[0]]
k = keys.pop(0)
while keys:
lines_normalized[k] = lines[keys[0]]
for i in range(len(lines_normalized[k_old]), len(lines_normalized[k])):
lines_normalized[k][i] = -1.
k_old = k
k = keys.pop(0)
first_line = sorted(lines_normalized.keys())[0]
mem_old = max(lines_normalized[first_line])
for l in linenos:
mem = ''
inc = ''
if l in lines_normalized:
mem = max(lines_normalized[l])
inc = mem - mem_old
mem_old = mem
mem = '{0:5.2f} MB'.format(mem)
inc = '{0:5.2f} MB'.format(inc)
line = linecache.getline(filename, l)
stream.write(template.format(l, mem, inc, line))
stream.write('\n')
def process_line_stats(line_stats):
"Converts line_profiler.LineStats instance into something more useful"
profile_results = []
if not line_stats:
return profile_results
# We want timings in ms (instead of CPython's microseconds)
multiplier = line_stats.unit / 1e-3
for key, timings in sorted(line_stats.timings.items()):
if not timings:
continue
filename, start_lineno, func_name = key
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno - 1:])
end_lineno = start_lineno + len(sublines)
line_to_timing = collections.defaultdict(lambda: (-1, 0))
for (lineno, nhits, time) in timings:
line_to_timing[lineno] = (nhits, time)
padded_timings = []
for lineno in range(start_lineno, end_lineno):
nhits, time = line_to_timing[lineno]
padded_timings.append((lineno, nhits, time))
profile_results.append({
'filename':
filename,
'start_lineno':
start_lineno,
'func_name':
func_name,
'timings': [(
lineno,
all_lines[lineno - 1],
time * multiplier,
nhits,
) for (lineno, nhits, time) in padded_timings],
'total_time':
sum([time for _, _, time in timings]) * multiplier
})
return profile_results
def get_source(self, filename, first_line_num):
'''see python's inspect.py - this gets the source from
a Python source file, given the file name and first line number'''
import inspect
try:
lines = inspect.linecache.getlines(filename)
line_num = first_line_num - 1
while line_num > 0:
if self.func_definition_re.match(lines[line_num]):
break
lnum = lnum - 1
return '\n'.join(inspect.getblock(lines[line_num:]))
except:
return ''
def __init__(self, func, sandbox):
self.path = func.func_code.co_filename
self.name = func.func_name
code = func.func_code
firstlineno = code.co_firstlineno
lines = sandbox._current_source.splitlines(True)
lines = inspect.getblock(lines[firstlineno - 1:])
# The code lines we get out of inspect.getsourcelines look like
# @template
# def Template(*args, **kwargs):
# VAR = 'value'
# ...
func_ast = ast.parse(''.join(lines), self.path)
# Remove decorators
func_ast.body[0].decorator_list = []
# Adjust line numbers accordingly
ast.increment_lineno(func_ast, firstlineno - 1)
# When using a custom dictionary for function globals/locals, Cpython
# actually never calls __getitem__ and __setitem__, so we need to
# modify the AST so that accesses to globals are properly directed
# to a dict.
self._global_name = b'_data' # AST wants str for this, not unicode
# In case '_data' is a name used for a variable in the function code,
# prepend more underscores until we find an unused name.
while (self._global_name in code.co_names or
self._global_name in code.co_varnames):
self._global_name += '_'
func_ast = self.RewriteName(sandbox, self._global_name).visit(func_ast)
# Execute the rewritten code. That code now looks like:
# def Template(*args, **kwargs):
# _data['VAR'] = 'value'
# ...
# The result of executing this code is the creation of a 'Template'
# function object in the global namespace.
glob = {'__builtins__': sandbox._builtins}
func = types.FunctionType(
compile(func_ast, self.path, 'exec'),
glob,
self.name,
func.func_defaults,
func.func_closure,
)
func()
self._func = glob[self.name]
def process_line_stats(line_stats):
"Converts line_profiler.LineStats instance into something more useful"
profile_results = []
if not line_stats:
return profile_results
# We want timings in ms (instead of CPython's microseconds)
multiplier = line_stats.unit / 1e-3
for key, timings in sorted(line_stats.timings.items()):
if not timings:
continue
filename, start_lineno, func_name = key
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno-1:])
end_lineno = start_lineno + len(sublines)
line_to_timing = collections.defaultdict(lambda: (-1, 0))
for (lineno, nhits, time) in timings:
line_to_timing[lineno] = (nhits, time)
padded_timings = []
for lineno in range(start_lineno, end_lineno):
nhits, time = line_to_timing[lineno]
padded_timings.append( (lineno, nhits, time) )
profile_results.append({
'filename': filename,
'start_lineno': start_lineno,
'func_name': func_name,
'timings': [
(
lineno,
unicode(all_lines[lineno - 1].decode("utf8")),
time * multiplier,
nhits,
) for (lineno, nhits, time) in padded_timings
],
'total_time': sum([time for _, _, time in timings]) * multiplier
})
return profile_results
def get_in_frame_range(self):
# find function definition and mark lines
if hasattr(self.code, 'co_firstlineno'):
lineno = self.code.co_firstlineno - 1
while lineno > 0:
if _funcdef_re.match(self.sourcelines[lineno]):
break
lineno -= 1
try:
offset = len(inspect.getblock([x + '\n' for x
in self.sourcelines[lineno:]]))
except TokenError:
offset = 0
return (lineno, lineno + offset)
return None
def get_in_frame_range(self):
# find function definition and mark lines
if hasattr(self.code, 'co_firstlineno'):
lineno = self.code.co_firstlineno - 1
while lineno > 0:
if _funcdef_re.match(self.sourcelines[lineno]):
break
lineno -= 1
try:
offset = len(inspect.getblock([x + '\n' for x
in self.sourcelines[lineno:]]))
except TokenError:
offset = 0
return (lineno, lineno + offset)
return None
def get_block(self, filename, start_lineno, strip='\n\r'):
'''get source code block. returns empty lines if file not accessable'''
if not os.path.exists(filename):
# Fake empty lines so we can see the timings, if not the code.
nlines = max(self.linenos) - min(min(self.linenos), start_lineno) + 1
sublines = [''] * nlines
else:
# Clear the cache to ensure that we get up-to-date results.
linecache.clearcache()
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno-1:])
if strip:
sublines = [l.strip(strip) for l in sublines]
return sublines
def process_line_stats(line_stats):
profile_results = []
if not line_stats:
return profile_results
multiplier = line_stats.unit / 1e-3
for key, timings in sorted(line_stats.timings.items()):
if not timings:
continue
filename, start_lineno, func_name = key
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno - 1:])
end_lineno = start_lineno + len(sublines)
line_to_timing = collections.defaultdict(lambda: (-1, 0))
for (lineno, nhits, time) in timings:
line_to_timing[lineno] = (nhits, time)
padded_timings = []
for lineno in range(start_lineno, end_lineno):
nhits, time = line_to_timing[lineno]
padded_timings.append((lineno, nhits, time))
profile_results.append({
'filename':
filename,
'start_lineno':
start_lineno,
'func_name':
func_name,
'timings': [(
lineno,
unicode(all_lines[lineno - 1].decode("utf8")),
time * multiplier,
nhits,
) for (lineno, nhits, time) in padded_timings],
'total_time':
sum([time for _, _, time in timings]) * multiplier
})
return profile_results
def get_func_code(func):
""" Attempts to retrieve a reliable function code hash.
The reason we don't use inspect.getsource is that it caches the
source, whereas we want this to be modified on the fly when the
function is modified.
Returns
-------
func_code: string
The function code
source_file: string
The path to the file in which the function is defined.
first_line: int
The first line of the code in the source file.
Notes
------
This function does a bit more magic than inspect, and is thus
more robust.
"""
source_file = None
try:
if func.__name__ == '<lambda>':
# On recent Python version, inspect is reliable with lambda
source_file = func.__code__.co_filename
return ''.join(inspect.getsourcelines(func)[0]), source_file, 1
# Try to retrieve the source code.
code = func.__code__
source_file = code.co_filename
with open(source_file) as source_file_obj:
first_line = code.co_firstlineno
# All the lines after the function definition:
source_lines = list(islice(source_file_obj, first_line - 1, None))
return ''.join(inspect.getblock(source_lines)), source_file, first_line
except:
# If the source code fails, we use the hash. This is fragile and
# might change from one session to another.
if hasattr(func, '__code__'):
# Python 3.X
return str(func.__code__.__hash__()), source_file, -1
else:
# Weird objects like numpy ufunc don't have __code__
# This is fragile, as quite often the id of the object is
# in the repr, so it might not persist accross sessions,
# however it will work for ufuncs.
return repr(func), source_file, -1
def get_func_code(func):
""" Attempts to retrieve a reliable function code hash.
The reason we don't use inspect.getsource is that it caches the
source, whereas we want this to be modified on the fly when the
function is modified.
Returns
-------
func_code: string
The function code
source_file: string
The path to the file in which the function is defined.
first_line: int
The first line of the code in the source file.
Notes
------
This function does a bit more magic than inspect, and is thus
more robust.
"""
source_file = None
try:
if func.__name__ == '<lambda>':
# On recent Python version, inspect is reliable with lambda
source_file = func.__code__.co_filename
return ''.join(inspect.getsourcelines(func)[0]), source_file, 1
# Try to retrieve the source code.
code = func.__code__
source_file = code.co_filename
with open(source_file) as source_file_obj:
first_line = code.co_firstlineno
# All the lines after the function definition:
source_lines = list(islice(source_file_obj, first_line - 1, None))
return ''.join(inspect.getblock(source_lines)), source_file, first_line
except:
# If the source code fails, we use the hash. This is fragile and
# might change from one session to another.
if hasattr(func, '__code__'):
# Python 3.X
return str(func.__code__.__hash__()), source_file, -1
else:
# Weird objects like numpy ufunc don't have __code__
# This is fragile, as quite often the id of the object is
# in the repr, so it might not persist across sessions,
# however it will work for ufuncs.
return repr(func), source_file, -1
def show_func(filename, start_lineno, func_name, timings, unit):
"""
Show results for a single function.
"""
print "File: %s" % filename
print "Function: %s at line %s" % (func_name, start_lineno)
d = {}
total_time = 0.0
linenos = []
for lineno, nhits, time in timings:
total_time += time
linenos.append(lineno)
#print "Total time: %g s" % (total_time * unit)
if not os.path.exists(filename):
raise Exception("Could not find file %s" % filename)
# Fake empty lines so we can see the timings, if not the code.
nlines = max(linenos) - min(min(linenos), start_lineno) + 1
sublines = [''] * nlines
else:
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno-1:])
for lineno, nhits, time in timings:
d[lineno] = (
nhits,
'%2.3f s' % (time * unit),
'%5.1f' % (float(time) / nhits),
'%5.1f' % (100*time / total_time)
)
linenos = range(start_lineno, start_lineno + len(sublines))
empty = ('', '', '', '')
# ('Line #', 'Hits', 'Time', 'Per Hit', '% Time', 'Line Contents')
results = []
for lineno, line in zip(linenos, sublines):
nhits, time, per_hit, percent = d.get(lineno, empty)
results.append((lineno, nhits, time, per_hit, percent,
line.rstrip('\n').rstrip('\r')))
return results
def show_results(prof, stream=None, precision=1):
if stream is None:
stream = sys.stdout
template = '{0:>6} {1:>12} {2:>12} {3:<}'
for code in prof.code_map:
lines = prof.code_map[code]
if not lines:
# .. measurements are empty ..
continue
filename = code.co_filename
if filename.endswith((".pyc", ".pyo")):
filename = filename[:-1]
stream.write('Filename: ' + filename + '\n\n')
if not os.path.exists(filename):
stream.write('ERROR: Could not find file ' + filename + '\n')
if any([
filename.startswith(k)
for k in ("ipython-input", "<ipython-input")
]):
print("NOTE: %mprun can only be used on functions defined in "
"physical files, and not in the IPython environment.")
continue
all_lines = linecache.getlines(filename)
sub_lines = inspect.getblock(all_lines[code.co_firstlineno - 1:])
linenos = range(code.co_firstlineno,
code.co_firstlineno + len(sub_lines))
header = template.format('Line #', 'Mem usage', 'Increment',
'Line Contents')
stream.write(header + '\n')
stream.write('=' * len(header) + '\n')
mem_old = lines[min(lines.keys())]
float_format = '{0}.{1}f'.format(precision + 4, precision)
template_mem = '{0:' + float_format + '} MiB'
for line in linenos:
mem = ''
inc = ''
if line in lines:
mem = lines[line]
inc = mem - mem_old
mem_old = mem
mem = template_mem.format(mem)
inc = template_mem.format(inc)
stream.write(template.format(line, mem, inc, all_lines[line - 1]))
stream.write('\n\n')
def _extract_source(lines, lineno):
r"""
Given a list of lines or a multiline string and a starting lineno,
_extract_source returns [source_lines]. [source_lines] is the smallest
indentation block starting at lineno.
"""
if lineno < 1:
raise ValueError, "Line numbering starts at 1! (tried to extract line %s)" % lineno
lineno -= 1
if isinstance(lines, str):
lines = lines.splitlines(True) # true keeps the '\n'
if len(lines) > 0:
# Fixes an issue with getblock
lines[-1] += '\n'
return inspect.getblock(lines[lineno:])
def _memory_profile_it(mem_profiler):
""" Returns a dictionary with the memory profile result
Args:
mem_profiler (class): instance of `SpeedIT.MemIt._LineMemoryProfiler`
Returns:
tuple: format: (max_mem, table): table = list_of_dictionaries
"""
memory_precision = 3
table = []
max_mem = 0
for code in mem_profiler.code_map:
lines = mem_profiler.code_map[code]
if not lines:
# .. measurements are empty ..
continue
filename = code.co_filename
if filename.endswith(('.pyc', '.pyo')):
filename = filename[:-1]
if not path.exists(filename):
print('\n_memory_profile_it() ERROR: Could not find file: {}'.format(filename))
continue
all_lines = getlines(filename)
sub_lines = getblock(all_lines[code.co_firstlineno - 1:])
mem_old = lines[min(lines.keys())]
for line in range(code.co_firstlineno, code.co_firstlineno + len(sub_lines)):
mem = 0.0
mem_increment = 0.0
if line in lines:
mem = lines[line]
if mem > max_mem:
max_mem = mem
mem_increment = mem - mem_old
mem_old = mem
dict_ = {
'line_num': '{}'.format(line),
'memory_usage': '{:.{}f} MiB'.format(mem, memory_precision),
'increment_memory_usage': '{:.{}f} MiB'.format(mem_increment, memory_precision),
'line': all_lines[line - 1].strip()
}
table.append(dict_)
max_mem = '{:.{}f} MiB'.format(max_mem, memory_precision)
return max_mem, table
def _extract_source(lines, lineno):
r"""
Given a list of lines or a multiline string and a starting lineno,
_extract_source returns [source_lines]. [source_lines] is the smallest
indentation block starting at lineno.
"""
if lineno < 1:
raise ValueError, "Line numbering starts at 1! (tried to extract line %s)" % lineno
lineno -= 1
if isinstance(lines, str):
lines = lines.splitlines(True) # true keeps the '\n'
if len(lines) > 0:
# Fixes an issue with getblock
lines[-1] += '\n'
return inspect.getblock(lines[lineno:])
def fix_single(match, lines, lineno):
if not lines[lineno + 1].startswith("def"):
return
block_lines = inspect.getblock(lines[lineno + 1 :])
func_code = "".join(block_lines)
if func_code[0].isspace():
node = ast.parse("if 1:\n" + func_code).body[0].body
else:
node = ast.parse(func_code).body[0]
response_param_name = looks_like_teardown_function(node)
if response_param_name is None:
return
before = lines[:lineno]
decorator = [match.group(1) + match.group(2).replace("after_", "teardown_") + match.group(3)]
body = [line.replace(response_param_name, "exception") for line in block_lines if not is_return_line(line)]
after = lines[lineno + len(block_lines) + 1 :]
return before + decorator + body + after
def getsource_no_unwrap(obj):
"""Return source code for an object. Does not unwrap TFDecorators.
The source code is returned literally, including indentation for functions not
at the top level. This function is analogous to inspect.getsource, with one
key difference - it doesn't unwrap decorators. For simplicity, support for
some Python object types is dropped (tracebacks, frames, code objects).
Args:
obj: a class, method, or function object.
Returns:
source code as a string
"""
lines, lnum = _inspect.findsource(obj)
return ''.join(_inspect.getblock(lines[lnum:]))
def getsource_no_unwrap(obj):
"""Return source code for an object. Does not unwrap TFDecorators.
The source code is returned literally, including indentation for functions not
at the top level. This function is analogous to inspect.getsource, with one
key difference - it doesn't unwrap decorators. For simplicity, support for
some Python object types is dropped (tracebacks, frames, code objects).
Args:
obj: a class, method, or function object.
Returns:
source code as a string
"""
lines, lnum = _inspect.findsource(obj)
return ''.join(_inspect.getblock(lines[lnum:]))
def getsource(func):
""" Returning source code for a decoratted function.
inspect.getsource fails on decoratted functions, hence, a more elaborate
way to retrieve the source-code of such functions
Parameters:
-----------
func: function object
"""
# Obtain the module of the function
module = inspect.getmodule(func)
# Get the source lines of the file containing the module
file = inspect.getsourcefile(module)
lines = linecache.getlines(file)
object = func.func_code
# Using regular expressions find the lines of the source-code that
# represent the function
lnum = len(lines) - 1
pat = re.compile(r'^(\s*def\s%s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)' %
func.__name__)
found, source, code_object = 0, '', None
while lnum >= 0 and found == 0:
if pat.match(lines[lnum]):
new_lines = inspect.getblock(lines[lnum:])
source = string.join(new_lines, '')
# Check if this is the right function, as reg-ex matches the
# line with decorator definitions, i.e., @decorator-name, too.
code_object = compiler.compile(source, '', 'exec')
if func.__name__ in code_object.co_varnames:
found = 1
lnum = lnum - 1
# Return the source code of the function block
if found:
return source
else:
return None
def show_results(prof, stream=None, precision=1):
if stream is None:
stream = sys.stdout
template = '{0:>6} {1:>12} {2:>12} {3:<}'
for code in prof.code_map:
lines = prof.code_map[code]
if not lines:
# .. measurements are empty ..
continue
filename = code.co_filename
if filename.endswith((".pyc", ".pyo")):
filename = filename[:-1]
stream.write('Filename: ' + filename + '\n\n')
if not os.path.exists(filename):
stream.write('ERROR: Could not find file ' + filename + '\n')
if any([filename.startswith(k) for k in
("ipython-input", "<ipython-input")]):
print("NOTE: %mprun can only be used on functions defined in "
"physical files, and not in the IPython environment.")
continue
all_lines = linecache.getlines(filename)
sub_lines = inspect.getblock(all_lines[code.co_firstlineno - 1:])
linenos = range(code.co_firstlineno,
code.co_firstlineno + len(sub_lines))
header = template.format('Line #', 'Mem usage', 'Increment',
'Line Contents')
stream.write(header + '\n')
stream.write('=' * len(header) + '\n')
mem_old = lines[min(lines.keys())]
float_format = '{0}.{1}f'.format(precision + 4, precision)
template_mem = '{0:' + float_format + '} MiB'
for line in linenos:
mem = ''
inc = ''
if line in lines:
mem = lines[line]
inc = mem - mem_old
mem_old = mem
mem = template_mem.format(mem)
inc = template_mem.format(inc)
stream.write(template.format(line, mem, inc, all_lines[line - 1]))
stream.write('\n\n')
def show_func(filename, start_lineno, func_name, timings, unit, stream=None):
""" Show results for a single function.
"""
if not timings:
return
if stream is None:
stream = sys.stdout
print >> stream, "File: %s" % filename
print >> stream, "Function: %s at line %s" % (func_name, start_lineno)
template = '%6s %9s %12s %8s %8s %-s'
d = {}
total_time = 0.0
linenos = []
for lineno, nhits, time in timings:
total_time += time
linenos.append(lineno)
print >> stream, "Total time: %g s" % (total_time * unit)
if not os.path.exists(filename):
print >> stream, ""
print >> stream, "Could not find file %s" % filename
print >> stream, "Are you sure you are running this program from the same directory"
print >> stream, "that you ran the profiler from?"
print >> stream, "Continuing without the function's contents."
# Fake empty lines so we can see the timings, if not the code.
nlines = max(linenos) - min(min(linenos), start_lineno) + 1
sublines = [''] * nlines
else:
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno - 1:])
for lineno, nhits, time in timings:
d[lineno] = (nhits, time, '%5.1f' % (float(time) / nhits),
'%5.1f' % (100 * time / total_time))
linenos = range(start_lineno, start_lineno + len(sublines))
empty = ('', '', '', '')
header = template % ('Line #', 'Hits', 'Time', 'Per Hit', '% Time',
'Line Contents')
print >> stream, ""
print >> stream, header
print >> stream, '=' * len(header)
for lineno, line in zip(linenos, sublines):
nhits, time, per_hit, percent = d.get(lineno, empty)
print >> stream, template % (lineno, nhits, time, per_hit, percent,
line.rstrip('\n').rstrip('\r'))
print >> stream, ""
def show_func(filename, start_lineno, func_name, timings, unit):
"""
Show results for a single function.
"""
print "File: %s" % filename
print "Function: %s at line %s" % (func_name, start_lineno)
d = {}
total_time = 0.0
linenos = []
for lineno, nhits, time in timings:
total_time += time
linenos.append(lineno)
#print "Total time: %g s" % (total_time * unit)
if not os.path.exists(filename):
raise Exception("Could not find file %s" % filename)
# Fake empty lines so we can see the timings, if not the code.
nlines = max(linenos) - min(min(linenos), start_lineno) + 1
sublines = [''] * nlines
else:
all_lines = linecache.getlines(filename)
sublines = inspect.getblock(all_lines[start_lineno - 1:])
for lineno, nhits, time in timings:
d[lineno] = (nhits, '%2.3f s' % (time * unit),
'%5.1f' % (float(time) / nhits),
'%5.1f' % (100 * time / total_time))
linenos = range(start_lineno, start_lineno + len(sublines))
empty = ('', '', '', '')
# ('Line #', 'Hits', 'Time', 'Per Hit', '% Time', 'Line Contents')
results = []
for lineno, line in zip(linenos, sublines):
nhits, time, per_hit, percent = d.get(lineno, empty)
results.append((lineno, nhits, time, per_hit, percent,
line.rstrip('\n').rstrip('\r')))
return results
def ast_from_source(source, keyword: str):
"""
Return the AST representation of `source`. `source` might be a function or
string of source code.
If `source` is a function, `keyword` is used to find the very start of its
source code.
"""
if inspect.isfunction(source):
lines, lnum = inspect.findsource(source.__code__)
# Append line-ending if necessary
for idx in range(len(lines)):
if not lines[idx].endswith("\n"):
lines[idx] += "\n"
# Lines starting from `lnum` may contain enclosing tokens of previous expression
# We use `keyword` to locate the true start point of source
while True:
first_keyword_loc = lines[lnum].find(keyword)
if first_keyword_loc >= 0:
break
lnum -= 1
lines = [lines[lnum][first_keyword_loc:]] + lines[lnum + 1:]
# Prepend the lines with newlines, so that parsed AST will have correct lineno
source_lines = ["\n"] * lnum + inspect.getblock(lines)
source = "".join(source_lines)
# Some garbage may still remain at the end, we alternatively try compiling
# and popping the last character until the source is valid.
exc = None
original_source = source
while source:
try:
return ast.parse(source).body[0]
except SyntaxError as e:
source = source[:-1]
exc = e
else:
# This should never happen
raise RuntimeError("cannot parse the snippet into AST:\n{}".format(
original_source)) from exc
def _update_code(self, module_source):
""" Regenerate the code for the function.
"""
# This isn't quite right as it will omit comments above the function.
# Need to scan upwards from the function.
ast = compile(module_source, '', 'exec', _ast.PyCF_ONLY_AST)
lineno = -1
for node in ast.body:
if isinstance(node, _ast.FunctionDef) and node.name == self.name:
lineno = node.lineno
if lineno >= 0:
codelines = module_source.split('\n')
codelines = [codeline + '\n' for codeline in codelines]
self.code = ''.join(inspect.getblock(codelines[lineno-1:]))
else:
self.code = ''
return