def _show_datasource(self, d, v, indent=""):
try:
filtered = "filtered" if dr.DELEGATES[
d].filterable else "unfiltered"
mode = "bytes" if dr.DELEGATES[d].raw else "lines"
except:
filtered = "unknown"
mode = "unknown"
desc = "{n} ({f} / {m})".format(n=dr.get_name(d), f=filtered, m=mode)
color = self._get_color(d)
results = []
results.append(indent + ansiformat(color, desc))
if not v:
return results
if not isinstance(v, list):
v = [v]
for i in v:
if isinstance(i, ContentProvider):
s = ansiformat(color, str(i))
else:
s = ansiformat(color, "<intermediate value>")
results.append("{}\u250A\u254C\u254C\u254C\u254C\u254C{}".format(
indent, s))
return results
def run(self, args=None):
args = args or sys.argv[1:]
try:
opts, args = getopt.getopt(args, 'h', ['help', 'version'])
opts = dict(opts)
except:
opts = {}
args = []
if '-h' in opts or '--help' in opts:
self.print_help()
if '--version' in opts:
print get_version()
sys.exit(0)
if not args:
self.print_help()
if args[0] == 'help':
if len(args) == 1:
self.print_help()
args = [args[1], '-h']
cmd = args.pop(0)
if cmd.startswith('-'):
self.print_help()
try:
command = REGISTRY[cmd]
command().run(args)
except KeyError:
print ansiformat('*red*', 'Error: No such command %r.' % cmd)
def find(self, match=None, ignore=None):
"""
Find components that might be available based on the data being
analyzed.
Args:
match (str, optional): regular expression for matching against
the fully qualified name of components to keep.
ignore (str, optional): regular expression for searching against
the fully qualified name of components to ignore.
"""
match, ignore = self._desugar_match_ignore(match, ignore)
mid_dashes = "\u250A\u254C\u254C"
bottom_dashes = "\u2514\u254C\u254C"
results = []
for p in sorted(self, key=str.lower):
comp = self[p]
name = dr.get_name(comp)
if match.test(name) and not ignore.test(name):
color = self._get_color(comp)
_type = self._get_type_name(comp)
suffix = self._get_rule_value(comp)
desc = ansiformat(color, "{p} ({n}, {t}".format(p=p,
n=name,
t=_type))
results.append(desc + suffix + ansiformat(color, ")"))
if comp in self._broker.exceptions:
exes = self._broker.exceptions[comp]
last = len(exes) - 1
for i, ex in enumerate(exes):
dashes = bottom_dashes if i == last else mid_dashes
results.append(ansiformat(color, dashes + str(ex)))
IPython.core.page.page(six.u(os.linesep.join(results)))
def format(self, tokensource, outfile):
enc = self.encoding
# hack: if the output is a terminal and has an encoding set,
# use that to avoid unicode encode problems
if not enc and hasattr(outfile, "encoding") and \
hasattr(outfile, "isatty") and outfile.isatty():
enc = outfile.encoding
for ttype, value in tokensource:
if enc:
value = value.encode(enc)
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(ansiformat(color, line))
outfile.write('\n')
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
def show_timings(self, match=None, ignore="spec", group=dr.GROUPS.single):
"""
Show timings for components that have successfully evaluated.
Args:
match (str, optional): regular expression for matching against
the fully qualified name of components to keep.
ignore (str, optional): regular expression for searching against
the fully qualified name of components to ignore.
"""
match, ignore = self._desugar_match_ignore(match, ignore)
results = []
total = 0.0
for comp in dr.COMPONENTS[group]:
name = dr.get_name(comp)
if comp in self._broker.exec_times and match.test(
name) and not ignore.test(name):
color = self._get_color(comp)
t = self._broker.exec_times[comp]
total += t
results.append((t, name, color))
report = [
ansiformat("brightmagenta",
"Total: {:.10f} seconds".format(total)), ""
]
for timing, name, color in sorted(results, reverse=True):
report.append(ansiformat(color, "{:.10f}: {}".format(timing,
name)))
IPython.core.page.page(six.u(os.linesep.join(report)))
def format(self, tokensource, outfile):
enc = self.encoding
# hack: if the output is a terminal and has an encoding set,
# use that to avoid unicode encode problems
if not enc and hasattr(outfile, "encoding") and \
hasattr(outfile, "isatty") and outfile.isatty():
enc = outfile.encoding
for ttype, value in tokensource:
if enc:
value = value.encode(enc)
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(ansiformat(color, line))
outfile.write('\n')
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
def __init__(self, filename, output=None, width=80, colors=True,
animation=True, typing_delay=40, logging=False):
if output is None:
self.output = sys.stdout
else:
self.output = output
self.logging = logging
if logging:
time = datetime.now().strftime('-%Y-%m-%d')
log_name = os.path.splitext(filename)[0] + time + '.log'
self.logger = open(log_name, 'at')
self.logger.write('\n')
bar = '=' * 34
self.log(bar, '===== AutoPython initiated =====', bar)
self.colors = colors
self.animation = animation
self.typing_delay = typing_delay
self.lexer = PythonConsoleLexer(python3=True)
self.width = self.current_width = width - 1
self.ps1 = self.hl_ps1 = '>>> '
self.ps2 = self.hl_ps2 = '... '
self.colorscheme = TERMINAL_COLORS
if self.colors:
self.hl_ps1 = ansiformat(self.colorscheme[Token.Generic.Prompt], self.ps1)
self.hl_ps2 = ansiformat(self.colorscheme[Token.Generic.Prompt], self.ps2)
self.reset_interpreter()
self.load_file(filename)
def show_rule_report(self, match=None, ignore=None):
"""
Print a rule report for the matching rules.
"""
match, ignore = self._desugar_match_ignore(match, ignore)
results = defaultdict(dict)
for comp, val in self._broker.items():
name = dr.get_name(comp)
if plugins.is_rule(comp) and match.test(
name) and not ignore.test(name):
kind = self._get_rule_value_kind(val)
if kind:
body = render(comp, val)
links = render_links(comp)
results[kind][name] = os.linesep.join([body, "", links])
report = []
for kind in ["info", "pass", "fail"]:
color = RULE_COLORS.get(kind, "")
hits = results.get(kind, {})
for name in sorted(hits):
report.append(ansiformat(color, name))
report.append(ansiformat(color, "-" * len(name)))
report.append(hits[name])
report.append("")
IPython.core.page.page(six.u(os.linesep.join(report)))
def to_string(self,
color=False,
fields_order=None,
sep=' -- ',
verify=False):
if fields_order is None:
fields_order = ['file', 'price', 'retailer', 'note']
for row in self.rows:
fmt_str = sep.join([row.format(field) for field in fields_order])
if verify and not color:
fmt_str += ' | {}'.format(row['verified'])
if verify and color:
if ansiformat:
if row['verified']:
fmt_str = ansiformat('green', fmt_str)
else:
fmt_str = ansiformat('red', fmt_str)
else:
warnings.warn('You must install pygments to have colored '
'output.')
fields = row.copy()
fields.update(self.max_len)
yield fmt_str.format(**fields)
def show_failed(self, match=None, ignore="spec"):
"""
Show names of any components that failed during evaluation. Ignores
"spec" by default.
Args:
match (str, optional): regular expression for matching against
the fully qualified name of components to keep.
ignore (str, optional): regular expression for searching against
the fully qualified name of components to ignore.
"""
match, ignore = self._desugar_match_ignore(match, ignore)
mid_dashes = "\u250A\u254C\u254C"
bottom_dashes = "\u2514\u254C\u254C"
results = []
for comp in sorted(self._broker.exceptions, key=dr.get_name):
name = dr.get_name(comp)
if match.test(name) and not ignore.test(name):
color = self._get_color(comp)
results.append(ansiformat(color, name))
exes = self._broker.exceptions[comp]
last = len(exes) - 1
for i, ex in enumerate(exes):
dashes = bottom_dashes if i == last else mid_dashes
results.append(ansiformat(color, dashes + str(ex)))
results.append("")
IPython.core.page.page(six.u(os.linesep.join(results)))
def error(self, msg):
"""Print the given error message and exit.
"""
if not isinstance(msg, basestring):
msg = '%s' % msg
if not msg.startswith('Error:'):
msg = 'Error: %s' % msg
print ansiformat('*red*', msg)
sys.exit(1)
def _show_exceptions(self, comp):
name = dr.get_name(comp)
results = [ansiformat("*brightred*", name)]
results.append(ansiformat("*brightred*", "-" * len(name)))
for e in self._broker.exceptions.get(comp, []):
t = self._broker.tracebacks.get(e)
if t:
results.append(t)
return results
def format_unencoded(self, tokensource, outfile):
for ttype, value in tokensource:
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(ansiformat(color, line))
outfile.write('\n')
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
def format_unencoded(self, tokensource, outfile):
for ttype, value in tokensource:
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(ansiformat(color, line))
outfile.write('\n')
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
def write(self, text, color):
color_text = text
if color:
color_text = ansiformat(color, color_text)
self.outfile.write(color_text)
self.column += len(text)
return self.column
def show_requested(self):
""" Show the components you've worked with so far. """
results = []
for name, comp in sorted(self._requested):
results.append(
ansiformat(self._get_color(comp),
"{} {}".format(name, dr.get_name(comp))))
IPython.core.page.page(six.u(os.linesep.join(results)))
def format_error(e):
"""Colorize the error message.
"""
if not isinstance(e, basestring):
e = '%s' % e
if not e.startswith('Error:'):
e = 'Error: %s' % e
return ansiformat('*red*', e)
def format_error(e):
"""Colorize the error message.
"""
if not isinstance(e, basestring):
e = '%s' % e
if not e.startswith('Error:'):
e = 'Error: %s' % e
return ansiformat('*red*', e)
def format_token(ttype, token, colorscheme=TERMINAL_COLORS, highlight='light'):
if 'plain' == highlight: return token
light_bg = 'light' == highlight
color = colorscheme.get(ttype)
if color:
color = color[light_bg]
return ansiformat(color, token)
pass
return token
def format_token(ttype, token, colorscheme=color_scheme, highlight='light'):
if 'plain' == highlight: return token
dark_bg = 'dark' == highlight
color = colorscheme.get(ttype)
if color:
color = color[dark_bg]
return ansiformat(color, token)
pass
return token
def _get_rule_value(self, comp):
try:
val = self._broker[comp]
if plugins.is_rule(comp):
_type = val.__class__.__name__
kind = self._get_rule_value_kind(val)
color = RULE_COLORS.get(kind, "")
return ansiformat(color, " [{}]".format(_type))
except:
pass
return ""
def format_token(ttype, token, colorscheme=color_scheme,
highlight='light' ):
if 'plain' == highlight: return token
dark_bg = 'dark' == highlight
color = colorscheme.get(ttype)
if color:
color = color[dark_bg]
return ansiformat(color, token)
pass
return token
def format_token(ttype, token, colorscheme=color_scheme, highlight="light"):
if "plain" == highlight:
return token
dark_bg = "dark" == highlight
color = colorscheme.get(ttype)
if color:
color = color[dark_bg]
return ansiformat(color, token)
pass
return token
def format_token(ttype, token, colorscheme=TERMINAL_COLORS,
highlight='light' ):
if 'plain' == highlight: return token
light_bg = 'light' == highlight
color = colorscheme.get(ttype)
if color:
color = color[light_bg]
return ansiformat(color, token)
pass
return token
def highlight(source):
"""
write `source` to stdout, highlighting all the found
features (in purple, of course)
"""
last_end = 0
attr = '*purple*'
for _, (start, end) in find_all(source):
sys.stdout.write(source[last_end:start])
sys.stdout.write(ansiformat(attr, source[start:end]))
last_end = end
sys.stdout.write(source[last_end:])
def format_unencoded(self, tokensource, outfile):
if self.linenos:
self._format_unencoded_with_lineno(tokensource, outfile)
return
for ttype, value in tokensource:
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(ansiformat(color, line))
outfile.write('\n')
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
def format_unencoded(self, tokensource, outfile):
if self.linenos:
self._format_unencoded_with_lineno(tokensource, outfile)
return
for ttype, value in tokensource:
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(ansiformat(color, line))
outfile.write('\n')
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
def show_source(self, comp):
"""
Show source for the given module, class, or function. Also accepts
the string names, with the side effect that the component will be
imported.
"""
try:
if isinstance(comp, six.string_types):
comp = self.get(comp) or dr.get_component(
comp) or importlib.import_module(comp)
comp = inspect.getmodule(comp)
ip = IPython.get_ipython()
if self._cov:
path, runnable, excluded, not_run, _ = self._cov.analysis2(
comp)
runnable, not_run = set(runnable), set(not_run)
src = ip.pycolorize(inspect.getsource(comp)).splitlines()
width = len(str(len(src)))
template = "{0:>%s}" % width
results = []
file_line = "{} {}".format(ansiformat("red", "File:"),
os.path.realpath(path))
explain_line = "{} numbered lines have executed. python standard libs are excluded.".format(
ansiformat("*brightgreen*", "Green"))
results.append(file_line)
results.append(explain_line)
results.append("")
for i, line in enumerate(src, start=1):
prefix = template.format(i)
if i in runnable and i not in not_run:
color = "*brightgreen*"
else:
color = "gray"
results.append("{} {}".format(ansiformat(color, prefix),
line))
IPython.core.page.page(six.u(os.linesep.join(results)))
else:
ip.inspector.pinfo(comp, detail_level=1)
except:
traceback.print_exc()
def interact(self):
if self.state != Presenter.READY_TO_TYPE:
self.cancel()
self.write('\r')
self.log('Entering interactive mode.')
if self.colors:
ps1 = ansiformat('*green*', self.ps1)
ps2 = ansiformat('*green*', self.ps2)
else:
ps1 = self.ps1
ps2 = self.ps2
lines = []
need_more = False
while True:
try:
try:
self.write(ps2 if need_more else ps1)
line = input()
lines.append(line)
except EOFError:
break
else:
source = '\n'.join(lines)
need_more = self.interpreter.runsource(source)
if not need_more:
self.log('Executing new statement:', *lines)
lines = []
except KeyboardInterrupt:
self.write('\n', ansiformat('*red*', 'KeyboardInterrupt') if self.colors else 'KeyboardInterrupt', '\n')
lines = []
need_more = False
self.log('Leaving interactive mode.')
self.write('\r', self.hl_ps1)
def _show_tree(self,
node,
indent="",
depth=None,
dep_getter=dr.get_dependencies):
if depth is not None and depth == 0:
return []
results = []
color = self._get_color(node)
if plugins.is_datasource(node):
results.extend(
self._show_datasource(node,
self._broker.get(node),
indent=indent))
else:
_type = self._get_type_name(node)
name = dr.get_name(node)
suffix = self._get_rule_value(node)
desc = ansiformat(color, "{n} ({t}".format(n=name, t=_type))
results.append(indent + desc + suffix + ansiformat(color, ")"))
dashes = "\u250A\u254C\u254C\u254C\u254C\u254C"
if node in self._broker.exceptions:
for ex in self._broker.exceptions[node]:
results.append(indent + dashes + ansiformat(color, str(ex)))
deps = dep_getter(node)
next_indent = indent + "\u250A "
for d in deps:
results.extend(
self._show_tree(d,
next_indent,
depth=depth if depth is None else depth - 1,
dep_getter=dep_getter))
return results
def _format_unencoded_with_lineno(self, tokensource, outfile):
self._write_lineno(outfile)
for ttype, value in tokensource:
if value.endswith("\n"):
self._write_lineno(outfile)
value = value[:-1]
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
self._write_lineno(outfile)
if line:
outfile.write(ansiformat(color, line[:-1]))
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
outfile.write("\n")
def _format_unencoded_with_lineno(self, tokensource, outfile):
self._write_lineno(outfile)
for ttype, value in tokensource:
if value.endswith("\n"):
self._write_lineno(outfile)
value = value[:-1]
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
self._write_lineno(outfile)
if line:
outfile.write(ansiformat(color, line[:-1]))
if spl[-1]:
outfile.write(ansiformat(color, spl[-1]))
else:
outfile.write(value)
outfile.write("\n")
def show(tokens):
anim = self.animation
delay = self.typing_delay / 1000
for text, line, simulate_typing, color in tokens:
if color is not None:
on, off = ansiformat(color, '|').split('|')
self.write(on)
if anim and simulate_typing:
for char in text:
self.write(char)
if not char.isspace():
time.sleep(delay * (0.5 + random.random()))
else:
self.write(text)
if color is not None:
self.write(off)
def format_unencoded(self, tokensource, outfile):
if self.linenos:
self._write_lineno(outfile)
for ttype, value in tokensource:
color = self._get_color(ttype)
for line in value.splitlines(True):
if color:
outfile.write(ansiformat(color, line.rstrip("\n")))
else:
outfile.write(line.rstrip("\n"))
if line.endswith("\n"):
if self.linenos:
self._write_lineno(outfile)
else:
outfile.write("\n")
if self.linenos:
outfile.write("\n")
def format_unencoded(self, tokensource, outfile):
if self.linenos:
self._write_lineno(outfile)
for ttype, value in tokensource:
color = self._get_color(ttype)
for line in value.splitlines(True):
if color:
outfile.write(ansiformat(color, line.rstrip('\n')))
else:
outfile.write(line.rstrip('\n'))
if line.endswith('\n'):
if self.linenos:
self._write_lineno(outfile)
else:
outfile.write('\n')
if self.linenos:
outfile.write("\n")
def _wrap_level(level, value: str):
return ansiformat(_LEVEL_COLORS[level], value)
def discreet(text):
return ansiformat('faint', ansiformat('lightgray', text))
def green_bg(text):
return ansiformat('green', text)
def _wrap_text(level: int, value: str):
return ansiformat(_TEXT_COLORS[level], value)
def write(self, text, color):
color_text = text
if color: color_text = ansiformat(color, color_text)
self.outfile.write(color_text)
self.column += len(text)
return self.column
def red_bg(text):
return ansiformat('red', text)
def init_logger():
"""Initialize the logger. Do nothing for google app engine.
"""
if settings.DATABASE_ENGINE == "gae":
return
level = settings.LOGGING.get('level') or 'INFO'
format = settings.LOGGING.get('format') or LOG_FORMAT
logfile = settings.LOGGING.get('logfile')
level = logging.DEBUG if settings.DEBUG else LOG_LEVELS.get(level)
logging.basicConfig(level=level, format=format)
if logfile:
try:
handler = logging.handlers.TimedRotatingFileHandler(
logfile, 'D', 1, 30)
except Exception, e:
sys.stderr.write('ERROR: unable to create log file: %s' % str(e))
else:
formatter = logging.Formatter(format)
handler.setFormatter(formatter)
# Get rid of all the handlers already attached to it.
del logging.getLogger().handlers[:]
logging.getLogger().addHandler(handler)
logging.getLogger().setLevel(level)
else:
for level, attr in COLORS.items():
level = LOG_LEVELS[level]
logging.addLevelName(level, ansiformat(attr, logging.getLevelName(level)))
def cancel(self):
self.write(' ^C\n', ansiformat('*red*', 'KeyboardInterrupt') if self.colors else 'KeyboardInterrupt', '\n')
self.prompt()
"""Initialize the logger. Do nothing for google app engine.
"""
if settings.DATABASE_ENGINE == "gae":
return
level = settings.LOGGING.get('level') or 'INFO'
format = settings.LOGGING.get('format') or LOG_FORMAT
logfile = settings.LOGGING.get('logfile')
level = logging.DEBUG if settings.DEBUG else LOG_LEVELS.get(level)
logging.basicConfig(level=level, format=format)
if logfile:
try:
handler = logging.handlers.TimedRotatingFileHandler(
logfile, 'D', 1, 30)
except Exception, e:
sys.stderr.write('ERROR: unable to create log file: %s' % str(e))
else:
formatter = logging.Formatter(format)
handler.setFormatter(formatter)
# Get rid of all the handlers already attached to it.
del logging.getLogger().handlers[:]
logging.getLogger().addHandler(handler)
logging.getLogger().setLevel(level)
else:
for level, attr in COLORS.items():
level = LOG_LEVELS[level]
logging.addLevelName(level,
ansiformat(attr, logging.getLevelName(level)))
