def __init__(self, color=True, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
self._color = color and _stderr_supports_color()
if self._color:
# The curses module has some str/bytes confusion in
# python3. Until version 3.2.3, most methods return
# bytes, but only accept strings. In addition, we want to
# output these strings with the logging module, which
# works with unicode strings. The explicit calls to
# unicode() below are harmless in python2 but will do the
# right conversion in python 3.
fg_color = (curses.tigetstr("setaf") or
curses.tigetstr("setf") or "")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = unicode(fg_color, "ascii")
self._colors = {
logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue
"ascii"),
logging.INFO: unicode(curses.tparm(fg_color, 2), # Green
"ascii"),
logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow
"ascii"),
logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red
"ascii"),
}
self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def __init__(self):
ProgressTracker.__init__(self)
self.act_started = False
self.ind_started = False
self.last_print_time = 0
self.clear_eol = ""
try:
import curses
if not os.isatty(sys.stdout.fileno()):
raise ProgressTrackerException()
curses.setupterm()
self.cr = curses.tigetstr("cr")
self.clear_eol = curses.tigetstr("el") or ""
except KeyboardInterrupt:
raise
except:
if portable.ostype == "windows" and \
os.isatty(sys.stdout.fileno()):
self.cr = '\r'
else:
raise ProgressTrackerException()
self.dl_started = False
self.spinner = 0
self.spinner_chars = "/-\|"
self.curstrlen = 0
def __init__(self, out_file, ttymode):
"""Create a printengine.
out_file -- the file object to print to
ttymode -- Boolean indicating need for tty support. Throws
PrintEngineException if out_file can't support.
"""
PrintEngine.__init__(self)
self._out_file = out_file
self.__nchars_printed = 0
self.__needs_nl = 0
self.__cr = None
self.__ttymode = ttymode
if not self.__ttymode:
return
self.__putp_re = re.compile("\$<[0-9]+>")
self.__el = None
if not self._out_file.isatty():
raise PrintEngineException("Not a TTY")
try:
curses.setupterm(None, self._out_file.fileno())
self.__cr = curses.tigetstr("cr")
self.__el = curses.tigetstr("el")
except curses.error:
raise PrintEngineException("Unknown terminal '%s'" %
os.environ.get("TERM", ""))
def __init__(self, color, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
self._color = color
if color:
# The curses module has some str/bytes confusion in python3.
# Most methods return bytes, but only accept strings.
# The explict calls to unicode() below are harmless in python2,
# but will do the right conversion in python3.
fg_color = unicode(curses.tigetstr("setaf") or
curses.tigetstr("setf") or "", "ascii")
self._colors = {
logging.DEBUG: unicode(
curses.tparm(fg_color, curses.COLOR_CYAN),
"ascii"),
logging.INFO: unicode(
curses.tparm(fg_color, curses.COLOR_GREEN),
"ascii"),
logging.WARNING: unicode(
curses.tparm(fg_color, curses.COLOR_YELLOW), # Yellow
"ascii"),
logging.ERROR: unicode(
curses.tparm(fg_color, curses.COLOR_RED), # Red
"ascii"),
}
self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def _SetupColours():
"""Initializes the colour constants.
"""
# pylint: disable=W0603
# due to global usage
global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
# Don't use colours if stdout isn't a terminal
if not sys.stdout.isatty():
return
try:
import curses
except ImportError:
# Don't use colours if curses module can't be imported
return
curses.setupterm()
_RESET_SEQ = curses.tigetstr("op")
setaf = curses.tigetstr("setaf")
_INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
_WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
_ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
def __init__(self, color=True, datefmt=None):
r"""
:arg bool color: Enables color support.
:arg string fmt: Log message format.
It will be applied to the attributes dict of log records. The
text between ``%(color)s`` and ``%(end_color)s`` will be colored
depending on the level if color support is on.
:arg dict colors: color mappings from logging level to terminal color
code
:arg string datefmt: Datetime format.
Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``.
.. versionchanged:: 3.2
Added ``fmt`` and ``datefmt`` arguments.
"""
logging.Formatter.__init__(self, datefmt=datefmt)
self._colors = {}
if color and _stderr_supports_color():
fg_color = (curses.tigetstr("setaf") or
curses.tigetstr("setf") or "")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = str(fg_color, "ascii")
for levelno, code in self.DEFAULT_COLORS.items():
self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii")
self._normal = str(curses.tigetstr("sgr0"), "ascii")
scr = curses.initscr()
self.termwidth = scr.getmaxyx()[1]
curses.endwin()
else:
self._normal = ''
# Default width is usually 80, but too wide is worse than too narrow
self.termwidth = 70
def _setupTerm(self):
import curses
curses.setupterm()
self.COLS=curses.tigetnum('cols')
self.BOL=curses.tigetstr('cr')
self.UP=curses.tigetstr('cuu1')
self.CLEAR_EOL=curses.tigetstr('el')
def draw(self, path, start_x, start_y, width, height):
# Save cursor
curses.putp(curses.tigetstr("sc"))
y = start_y
# Move to drawing zone
self._move_to(start_x, y)
# Write intent
sys.stdout.write("%s}ic#%d;%d;%s%s" % (
self.display_protocol,
width, height,
path,
self.close_protocol))
# Write Replacement commands ('#')
for _ in range(0, height):
sys.stdout.write("%s}ib%s%s%s}ie%s" % (
self.display_protocol,
self.close_protocol,
"#" * width,
self.display_protocol,
self.close_protocol))
y = y + 1
self._move_to(start_x, y)
# Restore cursor
curses.putp(curses.tigetstr("rc"))
sys.stdout.flush()
def printwyx(self, window, coord, arg, color = None, attribute = None):
if self.cursescolors == True:
try:
if color or attribute:
attr = 0
if color: attr |= self.color[color]
if attribute: attr |= self.attribute[attribute]
if coord:
window.addstr(coord[0], coord[1], arg, attr)
else:
window.addstr(arg, attr)
elif coord:
window.addstr(coord[0], coord[1], arg)
else:
window.addstr(arg)
except curses.error:
pass
else:
coordstr = ""
if coord: coordstr = curses.tparm(curses.tigetstr("cup"), coord[0], coord[1])
colorstr = self.attribute['normal']
if color: colorstr += self.color[color]
if attribute: colorstr += self.attribute[attribute]
clr_eol = curses.tparm(curses.tigetstr("el"))
curses.putp(coordstr + colorstr + arg + clr_eol)
def draw(self):
"""Draw all objects in the container"""
self.win.touchwin()
DisplayableContainer.draw(self)
if self._draw_title and self.settings.update_title:
cwd = self.fm.thisdir.path
if cwd.startswith(self.fm.home_path):
cwd = '~' + cwd[len(self.fm.home_path):]
if self.settings.shorten_title:
split = cwd.rsplit(os.sep, self.settings.shorten_title)
if os.sep in split[0]:
cwd = os.sep.join(split[1:])
try:
fixed_cwd = cwd.encode('utf-8', 'surrogateescape'). \
decode('utf-8', 'replace')
fmt_tup = (
curses.tigetstr('tsl').decode('latin-1'),
fixed_cwd,
curses.tigetstr('fsl').decode('latin-1'),
)
except UnicodeError:
pass
else:
sys.stdout.write("%sranger:%s%s" % fmt_tup)
sys.stdout.flush()
self.win.refresh()
def setup(cls):
if getattr(cls, "_setup", False):
return
curses.setupterm()
cls.BOL = curses.tigetstr("cr")
cls.CLEAR_EOL = curses.tigetstr("el")
cls._setup = True
def _tigetstr(cap_name):
import curses
if not curses.tigetstr(cap_name):
return None
from .utils import to_ascii
cap = to_ascii(curses.tigetstr(cap_name)) or ''
# String capabilities can include "delays" of the form "$<2>".
# For any modern terminal, we should be able to just ignore
# these, so strip them out.
# terminof(5) states that:
# A "/" suffix indicates that the padding is mandatory and forces a
# delay of the given number of milliseconds even on devices for which
# xon is present to indicate flow control.
# So let's respect that. But not the optional ones.
cap = re.sub(r'\$<\d+>[*]?', '', cap)
# To switch back to "NORMAL", we use sgr0, which resets "everything" to defaults.
# That on some terminals includes ending "alternate character set mode".
# Which is usually done by appending '\017'. Unfortunately, less -R
# does not eat that, but shows it as an annoying inverse '^O'
# Instead of falling back to less -r, which would have other implications as well,
# strip off that '\017': we don't use the alternative character set,
# so we won't need to reset it either.
if cap_name == 'sgr0':
cap = re.sub(r'\017$', '', cap)
return cap
def _InitColorize(output_file):
"""Initializes the colorization directives via --colorize as a mapping
between compiled regular expressions meant to be matched against log
filename and target curses color escape codes. The curses escape codes
are represented as a pair containing codes for prepending and appending.
Returns: [ColorDirective], or None if
colorization is disabled or unavailable on the current terminal.
Based on tornado's code to colorize log output.
"""
color = False
# Set up color if we are in a tty, curses is available, and --colorize
# was specified.
if options.options.colorize is not None and curses and \
(options.options.color == 'yes' or \
(options.options.color == 'auto' and output_file.isatty())):
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except Exception:
pass
if not color:
return []
directives = []
normal = unicode(curses.tigetstr("sgr0"), "ascii")
fg_color = unicode(curses.tigetstr("setaf") or curses.tigetstr("setf") or "", "ascii")
for directive in options.options.colorize:
regexp, color_index = directive.split('=')
color = unicode(curses.tparm(fg_color, int(color_index)), "ascii")
directives.append(ColorDirective(re.compile(regexp), color, normal))
return directives
def __init__(self, color=True, fmt=DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT,
colors=DEFAULT_COLORS, precision=3):
r"""
:arg bool color: Enables color support.
:arg string fmt: Log message format.
It will be applied to the attributes dict of log records. The
text between ``%(color)s`` and ``%(end_color)s`` will be colored
depending on the level if color support is on.
:arg dict colors: color mappings from logging level to terminal color
code
:arg string datefmt: Datetime format.
Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``.
.. versionchanged:: 3.2
Added ``fmt`` and ``datefmt`` arguments.
"""
super().__init__()
self.default_time_format = datefmt
self.precision = precision
self.default_msec_format = ''
self._fmt = fmt
self._colors = {}
if color and _stderr_supports_color():
fg_color = (curses.tigetstr('setaf') or
curses.tigetstr('setf') or '')
for levelno, code in colors.items():
self._colors[levelno] = curses.tparm(fg_color, code).decode()
self._normal = curses.tigetstr('sgr0').decode()
else:
self._normal = ''
def __init__(self, buffer=sys.stdout, no_color=False):
if not curses:
no_color = True
if not no_color:
no_color = isinstance(sys.stdout, StringIO) or \
not sys.stdout.isatty()
self.no_color = no_color
# Don't bother initiating color if there's no color.
if not no_color:
# Get curses all ready to write some stuff to the screen.
curses.setupterm()
# Initialize a store for the colors and pre-populate it
# with the un-color color.
self.colors = {'NORMAL': curses.tigetstr('sgr0') or ''}
# Determines capabilities of the terminal.
fgColorSeq = curses.tigetstr('setaf') or \
curses.tigetstr('setf') or ''
# Go through each color and figure out what the sequences
# are for each, then store the sequences in the store we
# made above.
for color in COLORS:
colorIndex = getattr(curses, 'COLOR_%s' % color)
self.colors[color] = curses.tparm(fgColorSeq,
colorIndex)
self.buffer = buffer
def __init__(self, main, helper, console, format):
self.main = main
self.helper = helper
self.cuu = None
self.stdinbackup = None
self.interactive = sys.stdout.isatty()
self.footer_present = False
self.lastpids = []
if not self.interactive:
return
import curses
import termios
import copy
self.curses = curses
self.termios = termios
try:
fd = sys.stdin.fileno()
self.stdinbackup = termios.tcgetattr(fd)
new = copy.deepcopy(self.stdinbackup)
new[3] = new[3] & ~termios.ECHO
termios.tcsetattr(fd, termios.TCSADRAIN, new)
curses.setupterm()
self.ed = curses.tigetstr("ed")
if self.ed:
self.cuu = curses.tigetstr("cuu")
except:
self.cuu = None
console.addFilter(InteractConsoleLogFilter(self, format))
def add_color_to_string(string, color, stream=sys.stdout, bold=False,
force=False):
"""Format the string to be printed with the given color.
Insert formatting characters that, when printed on a terminal, will
make the given string appear with the given foreground color if the
stream passed has color support. Else return the string as it is.
string (string): the string to color.
color (int): the color as a colors constant, like colors.BLACK.
stream (fileobj): a file-like object (that adheres to the API
declared in the `io' package). Defaults to sys.stdout.
bold (bool): True if the string should be bold.
force (bool): True if the string should be formatted even if the
given stream has no color support.
return (string): the formatted string.
"""
if force or has_color_support(stream):
return "%s%s%s%s" % (
curses.tparm(curses.tigetstr("setaf"), color).decode('ascii')
if color != colors.BLACK else "",
curses.tparm(curses.tigetstr("bold")).decode('ascii')
if bold else "",
string,
curses.tparm(curses.tigetstr("sgr0")).decode('ascii')
)
else:
return string
def move_cursor(direction, amount=1, stream=sys.stdout, erase=False):
"""Move the cursor.
If the stream is a TTY, print characters that will move the cursor
in the given direction and optionally erase the line. Else do nothing.
direction (int): the direction as a directions constant, like
directions.UP.
stream (fileobj): a file-like object (that adheres to the API
declared in the `io' package). Defaults to sys.stdout.
erase (bool): True if the line the cursor ends on should be erased.
"""
if stream.isatty():
if direction == directions.UP:
print(curses.tparm(curses.tigetstr("cuu"), amount),
file=stream, end='')
elif direction == directions.DOWN:
print(curses.tparm(curses.tigetstr("cud"), amount),
file=stream, end='')
elif direction == directions.LEFT:
print(curses.tparm(curses.tigetstr("cub"), amount),
file=stream, end='')
elif direction == directions.RIGHT:
print(curses.tparm(curses.tigetstr("cuf"), amount),
file=stream, end='')
if erase:
print(curses.tparm(curses.tigetstr("el")), file=stream, end='')
stream.flush()
def do_term(term, nick):
curses.setupterm(term)
sys.stdout.write("/* %s */\n" % term)
sys.stdout.write("const char *%s_keys[] = {\n\t" % nick)
for k, v in iter_pairs(keys):
sys.stdout.write('"')
puttc(curses.tigetstr(v))
sys.stdout.write('",')
sys.stdout.write(" 0\n};\n")
sys.stdout.write("const char *%s_funcs[] = {\n\t" % nick)
for k, v in iter_pairs(funcs):
# sys.stdout.write("\t[%s] = \"" % k)
sys.stdout.write('"')
if v == "sgr":
sys.stdout.write("\\033[3%u;4%um")
elif v == "cup":
sys.stdout.write("\\033[%u;%uH")
else:
tc = curses.tigetstr(v)
puttc(tc)
if v == "rmkx":
sys.stdout.write('"\n')
else:
sys.stdout.write('",')
sys.stdout.write("};\n\n")
def __init__(self, color=True, fmt=DEFAULT_FORMAT,
datefmt=DEFAULT_DATE_FORMAT, colors=DEFAULT_COLORS):
r"""
:arg bool color: Enables color support.
:arg string fmt: Log message format.
It will be applied to the attributes dict of log records. The
text between ``%(color)s`` and ``%(normal)s`` will be colored
depending on the level if color support is on.
:arg dict colors: color mappings from logging level to terminal color
code
:arg string datefmt: Datetime format.
Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``.
"""
logging.Formatter.__init__(self, datefmt=datefmt)
self._fmt = fmt
self._colors = {}
if color and _stderr_supports_color():
# The curses module has some str/bytes confusion in
# python3. Until version 3.2.3, most methods return
# bytes, but only accept strings. In addition, we want to
# output these strings with the logging module, which
# works with unicode strings. The explicit calls to
# unicode() below are harmless in python2 but will do the
# right conversion in python 3.
fg_color = (curses.tigetstr("setaf") or
curses.tigetstr("setf") or "")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = unicode_type(fg_color, "ascii")
for levelno, code in colors.items():
self._colors[levelno] = unicode_type(curses.tparm(fg_color, code), "ascii")
self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii")
else:
self._normal = ''
def __init__(self, color, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
self._color = color
if color:
# The curses module has some str/bytes confusion in python3.
# Most methods return bytes, but only accept strings.
# The explict calls to unicode() below are harmless in python2,
# but will do the right conversion in python3.
try:
fg_color = unicode(curses.tigetstr("setaf") or
curses.tigetstr("setf") or "", "ascii")
self._colors = {
logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue
"ascii"),
logging.INFO: unicode(curses.tparm(fg_color, 2), # Green
"ascii"),
logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow
"ascii"),
logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red
"ascii"),
}
self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
except: # curses is not setup when not running in a tty (service)
bash_colors = dict(red=31, green=32, yellow=33, blue=34, pink=35, cyan=36, white=37)
self._colors = {
logging.DEBUG: '\033[%s;%sm' % (0, bash_colors['blue']),
logging.INFO: '\033[%s;%sm' % (0, bash_colors['green']),
logging.WARNING: '\033[%s;%sm' % (0, bash_colors['yellow']),
logging.ERROR: '\033[%s;%sm' % (0, bash_colors['red'])
}
self._normal = '\033[0m'
def __init__(self):
""" Initialization """
dict.__init__(self, {"NORMAL": "", "BOLD": "", "ERASE": "\n", "RED": "", "YELLOW": "", "GREEN": ""})
try:
import curses as _curses
except ImportError:
# fixup if a submodule of curses failed.
if "curses" in _sys.modules:
del _sys.modules["curses"]
else:
try:
_curses.setupterm()
except (TypeError, _curses.error):
pass
else:
def make_color(color):
""" Make color control string """
seq = _curses.tigetstr("setaf").decode("ascii")
if seq is not None:
# XXX may fail - need better logic
seq = seq.replace("%p1", "") % color
return seq
self["NORMAL"] = _curses.tigetstr("sgr0").decode("ascii")
self["BOLD"] = _curses.tigetstr("bold").decode("ascii")
erase = _curses.tigetstr("el1").decode("ascii")
if erase is not None:
self["ERASE"] = erase + _curses.tigetstr("cr").decode("ascii")
self["RED"] = make_color(_curses.COLOR_RED)
self["YELLOW"] = make_color(_curses.COLOR_YELLOW)
self["GREEN"] = make_color(_curses.COLOR_GREEN)
def __init__(self, width=None):
"Constructor."
self._progress = None
self._lines = 0
self._PADDING = 7
curses.setupterm()
fgColorSeq = curses.tigetstr('setaf') or curses.tigetstr('setf') or ''
colorIndex = getattr(curses, 'COLOR_WHITE')
self._fgcolour = curses.tparm(fgColorSeq, colorIndex)
bgColorSeq = curses.tigetstr('setab') or curses.tigetstr('setb') or ''
colorIndex = getattr(curses, 'COLOR_BLACK')
self._bgcolour = curses.tparm(bgColorSeq, colorIndex)
self._control = {}
for control in _CONTROLS:
# Set the control escape sequence
self._control[control] = curses.tigetstr(_CONTROLS[control]) or ''
self._value = {}
for value in _VALUES:
# Set terminal related values
self._value[value] = curses.tigetnum(_VALUES[value])
if width and width < self._value["COLUMNS"] - self._PADDING:
self._width = width
else:
self._width = self._value["COLUMNS"] - self._PADDING
def __init__(self, colors=DEFAULT_COLORS):
enable = False
if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
enable = True
except Exception:
pass
self._colors = {}
if enable:
fg_color = (curses.tigetstr("setaf") or
curses.tigetstr("setf") or "")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = unicode_type(fg_color, "ascii")
for name, code in colors.items():
self._colors[name] = unicode_type(curses.tparm(fg_color, code), "ascii")
self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii")
else:
self._normal = ''
self.enable = enable
def colourise_output(start, line, end="reset"):
"""
Wrapper method to return colourised output when running in a terminal.
:param start: One of 'red', 'green', 'yellow' or 'cyan'
:type start: str.
:param line: Text to be colourised.
:type line: str.
:param end: Typically, 'reset' to clear the colourisation.
:type end: str.
:returns: str.
"""
# Simple coloured output
if not is_term():
return line
if colourise_output.init == 0:
curses.setupterm()
colourise_output.init = 1
colours = {
"reset": curses.tparm(curses.tigetstr("op")),
"red": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_RED),
"green": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_GREEN),
"yellow": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_YELLOW),
"cyan": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_CYAN),
}
return colours[start] + line + colours[end]
def render(char_map, characters, color_map=None):
"""
Takes ascii & color data, and display it on the screen
"""
import curses
curses.setupterm()
fg_normal = curses.tigetstr('sgr0')
fg_colors = [curses.tparm(curses.tigetstr('setaf'), i) for i in range(8)]
attr_bold = curses.tparm(curses.tigetstr('bold'), curses.A_BOLD)
attr_normal = curses.tparm(curses.tigetstr('sgr0'), curses.A_NORMAL)
def set_color(fg=None):
if fg is None:
return fg_normal + attr_normal
if fg in range(0, 8):
return fg_colors[fg]
if fg in range(8, 16):
return fg_colors[fg-8] + attr_bold
return ''
for y in range(len(char_map)):
for x in range(len(char_map[y])):
if color_map is not None:
print(set_color(color_map[y][x]), end='')
print(characters[char_map[y][x]], end='')
print('')
if color_map is not None:
print(set_color(), end='')
def __init__(self, console, *args, **kwargs):
#logging.Formatter.__init__(self, *args, **kwargs)
logging.Formatter.__init__(self, "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
color = False
if console:
if curses and sys.stderr.isatty():
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except Exception:
pass
self._color = color
if color:
# The curses module has some str/bytes confusion in python3.
# Most methods return bytes, but only accept strings.
# The explict calls to unicode() below are harmless in python2,
# but will do the right conversion in python3.
fg_color = unicode(curses.tigetstr("setaf") or
curses.tigetstr("setf") or "", "ascii")
self._colors = {
logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue
"ascii"),
logging.INFO: unicode(curses.tparm(fg_color, 2), # Green
"ascii"),
logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow
"ascii"),
logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red
"ascii"),
}
self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def setup(cls):
if getattr(cls, '_setup', False):
return
curses.setupterm()
cls.BOL = curses.tigetstr('cr')
cls.CLEAR_EOL = curses.tigetstr('el')
cls._setup = True
def terminal_has_colors():
if sys.platform=='cygwin' and 'USE_COLOR' not in os.environ:
# Avoid importing curses that causes illegal operation
# with a message:
# PYTHON2 caused an invalid page fault in
# module CYGNURSES7.DLL as 015f:18bbfc28
# Details: Python 2.3.3 [GCC 3.3.1 (cygming special)]
# ssh to Win32 machine from debian
# curses.version is 2.2
# CYGWIN_98-4.10, release 1.5.7(0.109/3/2))
return 0
if hasattr(sys.stdout,'isatty') and sys.stdout.isatty():
try:
import curses
curses.setupterm()
if (curses.tigetnum("colors") >= 0
and curses.tigetnum("pairs") >= 0
and ((curses.tigetstr("setf") is not None
and curses.tigetstr("setb") is not None)
or (curses.tigetstr("setaf") is not None
and curses.tigetstr("setab") is not None)
or curses.tigetstr("scp") is not None)):
return 1
except Exception:
pass
return 0
def __init__(self, color, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
self._color = color
if color:
fg_color = (curses.tigetstr("setaf") or
curses.tigetstr("setf") or "")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = six.text_type(fg_color, "ascii")
self._colors = {
logging.DEBUG: six.text_type(
curses.tparm(fg_color, 4),
"ascii"
), # Blue
logging.INFO: six.text_type(
curses.tparm(fg_color, 2),
"ascii"
), # Green
logging.WARNING: six.text_type(
curses.tparm(fg_color, 3),
"ascii"
), # Yellow
logging.ERROR: six.text_type(
curses.tparm(fg_color, 1),
"ascii"
), # Red
}
self._normal = six.text_type(curses.tigetstr("sgr0"), "ascii")
display = Display()
try:
import curses
# Nest the try except since curses.error is not available if curses did not import
try:
curses.setupterm()
HAS_CURSES = True
except curses.error:
HAS_CURSES = False
except ImportError:
HAS_CURSES = False
if HAS_CURSES:
MOVE_TO_BOL = curses.tigetstr('cr')
CLEAR_TO_EOL = curses.tigetstr('el')
else:
MOVE_TO_BOL = b'\r'
CLEAR_TO_EOL = b'\x1b[K'
def clear_line(stdout):
stdout.write(b'\x1b[%s' % MOVE_TO_BOL)
stdout.write(b'\x1b[%s' % CLEAR_TO_EOL)
class InnerFailure(Exception):
def __init__(self, result):
self.result = result
super().__init__()
def main(argv):
if len(argv) < 2 or len(argv) > 3:
print("Usage: droidbox.py filename.apk output <duration in seconds>")
sys.exit(1)
duration = 0
#Duration given?
if len(argv) == 3:
duration = int(argv[2])
apkName = sys.argv[1]
#APK existing?
if os.path.isfile(apkName) == False:
print("File %s not found" % argv[1])
sys.exit(1)
application = Application(apkName)
ret = application.processAPK()
#Error during the APK processing?
if (ret == 0):
print("Failed to analyze the APK. Terminate the analysis.")
sys.exit(1)
activities = application.getActivities()
mainActivity = application.getMainActivity()
packageName = application.getPackage()
recvsaction = application.getRecvsaction()
enfperm = application.getEnfperm()
#Get the hashes
hashes = application.getHashes()
curses.setupterm()
sys.stdout.write(curses.tigetstr("clear"))
sys.stdout.flush()
call(['adb', 'logcat', '-c'])
print " ____ __ ____"
print "/\ _`\ __ /\ \/\ _`\\"
print "\ \ \/\ \ _ __ ___ /\_\ \_\ \ \ \L\ \ ___ __ _"
print " \ \ \ \ \/\`'__\ __`\/\ \ /'_` \ \ _ <' / __`\/\ \/'\\"
print " \ \ \_\ \ \ \/\ \L\ \ \ \/\ \L\ \ \ \L\ \\ \L\ \/> </"
print " \ \____/\ \_\ \____/\ \_\ \___,_\ \____/ \____//\_/\_\\"
print " \/___/ \/_/\/___/ \/_/\/__,_ /\/___/ \/___/ \//\/_/"
#No Main acitvity found? Return an error
if mainActivity == None:
print("No activity to start. Terminate the analysis.")
sys.exit(1)
#No packages identified? Return an error
if packageName == None:
print("No package found. Terminate the analysis.")
sys.exit(1)
#Execute the application
ret = call(['monkeyrunner', 'monkeyrunner.py', apkName, packageName, mainActivity], stderr=PIPE, cwd=os.path.dirname(os.path.realpath(__file__)))
if (ret == 1):
print("Failed to execute the application.")
sys.exit(1)
print("Starting the activity %s..." % mainActivity)
#By default the application has not started
applicationStarted = 0
stringApplicationStarted = "Start proc %s" % packageName
#Open the adb logcat
adb = Popen(["adb", "logcat", "DroidBox:W", "dalvikvm:W", "ActivityManager:I"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
#Wait for the application to start
while 1:
try:
logcatInput = adb.stdout.readline()
if not logcatInput:
raise Exception("We have lost the connection with ADB.")
#Application started?
if (stringApplicationStarted in logcatInput):
applicationStarted = 1
break;
except:
break
if (applicationStarted == 0):
print("Analysis has not been done.")
#Kill ADB, otherwise it will never terminate
os.kill(adb.pid, signal.SIGTERM)
sys.exit(1)
print("Application started")
print("Analyzing the application during %s seconds..." % (duration if (duration !=0) else "infinite time"))
count = CountingThread()
count.start()
timeStamp = time.time()
if duration:
signal.signal(signal.SIGALRM, interruptHandler)
signal.alarm(duration)
#Collect DroidBox logs
lastScreenshot = 0
while 1:
if (time.time() - lastScreenshot) >=3:
#Take Screenshots every 3 seconds.
os.system("adb shell screencap -p | sed 's/\r$//' > /samples/out/screen_$(date +%Y-%m-%d_%H%M%S).png")
lastScreenshot = time.time()
try:
logcatInput = adb.stdout.readline()
if not logcatInput:
raise Exception("We have lost the connection with ADB.")
boxlog = logcatInput.split('DroidBox:')
if len(boxlog) > 1:
try:
load = json.loads(decode(boxlog[1]))
# DexClassLoader
if load.has_key('DexClassLoader'):
load['DexClassLoader']['type'] = 'dexload'
dexclass[time.time() - timeStamp] = load['DexClassLoader']
count.increaseCount()
# service started
if load.has_key('ServiceStart'):
load['ServiceStart']['type'] = 'service'
servicestart[time.time() - timeStamp] = load['ServiceStart']
count.increaseCount()
# received data from net
if load.has_key('RecvNet'):
host = load['RecvNet']['srchost']
port = load['RecvNet']['srcport']
recvnet[time.time() - timeStamp] = recvdata = {'type': 'net read', 'host': host, 'port': port, 'data': load['RecvNet']['data']}
count.increaseCount()
# fdaccess
if load.has_key('FdAccess'):
accessedfiles[load['FdAccess']['id']] = hexToStr(load['FdAccess']['path'])
# file read or write
if load.has_key('FileRW'):
load['FileRW']['path'] = accessedfiles[load['FileRW']['id']]
if load['FileRW']['operation'] == 'write':
load['FileRW']['type'] = 'file write'
else:
load['FileRW']['type'] = 'file read'
fdaccess[time.time()-timeStamp] = load['FileRW']
count.increaseCount()
# opened network connection log
if load.has_key('OpenNet'):
opennet[time.time()-timeStamp] = load['OpenNet']
count.increaseCount()
# closed socket
if load.has_key('CloseNet'):
closenet[time.time()-timeStamp] = load['CloseNet']
count.increaseCount()
# outgoing network activity log
if load.has_key('SendNet'):
load['SendNet']['type'] = 'net write'
sendnet[time.time()-timeStamp] = load['SendNet']
count.increaseCount()
# data leak log
if load.has_key('DataLeak'):
my_time = time.time()-timeStamp
load['DataLeak']['type'] = 'leak'
load['DataLeak']['tag'] = getTags(int(load['DataLeak']['tag'], 16))
dataleaks[my_time] = load['DataLeak']
count.increaseCount()
if load['DataLeak']['sink'] == 'Network':
load['DataLeak']['type'] = 'net write'
sendnet[my_time] = load['DataLeak']
count.increaseCount()
elif load['DataLeak']['sink'] == 'File':
load['DataLeak']['path'] = accessedfiles[load['DataLeak']['id']]
if load['DataLeak']['operation'] == 'write':
load['DataLeak']['type'] = 'file write'
else:
load['DataLeak']['type'] = 'file read'
fdaccess[my_time] = load['DataLeak']
count.increaseCount()
elif load['DataLeak']['sink'] == 'SMS':
load['DataLeak']['type'] = 'sms'
sendsms[my_time] = load['DataLeak']
count.increaseCount()
# sent sms log
if load.has_key('SendSMS'):
load['SendSMS']['type'] = 'sms'
sendsms[time.time()-timeStamp] = load['SendSMS']
count.increaseCount()
# phone call log
if load.has_key('PhoneCall'):
load['PhoneCall']['type'] = 'call'
phonecalls[time.time()-timeStamp] = load['PhoneCall']
count.increaseCount()
# crypto api usage log
if load.has_key('CryptoUsage'):
load['CryptoUsage']['type'] = 'crypto'
cryptousage[time.time()-timeStamp] = load['CryptoUsage']
count.increaseCount()
except ValueError:
pass
except:
try:
count.stopCounting()
count.join()
finally:
break;
#Kill ADB, otherwise it will never terminate
os.kill(adb.pid, signal.SIGTERM)
#Done? Store the objects in a dictionary, transform it in a JSON object and return it
output = dict()
#Sort the items by their key
output["dexclass"] = dexclass
output["servicestart"] = servicestart
output["recvnet"] = recvnet
output["opennet"] = opennet
output["sendnet"] = sendnet
output["closenet"] = closenet
output["accessedfiles"] = accessedfiles
output["dataleaks"] = dataleaks
output["fdaccess"] = fdaccess
output["sendsms"] = sendsms
output["phonecalls"] = phonecalls
output["cryptousage"] = cryptousage
output["recvsaction"] = recvsaction
output["enfperm"] = enfperm
output["hashes"] = hashes
output["apkName"] = apkName
with open("/tmp/analysis.json", "w", 0) as jsonfile:
jsonfile.write(json.dumps(output,sort_keys=True, indent=4))
jsonfile.flush()
os.fsync(jsonfile.fileno())
jsonfile.close()
print(json.dumps(output))
sys.stdout.flush()
sys.exit(0)
def main(username, sharer, wlist, num):
"""Create UDFs, folders and files for the given user using a wordlist."""
user = get_storage_user(None, username=utf2unicode(username))
sharer = get_storage_user(None, username=utf2unicode(sharer))
folders = [user.root]
names = [utf2unicode(i.strip()) for i in file(wlist) if i.strip()]
sample = random.sample(names, num)
if sys.stdout.isatty():
import curses
curses.setupterm()
cols = curses.tigetnum('cols') - 1
progbar = makeStatBar(cols, cols - 2)
home = curses.tigetstr('cr')
def progress(l):
"""progress bar writer."""
sys.stdout.write(home + progbar((cols - 2) * (num - len(l)) / num))
sys.stdout.flush()
else:
progress = lambda l: None
# UDF
udf = user.make_udf(u'~/abundant-files')
folders.append(user.volume(udf.id).get_node(udf.root_id))
# some UDFs
for i in range(num / 100):
progress(sample)
folders.append(make_udf(user, sample))
for i in range(num / 4):
progress(sample)
name = sample.pop()
folders.append(random.choice(folders).make_subdirectory(name))
sh_folders = [sharer.root]
for i in range(num / 10):
progress(sample)
sh_folders.append(make_udf(sharer, sample))
for i in range(num / 10):
progress(sample)
name = sample.pop()
sh_folders.append(random.choice(sh_folders).make_subdirectory(name))
for i in range(num / 20):
progress(sample)
name = sample.pop()
filename = u'shared by ' + sharer.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(sh_folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(user.id, folder.name, readonly)
user.get_share(share.id).accept()
for i in random.sample(folders, len(folders) / 4):
progress(sample)
name = sample.pop()
filename = u'shared by ' + user.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(sharer.id, folder.name, readonly)
sharer.get_share(share.id).accept()
for i in range(num / 20):
progress(sample)
name = sample.pop()
random.choice(folders).make_file(name)
fake_hash = get_fake_hash()
while sample:
progress(sample)
name = sample.pop()
random.choice(folders).make_file_with_content(name, fake_hash, 12345,
100, 10000, uuid.uuid4(),
u'image/tiff')
if sys.stdout.isatty():
sys.stdout.write(home + curses.tigetstr('el'))
def test_posix_printengine(output_file, ttymode):
"""Test driver for POSIX print engine. This is maintained as a
standalone function in order to support the 'runprintengine' test
utility in $SRC/tests/interactive/runprintengine.py; it is also
called by the test suite."""
pe = POSIXPrintEngine(output_file, ttymode=ttymode)
standout = ""
sgr0 = ""
if ttymode:
# We assume that setupterm() has been called already.
standout = curses.tigetstr("smso") or ""
sgr0 = curses.tigetstr("sgr0") or ""
pe.cprint("Testing POSIX print engine; ttymode is {0}\n".format(ttymode))
# If we're not in ttymode, then the testing is simple.
if not ttymode:
pe.cprint("testing 1 2 3")
pe.cprint("testing flush (2)")
pe.flush()
return
# We assume setupterm() has been called.
standout = curses.tigetstr("smso") or ""
sgr0 = curses.tigetstr("sgr0") or ""
pe.cprint("Now we'll print something and then erase it;")
pe.cprint("you should see a blank line below this line.")
pe.cprint("IF YOU CAN SEE THIS, THE TEST HAS FAILED", end='')
pe.cprint("", erase=True)
pe.cprint("You should see an X swishing back and forth; from")
pe.cprint("left to right it should be inverse.")
# Unused variable 'y'; pylint: disable=W0612
for y in range(0, 2):
for x in xrange(0, 30, 1):
pe.cprint(" " * x, erase=True, end='')
pe.putp(standout)
pe.cprint("X", end='')
pe.putp(sgr0)
time.sleep(0.050)
for x in xrange(30, -1, -1):
pe.cprint(" " * x + "X", erase=True, end='')
time.sleep(0.050)
pe.cprint("", erase=True)
pe.cprint("testing 1 2 3")
pe.cprint("testing XX XX XX", end="")
time.sleep(0.500)
pe.cprint("testing 4 5 6\ntesting XX XX XX", erase=True, end="")
time.sleep(0.500)
pe.cprint("testing YY YY", end="", erase=True)
time.sleep(0.500)
pe.cprint("testing 7 8 9\ntesting 10 11 12", erase=True)
time.sleep(0.500)
pe.cprint("testing ZZ ZZ ZZ ZZ ZZ", end="")
time.sleep(0.500)
pe.cprint("testing 13 14 15", erase=True)
pe.cprint("testing flush...", end='')
pe.flush()
pe.cprint("This should be on the next line.")
pe.cprint("testing flush (2)")
pe.flush()
pe.cprint("This should be on the next line (with no nl's intervening).")
# just test that it works
pe.isslow()
def get_tparm(key):
val = None
key = curses.tigetstr("sgr0")
if key: val = curses.tparm(key)
if val: val = val.decode('utf-8')
return val
def end_line(self):
return self.reset + curses.tigetstr("el")
def reset(self):
return curses.tigetstr('sgr0')
def foreground(color):
return curses.tparm(curses.tigetstr('setaf'), color)
def background(color):
return curses.tparm(curses.tigetstr('setab'), color)
def _tigetstr(self, cap_name):
import curses
cap = curses.tigetstr(cap_name) or ''
return re.sub('\\$<\\d+>[/*]?', '', cap)
def move_cur(to_y, to_x):
tparm = curses.tparm(curses.tigetstr("cup"), to_y, to_x)
# on python2 stdout is already in binary mode, in python3 is accessed via buffer
bin_stdout = getattr(sys.stdout, 'buffer', sys.stdout)
bin_stdout.write(tparm)
def _terminal_restore_color():
restore_code = curses.tigetstr('sgr0')
if restore_code:
sys.stdout.write(_unicode(restore_code))
def unicode_parm(cap, *parms):
"""Return the result of ``tparm(tigetstr())`` except as Unicode."""
return tparm(tigetstr(cap), *parms).decode('utf-8')
def cmd(self, name, *args):
s = curses.tigetstr(name)
sys.stdout.buffer.write(curses.tparm(s, *args))
def unicode_cap(cap):
"""Return the result of ``tigetstr`` except as Unicode."""
return tigetstr(cap).decode('utf-8')
def clear(self):
sys.stdout.write(
str(curses.tigetstr('cr')) + str(curses.tigetstr('el')))
def _tigetstr(self, cap_name):
import curses
cap = curses.tigetstr(cap_name) or b''
return re.sub(r'\$<\d+>[/*]?', '', cap.decode()).encode()
if sys.stdout and sys.stdout.isatty():
try:
import curses
curses.setupterm()
except (ImportError, OSError) as ex:
if __debug__:
print('Warning', ex, sep=': ', end='\n\n', file=sys.stderr)
curses = None
else:
curses = None
if curses is None:
TERMMODES = dict.fromkeys(map(itemgetter(0), TERMMODES), '')
else:
TERMMODES = {
k: (curses.tigetstr(capname) or b'').decode('ascii')
for k, capname in TERMMODES
}
def try_input(prompt=None, on_eof='', end='\n? '):
"""Similar to input() but return a default response on EOF or EBADF.
If input() fails with EOFError or due to a bad (e. g. closed) standard output
stream return 'on_eof' instead which defaults to the empty string.
Additionally wrap the prompt string using termwrap (see below). 'end' is
always appended to the prompt and defaults to '\n? '.
"""
if prompt:
def get_hex_chars(key_str):
if (key_str.startswith("0x")):
return key_str[2:].decode('hex')
else:
return curses.tigetstr(key_str)
import sys
import os
import re
import libvirt
import subprocess
import logging
from optparse import OptionParser
from lxml import etree as ET
# The expat parser fails on control characters
from xml.parsers.expat import ExpatError
try:
import curses
curses.setupterm()
NORMAL = curses.tigetstr('sgr0').decode('ascii')
SET_AF = curses.tigetstr('setaf')
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = [curses.tparm(SET_AF, i).decode('ascii') for i in range(8)]
except (ImportError, curses.error):
NORMAL = BLACK = RED = GREEN = YELLOW = BLUE = MAGENTA = CYAN = WHITE = ''
class Dotter(object):
"""Handle dot graph."""
RE_KEY = re.compile('[^0-9A-Za-z]')
def __init__(self, out=None):
self.dot_out = out
@staticmethod
def make_color(color):
""" Make color control string """
seq = _curses.tigetstr(bc('setaf'))
if seq is not None:
seq = _curses.tparm(seq, color).decode('ascii')
return seq
def module_funcs(stdscr):
"Test module-level functions"
for func in [curses.baudrate, curses.beep, curses.can_change_color,
curses.cbreak, curses.def_prog_mode, curses.doupdate,
curses.filter, curses.flash, curses.flushinp,
curses.has_colors, curses.has_ic, curses.has_il,
curses.isendwin, curses.killchar, curses.longname,
curses.nocbreak, curses.noecho, curses.nonl,
curses.noqiflush, curses.noraw,
curses.reset_prog_mode, curses.termattrs,
curses.termname, curses.erasechar, curses.getsyx]:
func()
# Functions that actually need arguments
if curses.tigetstr("cnorm"):
curses.curs_set(1)
curses.delay_output(1)
curses.echo() ; curses.echo(1)
f = tempfile.TemporaryFile()
stdscr.putwin(f)
f.seek(0)
curses.getwin(f)
f.close()
curses.halfdelay(1)
curses.intrflush(1)
curses.meta(1)
curses.napms(100)
curses.newpad(50,50)
win = curses.newwin(5,5)
win = curses.newwin(5,5, 1,1)
curses.nl() ; curses.nl(1)
curses.putp('abc')
curses.qiflush()
curses.raw() ; curses.raw(1)
curses.setsyx(5,5)
curses.tigetflag('hc')
curses.tigetnum('co')
curses.tigetstr('cr')
curses.tparm('cr')
curses.typeahead(sys.__stdin__.fileno())
curses.unctrl('a')
curses.ungetch('a')
curses.use_env(1)
# Functions only available on a few platforms
if curses.has_colors():
curses.start_color()
curses.init_pair(2, 1,1)
curses.color_content(1)
curses.color_pair(2)
curses.pair_content(curses.COLOR_PAIRS - 1)
curses.pair_number(0)
if hasattr(curses, 'use_default_colors'):
curses.use_default_colors()
if hasattr(curses, 'keyname'):
curses.keyname(13)
if hasattr(curses, 'has_key'):
curses.has_key(13)
if hasattr(curses, 'getmouse'):
(availmask, oldmask) = curses.mousemask(curses.BUTTON1_PRESSED)
# availmask indicates that mouse stuff not available.
if availmask != 0:
curses.mouseinterval(10)
# just verify these don't cause errors
m = curses.getmouse()
curses.ungetmouse(*m)
if hasattr(curses, 'is_term_resized'):
curses.is_term_resized(*stdscr.getmaxyx())
if hasattr(curses, 'resizeterm'):
curses.resizeterm(*stdscr.getmaxyx())
if hasattr(curses, 'resize_term'):
curses.resize_term(*stdscr.getmaxyx())
def build_word2vec(self,
f_name_training_sample,
f_name_prepared_subtitles=None,
f_name_enc_training_sample=None,
f_name_w2v_model=None,
f_name_w2v_vocab=None,
len_encode=None,
size=500,
min_count_repeat=1,
window=5,
epochs=None,
logging=False):
''' Предназначен для построения модели word2vec и кодирования обучающей выборки в вектора.
1. f_name_training_sample - имя входного .pkl файла с предварительно обработанными парами [вопрос,ответ]
2. f_name_prepared_subtitles - имя .pkl файла с предварительно обработанными субтитрами (для расширения словаря модели word2vec)
3. f_name_enc_training_sample - имя выходного .npz файла с векторным представлением слов в парах [вопрос,ответ] (по умолчанию encoded_+f_name_training_sample+.npz)
4. f_name_w2v_model - имя .bin файла для сохранения обученной модели word2vec (по умолчанию w2v_model_+f_name_training_sample+.bin)
5. f_name_w2v_vocab - имя .txt файла для сохранения словаря word2vec (по умолчанию w2v_vocabulary_+f_name_training_sample+.txt)
6. len_encode - если None: закодировать весь f_name_training_sample, иначе - первые len_encode элементов
7. size - рразмерность вектора, которым кодируется одно слово
Обычно используют значения в несколько сотен, большие значения требуют больше данных обучения, но могут привести к более точным моделям.
8. min_count - минимальное количество повторений (частота повторений) слова, что бы оно было включено в словарь
9. window - максимальное расстояние между текущим и прогнозируемым словом в предложении
10. epochs - число эпох обучения модели word2vec (если None, используется внутренний итератор)
11. logging - включение вывода данных в процессе обучения модели word2vec
Входные предложения должны иметь вид, например: [['<PAD>', ..., '<PAD>', '?', 'класс', 'этот', 'нужен', 'Зачем', '<GO>'],
['Для', 'кодирования', 'предложений', '<EOS>', '<PAD>', ..., '<PAD>']] '''
if f_name_enc_training_sample is None:
f_name_enc_training_sample = f_name_training_sample.replace(
'prepared_', 'encoded_')
f_name_enc_training_sample = f_name_enc_training_sample.replace(
'.pkl', '.npz')
if f_name_w2v_model is None:
f_name_w2v_model = f_name_training_sample.replace(
'prepared_', 'w2v_model_')
f_name_w2v_model = f_name_w2v_model.replace('.pkl', '.bin')
if f_name_w2v_vocab is None:
f_name_w2v_vocab = f_name_training_sample.replace(
'prepared_', 'w2v_vocabulary_')
f_name_w2v_vocab = f_name_w2v_vocab.replace('.pkl', '.txt')
if f_name_prepared_subtitles is not None:
print('[i] Загрузка субтитров из %s' % f_name_prepared_subtitles)
with open(f_name_prepared_subtitles, 'rb') as f_subtitles:
subtitles = pickle.load(f_subtitles)
print('\tколичество предложений: %i пар' % len(subtitles))
print('\tдлинна предложений: %i слов' % len(subtitles[0][0]))
print('[i] Загрузка данных из %s' % f_name_training_sample)
with open(f_name_training_sample, 'rb') as f_training_sample:
training_sample = pickle.load(f_training_sample)
print('\tколичество предложений: %i пар' % len(training_sample))
print('\tдлинна предложений: %i слов' % len(training_sample[0][0]))
if f_name_prepared_subtitles is not None:
print('[i] Объединение...')
dataset = subtitles + training_sample
else:
dataset = training_sample
print('[i] Подсчёт размера полного словаря...')
i = 0
vocabulary = []
for pair in dataset:
if i % 1000 == 0 or i == len(dataset) - 1:
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print('[i] Подсчёт размера полного словаря... %i из %i' %
(i, len(dataset)))
for sentence in pair:
for word in sentence:
vocabulary.append(word)
i += 1
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print('[i] Подсчёт размера полного словаря... %i из %i' %
(len(dataset), len(dataset)))
vocabulary = set(vocabulary)
print('\tпредполагаемый размер полного словаря: %i слов' %
len(vocabulary))
self.__w2v_fit(dataset, f_name_w2v_model, min_count_repeat, size,
window, epochs, logging)
dataset = training_sample
vocabulary = [w for w in self.model.wv.vocab.keys()]
print('[i] Размер словаря word2vec: %i слов' % len(vocabulary))
with open(f_name_w2v_vocab, 'w') as f_w2v_vocab:
for word in vocabulary:
print(word, file=f_w2v_vocab)
# Оптимизация, что бы модель word2vec занимала в оперативной памяти меньше места (почему-то искажает декодировку векторов в слова)
#self.model = self.model.wv
#self.model.init_sims(replace=True)
self.data_w2v_encode(dataset, f_name_enc_training_sample, len_encode)
def assessment_training_accuracy(self,
f_name_enc_training_sample,
f_name_wrong_answers=None):
''' Оценка точности обучения сети: подаёт на вход сети все вопросы из обучающей выборки и сравнивает полученный ответ сети с
ответом из обучающей выборки.
1. f_name_enc_training_sample - имя .npz файла с векторным представлением слов в парах [вопрос,ответ]
2. f_name_wrong_answers - имя .txt файла для сохранения неправильных ответов сети (по умолчанию data/wrong_answers.txt) '''
if f_name_wrong_answers is None:
f_name_wrong_answers = f_name_enc_training_sample.replace(
'encoded_', 'wrong_answers_')
f_name_wrong_answers = f_name_wrong_answers.replace('.npz', '.txt')
print('[i] Оценка точности обучения модели...')
npzfile = np.load(f_name_enc_training_sample)
questions, answers = npzfile['questions'], npzfile['answers']
questions = (questions + 1.0) * 0.5
correct_answers = 0
wrong_answers = []
len_questions = len(questions)
print(
'[i] Оценено 0 из %i, правильных ответов 0, текущая точность 0.00%%'
% len_questions)
for i in range(len_questions):
if i % 10 == 0 and i > 0:
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print(
'[i] Оценено %i из %i, правильных ответов %i, текущая точность %.2f%%'
% (i, len_questions, correct_answers,
correct_answers / i * 100))
answer = self.model.predict(questions[i][np.newaxis, :])
answer = answer * 2.0 - 1.0
answer = self.w2v.vec2word(answer[0])
#answer = self.stp.prepare_answer(answer)
answer_etalon = self.w2v.vec2word(answers[i])
#answer_etalon = self.stp.prepare_answer(answer_etalon)
if answer == answer_etalon:
#if np.all(answer[0] == answers[i]):
correct_answers += 1
else:
# Сохранение неправильных ответов для последующего вывода
quest = self.w2v.vec2word(questions[i])
quest = list(reversed(quest))
quest = self.stp.prepare_answer(quest)
#answer = self.w2v.vec2word(answer[0])
answer = self.stp.prepare_answer(answer)
wrong_answers.append([quest, answer])
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print(
'[i] Оценено %i из %i, правильных ответов %i, текущая точность %.2f%%'
% (len_questions, len_questions, correct_answers,
correct_answers / len_questions * 100))
accuracy = correct_answers / len(answers) * 100
print(
'[i] Количество правильных ответов %i из %i, итоговая точность %.2f%%'
% (correct_answers, len(answers), accuracy))
if len(wrong_answers) < 50:
i = 0
print('[i] Неправильные ответы:')
for phrase in wrong_answers:
i += 1
print('%i. %s %%%% %s' % (i, phrase[0], phrase[1]))
if accuracy > 75:
with open(f_name_wrong_answers, 'w') as f_wrong_answers:
for phrase in wrong_answers:
f_wrong_answers.write(phrase[0] + ' %% ' + phrase[1] +
'\n')
import curses
import selectors
import shutil
import signal
import sys
import termios
import tty
from contextlib import contextmanager
curses.setupterm()
CSI = '\033['
# tigertstr uses \033O (SS3) instead of \033[ (CSI) as prefix
# https://en.wikipedia.org/wiki/ANSI_escape_code
KEY_BACKSPACE = curses.tigetstr('kbs').decode('ascii')
KEY_ESC = '\x1b'
KEY_HOME = CSI + 'H'
KEY_END = CSI + 'F'
KEY_PPAGE = CSI + '5~'
KEY_NPAGE = CSI + '6~'
KEY_UP = CSI + 'A'
KEY_DOWN = CSI + 'B'
KEY_RIGHT = CSI + 'C'
KEY_LEFT = CSI + 'D'
def isatty():
return os.isatty(sys.stdout.fileno())
def cap_string(s, *args):
cap = curses.tigetstr(s)
if cap:
return curses.tparm(cap, *args).decode("utf-8")
else:
return ''
def clear_screen():
import curses, sys
curses.setupterm()
sys.stdout.write(curses.tigetstr("clear"))
sys.stdout.flush()
def _tigetstr(self, cap_name):
# String capabilities can include "delays" of the form "$<2>".
# For any modern terminal, we should be able to just ignore
# these, so strip them out.
import curses
return curses.tigetstr(cap_name) or ''
import sys
import os
if os.name == 'nt':
n_cols = 80
set_fg = None
normal = None
else:
import curses
curses.setupterm()
n_cols = curses.tigetnum('cols') or 80
set_fg = curses.tigetstr('setf') or None
normal = curses.tigetstr('sgr0') or None
cursor_pos = 0
COLOURS = {
'BLACK': 0,
'BLUE': 1,
'GREEN': 2,
'CYAN': 3,
'RED': 4,
'MAGENTA': 5,
'YELLOW': 6,
'WHITE': 7,
}
def checking(msg, indent=2):
global cursor_pos
def _terminal_restore_color():
import curses, sys
sys.stdout.write(curses.tigetstr('sgr0'))