def __init__(self, color=True, *args, **kwargs): logging.Formatter.__init__(self, *args, **kwargs) self._color = color and _stderr_supports_color() if self._color: # The curses module has some str/bytes confusion in # python3. Until version 3.2.3, most methods return # bytes, but only accept strings. In addition, we want to # output these strings with the logging module, which # works with unicode strings. The explicit calls to # unicode() below are harmless in python2 but will do the # right conversion in python 3. fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): fg_color = unicode(fg_color, "ascii") self._colors = { logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue "ascii"), logging.INFO: unicode(curses.tparm(fg_color, 2), # Green "ascii"), logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow "ascii"), logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red "ascii"), } self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def __init__(self): ProgressTracker.__init__(self) self.act_started = False self.ind_started = False self.last_print_time = 0 self.clear_eol = "" try: import curses if not os.isatty(sys.stdout.fileno()): raise ProgressTrackerException() curses.setupterm() self.cr = curses.tigetstr("cr") self.clear_eol = curses.tigetstr("el") or "" except KeyboardInterrupt: raise except: if portable.ostype == "windows" and \ os.isatty(sys.stdout.fileno()): self.cr = '\r' else: raise ProgressTrackerException() self.dl_started = False self.spinner = 0 self.spinner_chars = "/-\|" self.curstrlen = 0
def __init__(self, out_file, ttymode): """Create a printengine. out_file -- the file object to print to ttymode -- Boolean indicating need for tty support. Throws PrintEngineException if out_file can't support. """ PrintEngine.__init__(self) self._out_file = out_file self.__nchars_printed = 0 self.__needs_nl = 0 self.__cr = None self.__ttymode = ttymode if not self.__ttymode: return self.__putp_re = re.compile("\$<[0-9]+>") self.__el = None if not self._out_file.isatty(): raise PrintEngineException("Not a TTY") try: curses.setupterm(None, self._out_file.fileno()) self.__cr = curses.tigetstr("cr") self.__el = curses.tigetstr("el") except curses.error: raise PrintEngineException("Unknown terminal '%s'" % os.environ.get("TERM", ""))
def __init__(self, color, *args, **kwargs): logging.Formatter.__init__(self, *args, **kwargs) self._color = color if color: # The curses module has some str/bytes confusion in python3. # Most methods return bytes, but only accept strings. # The explict calls to unicode() below are harmless in python2, # but will do the right conversion in python3. fg_color = unicode(curses.tigetstr("setaf") or curses.tigetstr("setf") or "", "ascii") self._colors = { logging.DEBUG: unicode( curses.tparm(fg_color, curses.COLOR_CYAN), "ascii"), logging.INFO: unicode( curses.tparm(fg_color, curses.COLOR_GREEN), "ascii"), logging.WARNING: unicode( curses.tparm(fg_color, curses.COLOR_YELLOW), # Yellow "ascii"), logging.ERROR: unicode( curses.tparm(fg_color, curses.COLOR_RED), # Red "ascii"), } self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def _SetupColours(): """Initializes the colour constants. """ # pylint: disable=W0603 # due to global usage global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ # Don't use colours if stdout isn't a terminal if not sys.stdout.isatty(): return try: import curses except ImportError: # Don't use colours if curses module can't be imported return curses.setupterm() _RESET_SEQ = curses.tigetstr("op") setaf = curses.tigetstr("setaf") _INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN) _WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW) _ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
def __init__(self, color=True, datefmt=None): r""" :arg bool color: Enables color support. :arg string fmt: Log message format. It will be applied to the attributes dict of log records. The text between ``%(color)s`` and ``%(end_color)s`` will be colored depending on the level if color support is on. :arg dict colors: color mappings from logging level to terminal color code :arg string datefmt: Datetime format. Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. .. versionchanged:: 3.2 Added ``fmt`` and ``datefmt`` arguments. """ logging.Formatter.__init__(self, datefmt=datefmt) self._colors = {} if color and _stderr_supports_color(): fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): fg_color = str(fg_color, "ascii") for levelno, code in self.DEFAULT_COLORS.items(): self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") self._normal = str(curses.tigetstr("sgr0"), "ascii") scr = curses.initscr() self.termwidth = scr.getmaxyx()[1] curses.endwin() else: self._normal = '' # Default width is usually 80, but too wide is worse than too narrow self.termwidth = 70
def _setupTerm(self): import curses curses.setupterm() self.COLS=curses.tigetnum('cols') self.BOL=curses.tigetstr('cr') self.UP=curses.tigetstr('cuu1') self.CLEAR_EOL=curses.tigetstr('el')
def draw(self, path, start_x, start_y, width, height): # Save cursor curses.putp(curses.tigetstr("sc")) y = start_y # Move to drawing zone self._move_to(start_x, y) # Write intent sys.stdout.write("%s}ic#%d;%d;%s%s" % ( self.display_protocol, width, height, path, self.close_protocol)) # Write Replacement commands ('#') for _ in range(0, height): sys.stdout.write("%s}ib%s%s%s}ie%s" % ( self.display_protocol, self.close_protocol, "#" * width, self.display_protocol, self.close_protocol)) y = y + 1 self._move_to(start_x, y) # Restore cursor curses.putp(curses.tigetstr("rc")) sys.stdout.flush()
def printwyx(self, window, coord, arg, color = None, attribute = None): if self.cursescolors == True: try: if color or attribute: attr = 0 if color: attr |= self.color[color] if attribute: attr |= self.attribute[attribute] if coord: window.addstr(coord[0], coord[1], arg, attr) else: window.addstr(arg, attr) elif coord: window.addstr(coord[0], coord[1], arg) else: window.addstr(arg) except curses.error: pass else: coordstr = "" if coord: coordstr = curses.tparm(curses.tigetstr("cup"), coord[0], coord[1]) colorstr = self.attribute['normal'] if color: colorstr += self.color[color] if attribute: colorstr += self.attribute[attribute] clr_eol = curses.tparm(curses.tigetstr("el")) curses.putp(coordstr + colorstr + arg + clr_eol)
def draw(self): """Draw all objects in the container""" self.win.touchwin() DisplayableContainer.draw(self) if self._draw_title and self.settings.update_title: cwd = self.fm.thisdir.path if cwd.startswith(self.fm.home_path): cwd = '~' + cwd[len(self.fm.home_path):] if self.settings.shorten_title: split = cwd.rsplit(os.sep, self.settings.shorten_title) if os.sep in split[0]: cwd = os.sep.join(split[1:]) try: fixed_cwd = cwd.encode('utf-8', 'surrogateescape'). \ decode('utf-8', 'replace') fmt_tup = ( curses.tigetstr('tsl').decode('latin-1'), fixed_cwd, curses.tigetstr('fsl').decode('latin-1'), ) except UnicodeError: pass else: sys.stdout.write("%sranger:%s%s" % fmt_tup) sys.stdout.flush() self.win.refresh()
def setup(cls): if getattr(cls, "_setup", False): return curses.setupterm() cls.BOL = curses.tigetstr("cr") cls.CLEAR_EOL = curses.tigetstr("el") cls._setup = True
def _tigetstr(cap_name): import curses if not curses.tigetstr(cap_name): return None from .utils import to_ascii cap = to_ascii(curses.tigetstr(cap_name)) or '' # String capabilities can include "delays" of the form "$<2>". # For any modern terminal, we should be able to just ignore # these, so strip them out. # terminof(5) states that: # A "/" suffix indicates that the padding is mandatory and forces a # delay of the given number of milliseconds even on devices for which # xon is present to indicate flow control. # So let's respect that. But not the optional ones. cap = re.sub(r'\$<\d+>[*]?', '', cap) # To switch back to "NORMAL", we use sgr0, which resets "everything" to defaults. # That on some terminals includes ending "alternate character set mode". # Which is usually done by appending '\017'. Unfortunately, less -R # does not eat that, but shows it as an annoying inverse '^O' # Instead of falling back to less -r, which would have other implications as well, # strip off that '\017': we don't use the alternative character set, # so we won't need to reset it either. if cap_name == 'sgr0': cap = re.sub(r'\017$', '', cap) return cap
def _InitColorize(output_file): """Initializes the colorization directives via --colorize as a mapping between compiled regular expressions meant to be matched against log filename and target curses color escape codes. The curses escape codes are represented as a pair containing codes for prepending and appending. Returns: [ColorDirective], or None if colorization is disabled or unavailable on the current terminal. Based on tornado's code to colorize log output. """ color = False # Set up color if we are in a tty, curses is available, and --colorize # was specified. if options.options.colorize is not None and curses and \ (options.options.color == 'yes' or \ (options.options.color == 'auto' and output_file.isatty())): try: curses.setupterm() if curses.tigetnum("colors") > 0: color = True except Exception: pass if not color: return [] directives = [] normal = unicode(curses.tigetstr("sgr0"), "ascii") fg_color = unicode(curses.tigetstr("setaf") or curses.tigetstr("setf") or "", "ascii") for directive in options.options.colorize: regexp, color_index = directive.split('=') color = unicode(curses.tparm(fg_color, int(color_index)), "ascii") directives.append(ColorDirective(re.compile(regexp), color, normal)) return directives
def __init__(self, color=True, fmt=DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT, colors=DEFAULT_COLORS, precision=3): r""" :arg bool color: Enables color support. :arg string fmt: Log message format. It will be applied to the attributes dict of log records. The text between ``%(color)s`` and ``%(end_color)s`` will be colored depending on the level if color support is on. :arg dict colors: color mappings from logging level to terminal color code :arg string datefmt: Datetime format. Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. .. versionchanged:: 3.2 Added ``fmt`` and ``datefmt`` arguments. """ super().__init__() self.default_time_format = datefmt self.precision = precision self.default_msec_format = '' self._fmt = fmt self._colors = {} if color and _stderr_supports_color(): fg_color = (curses.tigetstr('setaf') or curses.tigetstr('setf') or '') for levelno, code in colors.items(): self._colors[levelno] = curses.tparm(fg_color, code).decode() self._normal = curses.tigetstr('sgr0').decode() else: self._normal = ''
def __init__(self, buffer=sys.stdout, no_color=False): if not curses: no_color = True if not no_color: no_color = isinstance(sys.stdout, StringIO) or \ not sys.stdout.isatty() self.no_color = no_color # Don't bother initiating color if there's no color. if not no_color: # Get curses all ready to write some stuff to the screen. curses.setupterm() # Initialize a store for the colors and pre-populate it # with the un-color color. self.colors = {'NORMAL': curses.tigetstr('sgr0') or ''} # Determines capabilities of the terminal. fgColorSeq = curses.tigetstr('setaf') or \ curses.tigetstr('setf') or '' # Go through each color and figure out what the sequences # are for each, then store the sequences in the store we # made above. for color in COLORS: colorIndex = getattr(curses, 'COLOR_%s' % color) self.colors[color] = curses.tparm(fgColorSeq, colorIndex) self.buffer = buffer
def __init__(self, main, helper, console, format): self.main = main self.helper = helper self.cuu = None self.stdinbackup = None self.interactive = sys.stdout.isatty() self.footer_present = False self.lastpids = [] if not self.interactive: return import curses import termios import copy self.curses = curses self.termios = termios try: fd = sys.stdin.fileno() self.stdinbackup = termios.tcgetattr(fd) new = copy.deepcopy(self.stdinbackup) new[3] = new[3] & ~termios.ECHO termios.tcsetattr(fd, termios.TCSADRAIN, new) curses.setupterm() self.ed = curses.tigetstr("ed") if self.ed: self.cuu = curses.tigetstr("cuu") except: self.cuu = None console.addFilter(InteractConsoleLogFilter(self, format))
def add_color_to_string(string, color, stream=sys.stdout, bold=False, force=False): """Format the string to be printed with the given color. Insert formatting characters that, when printed on a terminal, will make the given string appear with the given foreground color if the stream passed has color support. Else return the string as it is. string (string): the string to color. color (int): the color as a colors constant, like colors.BLACK. stream (fileobj): a file-like object (that adheres to the API declared in the `io' package). Defaults to sys.stdout. bold (bool): True if the string should be bold. force (bool): True if the string should be formatted even if the given stream has no color support. return (string): the formatted string. """ if force or has_color_support(stream): return "%s%s%s%s" % ( curses.tparm(curses.tigetstr("setaf"), color).decode('ascii') if color != colors.BLACK else "", curses.tparm(curses.tigetstr("bold")).decode('ascii') if bold else "", string, curses.tparm(curses.tigetstr("sgr0")).decode('ascii') ) else: return string
def move_cursor(direction, amount=1, stream=sys.stdout, erase=False): """Move the cursor. If the stream is a TTY, print characters that will move the cursor in the given direction and optionally erase the line. Else do nothing. direction (int): the direction as a directions constant, like directions.UP. stream (fileobj): a file-like object (that adheres to the API declared in the `io' package). Defaults to sys.stdout. erase (bool): True if the line the cursor ends on should be erased. """ if stream.isatty(): if direction == directions.UP: print(curses.tparm(curses.tigetstr("cuu"), amount), file=stream, end='') elif direction == directions.DOWN: print(curses.tparm(curses.tigetstr("cud"), amount), file=stream, end='') elif direction == directions.LEFT: print(curses.tparm(curses.tigetstr("cub"), amount), file=stream, end='') elif direction == directions.RIGHT: print(curses.tparm(curses.tigetstr("cuf"), amount), file=stream, end='') if erase: print(curses.tparm(curses.tigetstr("el")), file=stream, end='') stream.flush()
def do_term(term, nick): curses.setupterm(term) sys.stdout.write("/* %s */\n" % term) sys.stdout.write("const char *%s_keys[] = {\n\t" % nick) for k, v in iter_pairs(keys): sys.stdout.write('"') puttc(curses.tigetstr(v)) sys.stdout.write('",') sys.stdout.write(" 0\n};\n") sys.stdout.write("const char *%s_funcs[] = {\n\t" % nick) for k, v in iter_pairs(funcs): # sys.stdout.write("\t[%s] = \"" % k) sys.stdout.write('"') if v == "sgr": sys.stdout.write("\\033[3%u;4%um") elif v == "cup": sys.stdout.write("\\033[%u;%uH") else: tc = curses.tigetstr(v) puttc(tc) if v == "rmkx": sys.stdout.write('"\n') else: sys.stdout.write('",') sys.stdout.write("};\n\n")
def __init__(self, color=True, fmt=DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT, colors=DEFAULT_COLORS): r""" :arg bool color: Enables color support. :arg string fmt: Log message format. It will be applied to the attributes dict of log records. The text between ``%(color)s`` and ``%(normal)s`` will be colored depending on the level if color support is on. :arg dict colors: color mappings from logging level to terminal color code :arg string datefmt: Datetime format. Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. """ logging.Formatter.__init__(self, datefmt=datefmt) self._fmt = fmt self._colors = {} if color and _stderr_supports_color(): # The curses module has some str/bytes confusion in # python3. Until version 3.2.3, most methods return # bytes, but only accept strings. In addition, we want to # output these strings with the logging module, which # works with unicode strings. The explicit calls to # unicode() below are harmless in python2 but will do the # right conversion in python 3. fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): fg_color = unicode_type(fg_color, "ascii") for levelno, code in colors.items(): self._colors[levelno] = unicode_type(curses.tparm(fg_color, code), "ascii") self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii") else: self._normal = ''
def __init__(self, color, *args, **kwargs): logging.Formatter.__init__(self, *args, **kwargs) self._color = color if color: # The curses module has some str/bytes confusion in python3. # Most methods return bytes, but only accept strings. # The explict calls to unicode() below are harmless in python2, # but will do the right conversion in python3. try: fg_color = unicode(curses.tigetstr("setaf") or curses.tigetstr("setf") or "", "ascii") self._colors = { logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue "ascii"), logging.INFO: unicode(curses.tparm(fg_color, 2), # Green "ascii"), logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow "ascii"), logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red "ascii"), } self._normal = unicode(curses.tigetstr("sgr0"), "ascii") except: # curses is not setup when not running in a tty (service) bash_colors = dict(red=31, green=32, yellow=33, blue=34, pink=35, cyan=36, white=37) self._colors = { logging.DEBUG: '\033[%s;%sm' % (0, bash_colors['blue']), logging.INFO: '\033[%s;%sm' % (0, bash_colors['green']), logging.WARNING: '\033[%s;%sm' % (0, bash_colors['yellow']), logging.ERROR: '\033[%s;%sm' % (0, bash_colors['red']) } self._normal = '\033[0m'
def __init__(self): """ Initialization """ dict.__init__(self, {"NORMAL": "", "BOLD": "", "ERASE": "\n", "RED": "", "YELLOW": "", "GREEN": ""}) try: import curses as _curses except ImportError: # fixup if a submodule of curses failed. if "curses" in _sys.modules: del _sys.modules["curses"] else: try: _curses.setupterm() except (TypeError, _curses.error): pass else: def make_color(color): """ Make color control string """ seq = _curses.tigetstr("setaf").decode("ascii") if seq is not None: # XXX may fail - need better logic seq = seq.replace("%p1", "") % color return seq self["NORMAL"] = _curses.tigetstr("sgr0").decode("ascii") self["BOLD"] = _curses.tigetstr("bold").decode("ascii") erase = _curses.tigetstr("el1").decode("ascii") if erase is not None: self["ERASE"] = erase + _curses.tigetstr("cr").decode("ascii") self["RED"] = make_color(_curses.COLOR_RED) self["YELLOW"] = make_color(_curses.COLOR_YELLOW) self["GREEN"] = make_color(_curses.COLOR_GREEN)
def __init__(self, width=None): "Constructor." self._progress = None self._lines = 0 self._PADDING = 7 curses.setupterm() fgColorSeq = curses.tigetstr('setaf') or curses.tigetstr('setf') or '' colorIndex = getattr(curses, 'COLOR_WHITE') self._fgcolour = curses.tparm(fgColorSeq, colorIndex) bgColorSeq = curses.tigetstr('setab') or curses.tigetstr('setb') or '' colorIndex = getattr(curses, 'COLOR_BLACK') self._bgcolour = curses.tparm(bgColorSeq, colorIndex) self._control = {} for control in _CONTROLS: # Set the control escape sequence self._control[control] = curses.tigetstr(_CONTROLS[control]) or '' self._value = {} for value in _VALUES: # Set terminal related values self._value[value] = curses.tigetnum(_VALUES[value]) if width and width < self._value["COLUMNS"] - self._PADDING: self._width = width else: self._width = self._value["COLUMNS"] - self._PADDING
def __init__(self, colors=DEFAULT_COLORS): enable = False if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): try: curses.setupterm() if curses.tigetnum("colors") > 0: enable = True except Exception: pass self._colors = {} if enable: fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): fg_color = unicode_type(fg_color, "ascii") for name, code in colors.items(): self._colors[name] = unicode_type(curses.tparm(fg_color, code), "ascii") self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii") else: self._normal = '' self.enable = enable
def colourise_output(start, line, end="reset"): """ Wrapper method to return colourised output when running in a terminal. :param start: One of 'red', 'green', 'yellow' or 'cyan' :type start: str. :param line: Text to be colourised. :type line: str. :param end: Typically, 'reset' to clear the colourisation. :type end: str. :returns: str. """ # Simple coloured output if not is_term(): return line if colourise_output.init == 0: curses.setupterm() colourise_output.init = 1 colours = { "reset": curses.tparm(curses.tigetstr("op")), "red": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_RED), "green": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_GREEN), "yellow": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_YELLOW), "cyan": curses.tparm(curses.tigetstr("setaf"), curses.COLOR_CYAN), } return colours[start] + line + colours[end]
def render(char_map, characters, color_map=None): """ Takes ascii & color data, and display it on the screen """ import curses curses.setupterm() fg_normal = curses.tigetstr('sgr0') fg_colors = [curses.tparm(curses.tigetstr('setaf'), i) for i in range(8)] attr_bold = curses.tparm(curses.tigetstr('bold'), curses.A_BOLD) attr_normal = curses.tparm(curses.tigetstr('sgr0'), curses.A_NORMAL) def set_color(fg=None): if fg is None: return fg_normal + attr_normal if fg in range(0, 8): return fg_colors[fg] if fg in range(8, 16): return fg_colors[fg-8] + attr_bold return '' for y in range(len(char_map)): for x in range(len(char_map[y])): if color_map is not None: print(set_color(color_map[y][x]), end='') print(characters[char_map[y][x]], end='') print('') if color_map is not None: print(set_color(), end='')
def __init__(self, console, *args, **kwargs): #logging.Formatter.__init__(self, *args, **kwargs) logging.Formatter.__init__(self, "%(asctime)s - %(name)s - %(levelname)s - %(message)s") color = False if console: if curses and sys.stderr.isatty(): try: curses.setupterm() if curses.tigetnum("colors") > 0: color = True except Exception: pass self._color = color if color: # The curses module has some str/bytes confusion in python3. # Most methods return bytes, but only accept strings. # The explict calls to unicode() below are harmless in python2, # but will do the right conversion in python3. fg_color = unicode(curses.tigetstr("setaf") or curses.tigetstr("setf") or "", "ascii") self._colors = { logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue "ascii"), logging.INFO: unicode(curses.tparm(fg_color, 2), # Green "ascii"), logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow "ascii"), logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red "ascii"), } self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def setup(cls): if getattr(cls, '_setup', False): return curses.setupterm() cls.BOL = curses.tigetstr('cr') cls.CLEAR_EOL = curses.tigetstr('el') cls._setup = True
def terminal_has_colors(): if sys.platform=='cygwin' and 'USE_COLOR' not in os.environ: # Avoid importing curses that causes illegal operation # with a message: # PYTHON2 caused an invalid page fault in # module CYGNURSES7.DLL as 015f:18bbfc28 # Details: Python 2.3.3 [GCC 3.3.1 (cygming special)] # ssh to Win32 machine from debian # curses.version is 2.2 # CYGWIN_98-4.10, release 1.5.7(0.109/3/2)) return 0 if hasattr(sys.stdout,'isatty') and sys.stdout.isatty(): try: import curses curses.setupterm() if (curses.tigetnum("colors") >= 0 and curses.tigetnum("pairs") >= 0 and ((curses.tigetstr("setf") is not None and curses.tigetstr("setb") is not None) or (curses.tigetstr("setaf") is not None and curses.tigetstr("setab") is not None) or curses.tigetstr("scp") is not None)): return 1 except Exception: pass return 0
def __init__(self, color, *args, **kwargs): logging.Formatter.__init__(self, *args, **kwargs) self._color = color if color: fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): fg_color = six.text_type(fg_color, "ascii") self._colors = { logging.DEBUG: six.text_type( curses.tparm(fg_color, 4), "ascii" ), # Blue logging.INFO: six.text_type( curses.tparm(fg_color, 2), "ascii" ), # Green logging.WARNING: six.text_type( curses.tparm(fg_color, 3), "ascii" ), # Yellow logging.ERROR: six.text_type( curses.tparm(fg_color, 1), "ascii" ), # Red } self._normal = six.text_type(curses.tigetstr("sgr0"), "ascii")
display = Display() try: import curses # Nest the try except since curses.error is not available if curses did not import try: curses.setupterm() HAS_CURSES = True except curses.error: HAS_CURSES = False except ImportError: HAS_CURSES = False if HAS_CURSES: MOVE_TO_BOL = curses.tigetstr('cr') CLEAR_TO_EOL = curses.tigetstr('el') else: MOVE_TO_BOL = b'\r' CLEAR_TO_EOL = b'\x1b[K' def clear_line(stdout): stdout.write(b'\x1b[%s' % MOVE_TO_BOL) stdout.write(b'\x1b[%s' % CLEAR_TO_EOL) class InnerFailure(Exception): def __init__(self, result): self.result = result super().__init__()
def main(argv): if len(argv) < 2 or len(argv) > 3: print("Usage: droidbox.py filename.apk output <duration in seconds>") sys.exit(1) duration = 0 #Duration given? if len(argv) == 3: duration = int(argv[2]) apkName = sys.argv[1] #APK existing? if os.path.isfile(apkName) == False: print("File %s not found" % argv[1]) sys.exit(1) application = Application(apkName) ret = application.processAPK() #Error during the APK processing? if (ret == 0): print("Failed to analyze the APK. Terminate the analysis.") sys.exit(1) activities = application.getActivities() mainActivity = application.getMainActivity() packageName = application.getPackage() recvsaction = application.getRecvsaction() enfperm = application.getEnfperm() #Get the hashes hashes = application.getHashes() curses.setupterm() sys.stdout.write(curses.tigetstr("clear")) sys.stdout.flush() call(['adb', 'logcat', '-c']) print " ____ __ ____" print "/\ _`\ __ /\ \/\ _`\\" print "\ \ \/\ \ _ __ ___ /\_\ \_\ \ \ \L\ \ ___ __ _" print " \ \ \ \ \/\`'__\ __`\/\ \ /'_` \ \ _ <' / __`\/\ \/'\\" print " \ \ \_\ \ \ \/\ \L\ \ \ \/\ \L\ \ \ \L\ \\ \L\ \/> </" print " \ \____/\ \_\ \____/\ \_\ \___,_\ \____/ \____//\_/\_\\" print " \/___/ \/_/\/___/ \/_/\/__,_ /\/___/ \/___/ \//\/_/" #No Main acitvity found? Return an error if mainActivity == None: print("No activity to start. Terminate the analysis.") sys.exit(1) #No packages identified? Return an error if packageName == None: print("No package found. Terminate the analysis.") sys.exit(1) #Execute the application ret = call(['monkeyrunner', 'monkeyrunner.py', apkName, packageName, mainActivity], stderr=PIPE, cwd=os.path.dirname(os.path.realpath(__file__))) if (ret == 1): print("Failed to execute the application.") sys.exit(1) print("Starting the activity %s..." % mainActivity) #By default the application has not started applicationStarted = 0 stringApplicationStarted = "Start proc %s" % packageName #Open the adb logcat adb = Popen(["adb", "logcat", "DroidBox:W", "dalvikvm:W", "ActivityManager:I"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) #Wait for the application to start while 1: try: logcatInput = adb.stdout.readline() if not logcatInput: raise Exception("We have lost the connection with ADB.") #Application started? if (stringApplicationStarted in logcatInput): applicationStarted = 1 break; except: break if (applicationStarted == 0): print("Analysis has not been done.") #Kill ADB, otherwise it will never terminate os.kill(adb.pid, signal.SIGTERM) sys.exit(1) print("Application started") print("Analyzing the application during %s seconds..." % (duration if (duration !=0) else "infinite time")) count = CountingThread() count.start() timeStamp = time.time() if duration: signal.signal(signal.SIGALRM, interruptHandler) signal.alarm(duration) #Collect DroidBox logs lastScreenshot = 0 while 1: if (time.time() - lastScreenshot) >=3: #Take Screenshots every 3 seconds. os.system("adb shell screencap -p | sed 's/\r$//' > /samples/out/screen_$(date +%Y-%m-%d_%H%M%S).png") lastScreenshot = time.time() try: logcatInput = adb.stdout.readline() if not logcatInput: raise Exception("We have lost the connection with ADB.") boxlog = logcatInput.split('DroidBox:') if len(boxlog) > 1: try: load = json.loads(decode(boxlog[1])) # DexClassLoader if load.has_key('DexClassLoader'): load['DexClassLoader']['type'] = 'dexload' dexclass[time.time() - timeStamp] = load['DexClassLoader'] count.increaseCount() # service started if load.has_key('ServiceStart'): load['ServiceStart']['type'] = 'service' servicestart[time.time() - timeStamp] = load['ServiceStart'] count.increaseCount() # received data from net if load.has_key('RecvNet'): host = load['RecvNet']['srchost'] port = load['RecvNet']['srcport'] recvnet[time.time() - timeStamp] = recvdata = {'type': 'net read', 'host': host, 'port': port, 'data': load['RecvNet']['data']} count.increaseCount() # fdaccess if load.has_key('FdAccess'): accessedfiles[load['FdAccess']['id']] = hexToStr(load['FdAccess']['path']) # file read or write if load.has_key('FileRW'): load['FileRW']['path'] = accessedfiles[load['FileRW']['id']] if load['FileRW']['operation'] == 'write': load['FileRW']['type'] = 'file write' else: load['FileRW']['type'] = 'file read' fdaccess[time.time()-timeStamp] = load['FileRW'] count.increaseCount() # opened network connection log if load.has_key('OpenNet'): opennet[time.time()-timeStamp] = load['OpenNet'] count.increaseCount() # closed socket if load.has_key('CloseNet'): closenet[time.time()-timeStamp] = load['CloseNet'] count.increaseCount() # outgoing network activity log if load.has_key('SendNet'): load['SendNet']['type'] = 'net write' sendnet[time.time()-timeStamp] = load['SendNet'] count.increaseCount() # data leak log if load.has_key('DataLeak'): my_time = time.time()-timeStamp load['DataLeak']['type'] = 'leak' load['DataLeak']['tag'] = getTags(int(load['DataLeak']['tag'], 16)) dataleaks[my_time] = load['DataLeak'] count.increaseCount() if load['DataLeak']['sink'] == 'Network': load['DataLeak']['type'] = 'net write' sendnet[my_time] = load['DataLeak'] count.increaseCount() elif load['DataLeak']['sink'] == 'File': load['DataLeak']['path'] = accessedfiles[load['DataLeak']['id']] if load['DataLeak']['operation'] == 'write': load['DataLeak']['type'] = 'file write' else: load['DataLeak']['type'] = 'file read' fdaccess[my_time] = load['DataLeak'] count.increaseCount() elif load['DataLeak']['sink'] == 'SMS': load['DataLeak']['type'] = 'sms' sendsms[my_time] = load['DataLeak'] count.increaseCount() # sent sms log if load.has_key('SendSMS'): load['SendSMS']['type'] = 'sms' sendsms[time.time()-timeStamp] = load['SendSMS'] count.increaseCount() # phone call log if load.has_key('PhoneCall'): load['PhoneCall']['type'] = 'call' phonecalls[time.time()-timeStamp] = load['PhoneCall'] count.increaseCount() # crypto api usage log if load.has_key('CryptoUsage'): load['CryptoUsage']['type'] = 'crypto' cryptousage[time.time()-timeStamp] = load['CryptoUsage'] count.increaseCount() except ValueError: pass except: try: count.stopCounting() count.join() finally: break; #Kill ADB, otherwise it will never terminate os.kill(adb.pid, signal.SIGTERM) #Done? Store the objects in a dictionary, transform it in a JSON object and return it output = dict() #Sort the items by their key output["dexclass"] = dexclass output["servicestart"] = servicestart output["recvnet"] = recvnet output["opennet"] = opennet output["sendnet"] = sendnet output["closenet"] = closenet output["accessedfiles"] = accessedfiles output["dataleaks"] = dataleaks output["fdaccess"] = fdaccess output["sendsms"] = sendsms output["phonecalls"] = phonecalls output["cryptousage"] = cryptousage output["recvsaction"] = recvsaction output["enfperm"] = enfperm output["hashes"] = hashes output["apkName"] = apkName with open("/tmp/analysis.json", "w", 0) as jsonfile: jsonfile.write(json.dumps(output,sort_keys=True, indent=4)) jsonfile.flush() os.fsync(jsonfile.fileno()) jsonfile.close() print(json.dumps(output)) sys.stdout.flush() sys.exit(0)
def main(username, sharer, wlist, num): """Create UDFs, folders and files for the given user using a wordlist.""" user = get_storage_user(None, username=utf2unicode(username)) sharer = get_storage_user(None, username=utf2unicode(sharer)) folders = [user.root] names = [utf2unicode(i.strip()) for i in file(wlist) if i.strip()] sample = random.sample(names, num) if sys.stdout.isatty(): import curses curses.setupterm() cols = curses.tigetnum('cols') - 1 progbar = makeStatBar(cols, cols - 2) home = curses.tigetstr('cr') def progress(l): """progress bar writer.""" sys.stdout.write(home + progbar((cols - 2) * (num - len(l)) / num)) sys.stdout.flush() else: progress = lambda l: None # UDF udf = user.make_udf(u'~/abundant-files') folders.append(user.volume(udf.id).get_node(udf.root_id)) # some UDFs for i in range(num / 100): progress(sample) folders.append(make_udf(user, sample)) for i in range(num / 4): progress(sample) name = sample.pop() folders.append(random.choice(folders).make_subdirectory(name)) sh_folders = [sharer.root] for i in range(num / 10): progress(sample) sh_folders.append(make_udf(sharer, sample)) for i in range(num / 10): progress(sample) name = sample.pop() sh_folders.append(random.choice(sh_folders).make_subdirectory(name)) for i in range(num / 20): progress(sample) name = sample.pop() filename = u'shared by ' + sharer.username readonly = random.choice((False, True)) if readonly: name += ' (ro)' filename += ' (ro)' folder = random.choice(sh_folders).make_subdirectory(name) folder.make_file(filename) share = folder.share(user.id, folder.name, readonly) user.get_share(share.id).accept() for i in random.sample(folders, len(folders) / 4): progress(sample) name = sample.pop() filename = u'shared by ' + user.username readonly = random.choice((False, True)) if readonly: name += ' (ro)' filename += ' (ro)' folder = random.choice(folders).make_subdirectory(name) folder.make_file(filename) share = folder.share(sharer.id, folder.name, readonly) sharer.get_share(share.id).accept() for i in range(num / 20): progress(sample) name = sample.pop() random.choice(folders).make_file(name) fake_hash = get_fake_hash() while sample: progress(sample) name = sample.pop() random.choice(folders).make_file_with_content(name, fake_hash, 12345, 100, 10000, uuid.uuid4(), u'image/tiff') if sys.stdout.isatty(): sys.stdout.write(home + curses.tigetstr('el'))
def test_posix_printengine(output_file, ttymode): """Test driver for POSIX print engine. This is maintained as a standalone function in order to support the 'runprintengine' test utility in $SRC/tests/interactive/runprintengine.py; it is also called by the test suite.""" pe = POSIXPrintEngine(output_file, ttymode=ttymode) standout = "" sgr0 = "" if ttymode: # We assume that setupterm() has been called already. standout = curses.tigetstr("smso") or "" sgr0 = curses.tigetstr("sgr0") or "" pe.cprint("Testing POSIX print engine; ttymode is {0}\n".format(ttymode)) # If we're not in ttymode, then the testing is simple. if not ttymode: pe.cprint("testing 1 2 3") pe.cprint("testing flush (2)") pe.flush() return # We assume setupterm() has been called. standout = curses.tigetstr("smso") or "" sgr0 = curses.tigetstr("sgr0") or "" pe.cprint("Now we'll print something and then erase it;") pe.cprint("you should see a blank line below this line.") pe.cprint("IF YOU CAN SEE THIS, THE TEST HAS FAILED", end='') pe.cprint("", erase=True) pe.cprint("You should see an X swishing back and forth; from") pe.cprint("left to right it should be inverse.") # Unused variable 'y'; pylint: disable=W0612 for y in range(0, 2): for x in xrange(0, 30, 1): pe.cprint(" " * x, erase=True, end='') pe.putp(standout) pe.cprint("X", end='') pe.putp(sgr0) time.sleep(0.050) for x in xrange(30, -1, -1): pe.cprint(" " * x + "X", erase=True, end='') time.sleep(0.050) pe.cprint("", erase=True) pe.cprint("testing 1 2 3") pe.cprint("testing XX XX XX", end="") time.sleep(0.500) pe.cprint("testing 4 5 6\ntesting XX XX XX", erase=True, end="") time.sleep(0.500) pe.cprint("testing YY YY", end="", erase=True) time.sleep(0.500) pe.cprint("testing 7 8 9\ntesting 10 11 12", erase=True) time.sleep(0.500) pe.cprint("testing ZZ ZZ ZZ ZZ ZZ", end="") time.sleep(0.500) pe.cprint("testing 13 14 15", erase=True) pe.cprint("testing flush...", end='') pe.flush() pe.cprint("This should be on the next line.") pe.cprint("testing flush (2)") pe.flush() pe.cprint("This should be on the next line (with no nl's intervening).") # just test that it works pe.isslow()
def get_tparm(key): val = None key = curses.tigetstr("sgr0") if key: val = curses.tparm(key) if val: val = val.decode('utf-8') return val
def end_line(self): return self.reset + curses.tigetstr("el")
def reset(self): return curses.tigetstr('sgr0')
def foreground(color): return curses.tparm(curses.tigetstr('setaf'), color)
def background(color): return curses.tparm(curses.tigetstr('setab'), color)
def _tigetstr(self, cap_name): import curses cap = curses.tigetstr(cap_name) or '' return re.sub('\\$<\\d+>[/*]?', '', cap)
def move_cur(to_y, to_x): tparm = curses.tparm(curses.tigetstr("cup"), to_y, to_x) # on python2 stdout is already in binary mode, in python3 is accessed via buffer bin_stdout = getattr(sys.stdout, 'buffer', sys.stdout) bin_stdout.write(tparm)
def _terminal_restore_color(): restore_code = curses.tigetstr('sgr0') if restore_code: sys.stdout.write(_unicode(restore_code))
def unicode_parm(cap, *parms): """Return the result of ``tparm(tigetstr())`` except as Unicode.""" return tparm(tigetstr(cap), *parms).decode('utf-8')
def cmd(self, name, *args): s = curses.tigetstr(name) sys.stdout.buffer.write(curses.tparm(s, *args))
def unicode_cap(cap): """Return the result of ``tigetstr`` except as Unicode.""" return tigetstr(cap).decode('utf-8')
def clear(self): sys.stdout.write( str(curses.tigetstr('cr')) + str(curses.tigetstr('el')))
def _tigetstr(self, cap_name): import curses cap = curses.tigetstr(cap_name) or b'' return re.sub(r'\$<\d+>[/*]?', '', cap.decode()).encode()
if sys.stdout and sys.stdout.isatty(): try: import curses curses.setupterm() except (ImportError, OSError) as ex: if __debug__: print('Warning', ex, sep=': ', end='\n\n', file=sys.stderr) curses = None else: curses = None if curses is None: TERMMODES = dict.fromkeys(map(itemgetter(0), TERMMODES), '') else: TERMMODES = { k: (curses.tigetstr(capname) or b'').decode('ascii') for k, capname in TERMMODES } def try_input(prompt=None, on_eof='', end='\n? '): """Similar to input() but return a default response on EOF or EBADF. If input() fails with EOFError or due to a bad (e. g. closed) standard output stream return 'on_eof' instead which defaults to the empty string. Additionally wrap the prompt string using termwrap (see below). 'end' is always appended to the prompt and defaults to '\n? '. """ if prompt:
def get_hex_chars(key_str): if (key_str.startswith("0x")): return key_str[2:].decode('hex') else: return curses.tigetstr(key_str)
import sys import os import re import libvirt import subprocess import logging from optparse import OptionParser from lxml import etree as ET # The expat parser fails on control characters from xml.parsers.expat import ExpatError try: import curses curses.setupterm() NORMAL = curses.tigetstr('sgr0').decode('ascii') SET_AF = curses.tigetstr('setaf') BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = [curses.tparm(SET_AF, i).decode('ascii') for i in range(8)] except (ImportError, curses.error): NORMAL = BLACK = RED = GREEN = YELLOW = BLUE = MAGENTA = CYAN = WHITE = '' class Dotter(object): """Handle dot graph.""" RE_KEY = re.compile('[^0-9A-Za-z]') def __init__(self, out=None): self.dot_out = out @staticmethod
def make_color(color): """ Make color control string """ seq = _curses.tigetstr(bc('setaf')) if seq is not None: seq = _curses.tparm(seq, color).decode('ascii') return seq
def module_funcs(stdscr): "Test module-level functions" for func in [curses.baudrate, curses.beep, curses.can_change_color, curses.cbreak, curses.def_prog_mode, curses.doupdate, curses.filter, curses.flash, curses.flushinp, curses.has_colors, curses.has_ic, curses.has_il, curses.isendwin, curses.killchar, curses.longname, curses.nocbreak, curses.noecho, curses.nonl, curses.noqiflush, curses.noraw, curses.reset_prog_mode, curses.termattrs, curses.termname, curses.erasechar, curses.getsyx]: func() # Functions that actually need arguments if curses.tigetstr("cnorm"): curses.curs_set(1) curses.delay_output(1) curses.echo() ; curses.echo(1) f = tempfile.TemporaryFile() stdscr.putwin(f) f.seek(0) curses.getwin(f) f.close() curses.halfdelay(1) curses.intrflush(1) curses.meta(1) curses.napms(100) curses.newpad(50,50) win = curses.newwin(5,5) win = curses.newwin(5,5, 1,1) curses.nl() ; curses.nl(1) curses.putp('abc') curses.qiflush() curses.raw() ; curses.raw(1) curses.setsyx(5,5) curses.tigetflag('hc') curses.tigetnum('co') curses.tigetstr('cr') curses.tparm('cr') curses.typeahead(sys.__stdin__.fileno()) curses.unctrl('a') curses.ungetch('a') curses.use_env(1) # Functions only available on a few platforms if curses.has_colors(): curses.start_color() curses.init_pair(2, 1,1) curses.color_content(1) curses.color_pair(2) curses.pair_content(curses.COLOR_PAIRS - 1) curses.pair_number(0) if hasattr(curses, 'use_default_colors'): curses.use_default_colors() if hasattr(curses, 'keyname'): curses.keyname(13) if hasattr(curses, 'has_key'): curses.has_key(13) if hasattr(curses, 'getmouse'): (availmask, oldmask) = curses.mousemask(curses.BUTTON1_PRESSED) # availmask indicates that mouse stuff not available. if availmask != 0: curses.mouseinterval(10) # just verify these don't cause errors m = curses.getmouse() curses.ungetmouse(*m) if hasattr(curses, 'is_term_resized'): curses.is_term_resized(*stdscr.getmaxyx()) if hasattr(curses, 'resizeterm'): curses.resizeterm(*stdscr.getmaxyx()) if hasattr(curses, 'resize_term'): curses.resize_term(*stdscr.getmaxyx())
def build_word2vec(self, f_name_training_sample, f_name_prepared_subtitles=None, f_name_enc_training_sample=None, f_name_w2v_model=None, f_name_w2v_vocab=None, len_encode=None, size=500, min_count_repeat=1, window=5, epochs=None, logging=False): ''' Предназначен для построения модели word2vec и кодирования обучающей выборки в вектора. 1. f_name_training_sample - имя входного .pkl файла с предварительно обработанными парами [вопрос,ответ] 2. f_name_prepared_subtitles - имя .pkl файла с предварительно обработанными субтитрами (для расширения словаря модели word2vec) 3. f_name_enc_training_sample - имя выходного .npz файла с векторным представлением слов в парах [вопрос,ответ] (по умолчанию encoded_+f_name_training_sample+.npz) 4. f_name_w2v_model - имя .bin файла для сохранения обученной модели word2vec (по умолчанию w2v_model_+f_name_training_sample+.bin) 5. f_name_w2v_vocab - имя .txt файла для сохранения словаря word2vec (по умолчанию w2v_vocabulary_+f_name_training_sample+.txt) 6. len_encode - если None: закодировать весь f_name_training_sample, иначе - первые len_encode элементов 7. size - рразмерность вектора, которым кодируется одно слово Обычно используют значения в несколько сотен, большие значения требуют больше данных обучения, но могут привести к более точным моделям. 8. min_count - минимальное количество повторений (частота повторений) слова, что бы оно было включено в словарь 9. window - максимальное расстояние между текущим и прогнозируемым словом в предложении 10. epochs - число эпох обучения модели word2vec (если None, используется внутренний итератор) 11. logging - включение вывода данных в процессе обучения модели word2vec Входные предложения должны иметь вид, например: [['<PAD>', ..., '<PAD>', '?', 'класс', 'этот', 'нужен', 'Зачем', '<GO>'], ['Для', 'кодирования', 'предложений', '<EOS>', '<PAD>', ..., '<PAD>']] ''' if f_name_enc_training_sample is None: f_name_enc_training_sample = f_name_training_sample.replace( 'prepared_', 'encoded_') f_name_enc_training_sample = f_name_enc_training_sample.replace( '.pkl', '.npz') if f_name_w2v_model is None: f_name_w2v_model = f_name_training_sample.replace( 'prepared_', 'w2v_model_') f_name_w2v_model = f_name_w2v_model.replace('.pkl', '.bin') if f_name_w2v_vocab is None: f_name_w2v_vocab = f_name_training_sample.replace( 'prepared_', 'w2v_vocabulary_') f_name_w2v_vocab = f_name_w2v_vocab.replace('.pkl', '.txt') if f_name_prepared_subtitles is not None: print('[i] Загрузка субтитров из %s' % f_name_prepared_subtitles) with open(f_name_prepared_subtitles, 'rb') as f_subtitles: subtitles = pickle.load(f_subtitles) print('\tколичество предложений: %i пар' % len(subtitles)) print('\tдлинна предложений: %i слов' % len(subtitles[0][0])) print('[i] Загрузка данных из %s' % f_name_training_sample) with open(f_name_training_sample, 'rb') as f_training_sample: training_sample = pickle.load(f_training_sample) print('\tколичество предложений: %i пар' % len(training_sample)) print('\tдлинна предложений: %i слов' % len(training_sample[0][0])) if f_name_prepared_subtitles is not None: print('[i] Объединение...') dataset = subtitles + training_sample else: dataset = training_sample print('[i] Подсчёт размера полного словаря...') i = 0 vocabulary = [] for pair in dataset: if i % 1000 == 0 or i == len(dataset) - 1: os.write(sys.stdout.fileno(), curses.tigetstr('cuu1')) print('[i] Подсчёт размера полного словаря... %i из %i' % (i, len(dataset))) for sentence in pair: for word in sentence: vocabulary.append(word) i += 1 os.write(sys.stdout.fileno(), curses.tigetstr('cuu1')) print('[i] Подсчёт размера полного словаря... %i из %i' % (len(dataset), len(dataset))) vocabulary = set(vocabulary) print('\tпредполагаемый размер полного словаря: %i слов' % len(vocabulary)) self.__w2v_fit(dataset, f_name_w2v_model, min_count_repeat, size, window, epochs, logging) dataset = training_sample vocabulary = [w for w in self.model.wv.vocab.keys()] print('[i] Размер словаря word2vec: %i слов' % len(vocabulary)) with open(f_name_w2v_vocab, 'w') as f_w2v_vocab: for word in vocabulary: print(word, file=f_w2v_vocab) # Оптимизация, что бы модель word2vec занимала в оперативной памяти меньше места (почему-то искажает декодировку векторов в слова) #self.model = self.model.wv #self.model.init_sims(replace=True) self.data_w2v_encode(dataset, f_name_enc_training_sample, len_encode)
def assessment_training_accuracy(self, f_name_enc_training_sample, f_name_wrong_answers=None): ''' Оценка точности обучения сети: подаёт на вход сети все вопросы из обучающей выборки и сравнивает полученный ответ сети с ответом из обучающей выборки. 1. f_name_enc_training_sample - имя .npz файла с векторным представлением слов в парах [вопрос,ответ] 2. f_name_wrong_answers - имя .txt файла для сохранения неправильных ответов сети (по умолчанию data/wrong_answers.txt) ''' if f_name_wrong_answers is None: f_name_wrong_answers = f_name_enc_training_sample.replace( 'encoded_', 'wrong_answers_') f_name_wrong_answers = f_name_wrong_answers.replace('.npz', '.txt') print('[i] Оценка точности обучения модели...') npzfile = np.load(f_name_enc_training_sample) questions, answers = npzfile['questions'], npzfile['answers'] questions = (questions + 1.0) * 0.5 correct_answers = 0 wrong_answers = [] len_questions = len(questions) print( '[i] Оценено 0 из %i, правильных ответов 0, текущая точность 0.00%%' % len_questions) for i in range(len_questions): if i % 10 == 0 and i > 0: os.write(sys.stdout.fileno(), curses.tigetstr('cuu1')) print( '[i] Оценено %i из %i, правильных ответов %i, текущая точность %.2f%%' % (i, len_questions, correct_answers, correct_answers / i * 100)) answer = self.model.predict(questions[i][np.newaxis, :]) answer = answer * 2.0 - 1.0 answer = self.w2v.vec2word(answer[0]) #answer = self.stp.prepare_answer(answer) answer_etalon = self.w2v.vec2word(answers[i]) #answer_etalon = self.stp.prepare_answer(answer_etalon) if answer == answer_etalon: #if np.all(answer[0] == answers[i]): correct_answers += 1 else: # Сохранение неправильных ответов для последующего вывода quest = self.w2v.vec2word(questions[i]) quest = list(reversed(quest)) quest = self.stp.prepare_answer(quest) #answer = self.w2v.vec2word(answer[0]) answer = self.stp.prepare_answer(answer) wrong_answers.append([quest, answer]) os.write(sys.stdout.fileno(), curses.tigetstr('cuu1')) print( '[i] Оценено %i из %i, правильных ответов %i, текущая точность %.2f%%' % (len_questions, len_questions, correct_answers, correct_answers / len_questions * 100)) accuracy = correct_answers / len(answers) * 100 print( '[i] Количество правильных ответов %i из %i, итоговая точность %.2f%%' % (correct_answers, len(answers), accuracy)) if len(wrong_answers) < 50: i = 0 print('[i] Неправильные ответы:') for phrase in wrong_answers: i += 1 print('%i. %s %%%% %s' % (i, phrase[0], phrase[1])) if accuracy > 75: with open(f_name_wrong_answers, 'w') as f_wrong_answers: for phrase in wrong_answers: f_wrong_answers.write(phrase[0] + ' %% ' + phrase[1] + '\n')
import curses import selectors import shutil import signal import sys import termios import tty from contextlib import contextmanager curses.setupterm() CSI = '\033[' # tigertstr uses \033O (SS3) instead of \033[ (CSI) as prefix # https://en.wikipedia.org/wiki/ANSI_escape_code KEY_BACKSPACE = curses.tigetstr('kbs').decode('ascii') KEY_ESC = '\x1b' KEY_HOME = CSI + 'H' KEY_END = CSI + 'F' KEY_PPAGE = CSI + '5~' KEY_NPAGE = CSI + '6~' KEY_UP = CSI + 'A' KEY_DOWN = CSI + 'B' KEY_RIGHT = CSI + 'C' KEY_LEFT = CSI + 'D' def isatty(): return os.isatty(sys.stdout.fileno())
def cap_string(s, *args): cap = curses.tigetstr(s) if cap: return curses.tparm(cap, *args).decode("utf-8") else: return ''
def clear_screen(): import curses, sys curses.setupterm() sys.stdout.write(curses.tigetstr("clear")) sys.stdout.flush()
def _tigetstr(self, cap_name): # String capabilities can include "delays" of the form "$<2>". # For any modern terminal, we should be able to just ignore # these, so strip them out. import curses return curses.tigetstr(cap_name) or ''
import sys import os if os.name == 'nt': n_cols = 80 set_fg = None normal = None else: import curses curses.setupterm() n_cols = curses.tigetnum('cols') or 80 set_fg = curses.tigetstr('setf') or None normal = curses.tigetstr('sgr0') or None cursor_pos = 0 COLOURS = { 'BLACK': 0, 'BLUE': 1, 'GREEN': 2, 'CYAN': 3, 'RED': 4, 'MAGENTA': 5, 'YELLOW': 6, 'WHITE': 7, } def checking(msg, indent=2): global cursor_pos
def _terminal_restore_color(): import curses, sys sys.stdout.write(curses.tigetstr('sgr0'))