def _init_term(self):
"""
Initialize term control codes.
@rtype: bool
@returns: True if term codes were successfully initialized,
False otherwise.
"""
term_type = os.environ.get("TERM", "").strip()
if not term_type:
return False
tigetstr = None
try:
import curses
try:
curses.setupterm(term_type, self.out.fileno())
tigetstr = curses.tigetstr
except curses.error:
pass
except ImportError:
pass
if tigetstr is None:
return False
term_codes = {}
for k, capname in self._termcap_name_map.items():
code = tigetstr(capname)
if code is None:
code = self._default_term_codes[capname]
term_codes[k] = code
object.__setattr__(self, "_term_codes", term_codes)
return True
def enable_pretty_logging():
"""Turns on formatted logging output as configured."""
root_logger = logging.getLogger()
if options.log_file_prefix:
channel = logging.handlers.RotatingFileHandler(
filename=options.log_file_prefix,
maxBytes=options.log_file_max_size,
backupCount=options.log_file_num_backups)
channel.setFormatter(_LogFormatter(color=False))
root_logger.addHandler(channel)
if (options.log_to_stderr or
(options.log_to_stderr is None and not root_logger.handlers)):
# Set up color if we are in a tty and curses is installed
color = False
if curses and sys.stderr.isatty():
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except:
pass
channel = logging.StreamHandler()
channel.setFormatter(_LogFormatter(color=color))
root_logger.addHandler(channel)
def _SetupColours():
"""Initializes the colour constants.
"""
# pylint: disable=W0603
# due to global usage
global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
# Don't use colours if stdout isn't a terminal
if not sys.stdout.isatty():
return
try:
import curses
except ImportError:
# Don't use colours if curses module can't be imported
return
curses.setupterm()
_RESET_SEQ = curses.tigetstr("op")
setaf = curses.tigetstr("setaf")
_INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
_WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
_ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
def setterm(cls):
if hasattr(cls, '_setterm_called'):
pass
else:
import curses
curses.setupterm()
cls._setterm_called = True
def __init__(self, stream):
import curses
self.haslogged = False;
self.tc = True;
self.stream=stream;
self.caps=None;
self.setf=None;
self.col = {};
self.c = TermCapHolder();
if not self.stream.isatty():
self.blankcaps();
self.tc = False;
return
try:
curses.setupterm(os.environ.get("TERM", "xterm"), self.stream.fileno());
except Exception, e:
self.blankcaps();
self.tc = False;
# if caps for some reason are not possible. Set them to blanks.
self.warning("Cannot get capabilities: " + str(e));
return;
def __init__(self, stream = None):
"""
Construct this formatter.
Provides colored output if the stream parameter is specified and is an acceptable TTY.
We print hardwired escape sequences, which will probably break in some circumstances;
for this unfortunate shortcoming, we apologize.
"""
# select and construct format string
import curses
format = None
if stream and hasattr(stream, "isatty") and stream.isatty():
curses.setupterm()
# FIXME do nice block formatting, increasing column sizes as necessary
if curses.tigetnum("colors") > 2:
format = \
"%s%%(asctime)s%s %%(message)s" % (
TTY_Formatter._NAME_COLOR,
TTY_Formatter._COLOR_END,
)
if format is None:
format = "%(name)s - %(levelname)s - %(message)s"
# initialize this formatter
logging.Formatter.__init__(self, format, TTY_Formatter._DATE_FORMAT)
def terminal_width(self):
"""Return the terminal width if possible, otherwise return 0.
"""
ncols = 0
try:
import curses
try:
curses.setupterm()
ncols = curses.tigetnum('cols')
except AttributeError:
# windows curses doesn't implement setupterm or tigetnum
# code below from
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/440694
from ctypes import windll, create_string_buffer
# stdin handle is -10
# stdout handle is -11
# stderr handle is -12
h = windll.kernel32.GetStdHandle(-12)
csbi = create_string_buffer(22)
res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
if res:
import struct
(bufx, bufy, curx, cury, wattr,
left, top, right, bottom, maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
ncols = right - left + 1
except curses.error:
pass
except (ImportError, TypeError):
pass
return ncols
def init(self):
# Disable the invalid description auto-filtering feature.
# This will not affect our own validation.
self.config.filter.show_invalid_results = True
# Always disable terminal formatting, as it won't work properly with colorized output
self.config.display.fit_to_screen = False
# Set the block size (aka, hexdump line size)
self.block = self.config.block
if not self.block:
self.block = self.DEFAULT_BLOCK_SIZE
# Build a list of files to hexdiff
self.hex_target_files = [x for x in iter(self.next_file, None)]
# Build the header format string
header_width = (self.block * 4) + 2
if self.terse:
file_count = 1
else:
file_count = len(self.hex_target_files)
self.HEADER_FORMAT = "OFFSET " + ("%%-%ds " % header_width) * file_count
# Build the header argument list
self.HEADER = [fp.name for fp in self.hex_target_files]
if self.terse and len(self.HEADER) > 1:
self.HEADER = self.HEADER[0]
# Set up the tty for colorization, if it is supported
if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() and platform.system() != 'Windows':
curses.setupterm()
self.colorize = self._colorize
else:
self.colorize = self._no_colorize
def report(self):
curses.setupterm()
cols = curses.tigetnum('cols')
result = {}
#result['Serious'] = 0
#result['High'] = 0
#result['Medium'] = 0
#result['Low'] = 0
print '-' * cols
print '| {0} | {1} | {2} |'.format('LEVEL'.ljust(7), 'TYPE'.ljust(20), 'DETAIL'.ljust(cols - 37))
print '-' * cols
rresult = []
acc = 1
for rule in self.results:
print '| {0} | {1} | {2} |'.format(rule.severity.ljust(7), rule.type.ljust(20), rule.xurl[:cols - 37].ljust(cols - 37))
rresult.append({'type':rule.type, 'severity':rule.severity, 'payload':rule.url, 'acc':acc, 'description':rule.description, 'suggestion':rule.suggestion, 'file':rule.file})
acc = acc + 1
try:
result[rule.severity] += 1
except:
result[rule.severity] = 1
print '-' * cols
for i in result:
print result[i], i
trender = {'site':self.urlroot, 'list':rresult}
env = jinja2.Environment(loader = jinja2.PackageLoader('Scanner', '.'))
template = env.get_template('report-template.html')
f = open(self.output, 'w')
#print trender
f.write(template.render(trender).encode('utf8'))
f.close()
print 'Output: ', os.path.realpath(self.output)
def initcurse(self):
# Check the terminal type. If we fail, then assume that the
# terminal has no capabilities.
try: curses.setupterm()
except: return
#self.colors = cursescolors()
self.cols = curses.tigetnum('cols')
self.lines = curses.tigetnum('lines')
self.wfilelines = self.lines-3
self.w = curses.initscr()
# fenetre hauteur largeur y x
self.wheader = curses.newwin( 1, self.cols, 0, 0)
self.wfile = curses.newwin(self.wfilelines, self.cols, 1, 0)
self.wfooter = curses.newwin( 1, self.cols, self.lines-2, 0)
self.wcmd = curses.newwin( 1, self.cols, self.lines-1, 0)
curses.noecho()
curses.cbreak()
curses.nonl() # permet d'activer la touche enter (code 13)
curses.curs_set(0)
curses.mousemask(0)
curses.mouseinterval(0)
self.w.keypad(1)
self.w.nodelay(0)
self.w.timeout(self.timeout)
try:
self.initcolors_1()
except:
self.initcolors_2()
def __init__(self, term_stream=sys.stdout):
"""
Create a `TerminalController` and initialize its attributes
with appropriate values for the current terminal.
`term_stream` is the stream that will be used for terminal
output; if this stream is not a tty, then the terminal is
assumed to be a dumb terminal (i.e., have no capabilities).
"""
# Curses isn't available on all platforms
try:
import curses
except:
return
# If the stream isn't a tty, then assume it has no capabilities.
if not term_stream.isatty():
return
# Check the terminal type. If we fail, then assume that the
# terminal has no capabilities.
try:
curses.setupterm()
except:
return
# Look up numeric capabilities.
self.COLS = curses.tigetnum('cols')
self.LINES = curses.tigetnum('lines')
# Look up string capabilities.
for capability in self._STRING_CAPABILITIES:
(attrib, cap_name) = capability.split('=')
setattr(self, attrib, self._tigetstr(cap_name) or '')
# Colors
set_fg = self._tigetstr('setf')
if set_fg:
for i, color in zip(range(len(self._COLORS)), self._COLORS):
setattr(self, color, curses.tparm(set_fg, i) or '')
set_fg_ansi = self._tigetstr('setaf')
if set_fg_ansi:
for i, color in zip(
range(len(self._ANSICOLORS)),
self._ANSICOLORS):
setattr(self, color, curses.tparm(set_fg_ansi, i) or '')
set_bg = self._tigetstr('setb')
if set_bg:
for i, color in zip(range(len(self._COLORS)), self._COLORS):
setattr(self, 'BG_' + color, curses.tparm(set_bg, i) or '')
set_bg_ansi = self._tigetstr('setab')
if set_bg_ansi:
for i, color in zip(
range(len(self._ANSICOLORS)),
self._ANSICOLORS):
setattr(
self,
'BG_' + color,
curses.tparm(
set_bg_ansi,
i) or '')
def __init__(self, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
self._coloured = COLOURED and _stderr_supports_color()
if self._coloured:
curses.setupterm()
# The curses module has some str/bytes confusion in
# python3. Until version 3.2.3, most methods return
# bytes, but only accept strings. In addition, we want to
# output these strings with the logging module, which
# works with unicode strings. The explicit calls to
# unicode() below are harmless in python2 but will do the
# right conversion in python 3.
fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or
"")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = unicode(fg_color, "ascii")
self._colors = {
# blues
logging.DEBUG: unicode(curses.tparm(fg_color, 4), "ascii"),
# green
logging.INFO: unicode(curses.tparm(fg_color, 2), "ascii"),
# yellow
logging.WARNING: unicode(curses.tparm(fg_color, 3), "ascii"),
# red
logging.ERROR: unicode(curses.tparm(fg_color, 1), "ascii")
}
self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def setup(cls):
if getattr(cls, "_setup", False):
return
curses.setupterm()
cls.BOL = curses.tigetstr("cr")
cls.CLEAR_EOL = curses.tigetstr("el")
cls._setup = True
def _InitColorize(output_file):
"""Initializes the colorization directives via --colorize as a mapping
between compiled regular expressions meant to be matched against log
filename and target curses color escape codes. The curses escape codes
are represented as a pair containing codes for prepending and appending.
Returns: [ColorDirective], or None if
colorization is disabled or unavailable on the current terminal.
Based on tornado's code to colorize log output.
"""
color = False
# Set up color if we are in a tty, curses is available, and --colorize
# was specified.
if options.options.colorize is not None and curses and \
(options.options.color == 'yes' or \
(options.options.color == 'auto' and output_file.isatty())):
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except Exception:
pass
if not color:
return []
directives = []
normal = unicode(curses.tigetstr("sgr0"), "ascii")
fg_color = unicode(curses.tigetstr("setaf") or curses.tigetstr("setf") or "", "ascii")
for directive in options.options.colorize:
regexp, color_index = directive.split('=')
color = unicode(curses.tparm(fg_color, int(color_index)), "ascii")
directives.append(ColorDirective(re.compile(regexp), color, normal))
return directives
def enable_pretty_logging(options=options):
"""Turns on formatted logging output as configured.
This is called automatically by `parse_command_line`.
"""
root_logger = logging.getLogger()
if options.log_file_prefix:
channel = logging.FileHandler(options.log_file_prefix, 'w') # not append
channel.setFormatter(_LogFormatter(color=False))
root_logger.addHandler(channel)
if (options.log_to_stderr or
(options.log_to_stderr is None and not root_logger.handlers)):
# Set up color if we are in a tty and curses is installed
color = False
if curses and sys.stderr.isatty():
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except Exception:
pass
channel = logging.StreamHandler()
channel.setFormatter(_LogFormatter(color=color))
root_logger.addHandler(channel)
def hascolor(self): # Based on Python cookbook, #475186
try:
if self.handle.isatty():
curses.setupterm()
return curses.tigetnum('colors') > 2
except Exception, e:
pass
def enable_pretty_logging(level=logging.DEBUG, handler=None, color=None):
'''
handler: specify a handler instead of default StreamHandler
color: boolean, force color to be on / off. Default to be on only when
``handler`` isn't specified and the term supports color
'''
logger = logging.getLogger()
if handler is None:
h = logging.StreamHandler()
else:
h = handler
if color is None:
color = False
if handler is None and sys.stderr.isatty():
try:
import curses
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except:
import traceback
traceback.print_exc()
formatter = TornadoLogFormatter(color=color)
h.setLevel(level)
h.setFormatter(formatter)
logger.setLevel(level)
logger.addHandler(h)
def __init__(self, *args):
super(PosixTerminal, self).__init__(*args)
try:
self._tty = os.open('/dev/tty', os.O_RDWR)
curses.setupterm()
except OSError:
self._tty = None
def setup(level=INFO, color=False):
"""
Setup Pika logging, useful for console debugging and logging pika info,
warning, and error messages.
Parameters:
- level: Logging level. One of DEBUG, ERROR, INFO, WARNING.
Default: INFO
- color: Use colorized output. Default: False
"""
global curses
if curses and not color:
curses = None
logging.basicConfig(level=level)
logging.getLogger("pika").setLevel(level)
if color and curses and sys.stderr.isatty():
# Setup Curses
curses.setupterm()
# If we have color support
if curses.tigetnum("colors") > 0:
# Assign a FormatOutput Formatter to a StreamHandler instance
for handler in logging.getLogger("").handlers:
if isinstance(handler, logging.StreamHandler):
handler.setFormatter(FormatOutput())
break
def __init__(self, buffer=sys.stdout, no_color=False):
if not curses:
no_color = True
if not no_color:
no_color = isinstance(sys.stdout, StringIO) or \
not sys.stdout.isatty()
self.no_color = no_color
# Don't bother initiating color if there's no color.
if not no_color:
# Get curses all ready to write some stuff to the screen.
curses.setupterm()
# Initialize a store for the colors and pre-populate it
# with the un-color color.
self.colors = {'NORMAL': curses.tigetstr('sgr0') or ''}
# Determines capabilities of the terminal.
fgColorSeq = curses.tigetstr('setaf') or \
curses.tigetstr('setf') or ''
# Go through each color and figure out what the sequences
# are for each, then store the sequences in the store we
# made above.
for color in COLORS:
colorIndex = getattr(curses, 'COLOR_%s' % color)
self.colors[color] = curses.tparm(fgColorSeq,
colorIndex)
self.buffer = buffer
def hascolor(self): # Based on Python cookbook, #475186
try:
if self.handle.isatty():
curses.setupterm(fd=self.handle.fileno())
return curses.tigetnum("colors") > 2
except Exception as e:
pass
def get_term_size():
"""
Get the number of lines and columns of the tty that is connected to
stdout. Returns a tuple of (lines, columns) or (-1, -1) if an error
occurs. The curses module is used if available, otherwise the output of
`stty size` is parsed.
"""
if not sys.stdout.isatty():
return -1, -1
try:
import curses
try:
curses.setupterm()
return curses.tigetnum('lines'), curses.tigetnum('cols')
except curses.error:
pass
except ImportError:
pass
st, out = portage.subprocess_getstatusoutput('stty size')
if st == os.EX_OK:
out = out.split()
if len(out) == 2:
try:
return int(out[0]), int(out[1])
except ValueError:
pass
return -1, -1
def tigetnum(attr, default=None):
"""Return a value from the terminfo database.
Terminfo is used on Unix-like systems to report various terminal attributes
(such as width, height or the number of supported colors).
Returns ``default`` when the ``curses`` module is not available, or when
sys.stdout is not a terminal.
"""
try:
import curses
except ImportError:
# avoid reimporting a broken module in python 2.3
sys.modules['curses'] = None
else:
# If sys.stdout is not a real file object (e.g. in unit tests that
# use various wrappers), you get an error, different depending on
# Python version:
if sys.version_info >= (3,):
import io
expected_exceptions = (curses.error, TypeError, io.UnsupportedOperation)
else:
expected_exceptions = (curses.error, TypeError)
try:
curses.setupterm()
except expected_exceptions:
# You get curses.error when $TERM is set to an unknown name
pass
else:
return curses.tigetnum(attr)
return default
def initialize(self):
"""initialize curses, then call setup (at the first time) and resize."""
self.win.leaveok(0)
self.win.keypad(1)
self.load_mode = False
curses.cbreak()
curses.noecho()
curses.halfdelay(20)
try:
curses.curs_set(int(bool(self.settings.show_cursor)))
except:
pass
curses.start_color()
curses.use_default_colors()
self.settings.signal_bind('setopt.mouse_enabled', _setup_mouse)
_setup_mouse(dict(value=self.settings.mouse_enabled))
if not self.is_set_up:
self.is_set_up = True
self.setup()
self.win.addstr("loading...")
self.win.refresh()
curses.setupterm()
self._draw_title = curses.tigetflag('hs') # has_status_line
self.update_size()
self.is_on = True
if self.settings.update_tmux_title:
sys.stdout.write("\033kranger\033\\")
sys.stdout.flush()
def setup_module(mod):
try:
import curses
curses.setupterm()
except:
py.test.skip("Cannot test this here")
def supported(cls, stream=sys.stdout):
"""
A class method that returns True if the current platform supports
coloring terminal output using this method. Returns False otherwise.
"""
if not stream.isatty():
return False # auto color only on TTYs
try:
import curses
except ImportError:
return False
else:
try:
try:
return curses.tigetnum('colors') > 2
except curses.error:
curses.setupterm()
return curses.tigetnum('colors') > 2
except Exception:
raise
# guess false in case of error
return False
def init():
global cache
if 'PWNLIB_NOTERM' not in os.environ:
curses.setupterm()
cache = {}
def init():
"""
Initialize attributes with appropriate values for the current terminal.
`_term_stream` is the stream that will be used for terminal
output; if this stream is not a tty, then the terminal is
assumed to be a dumb terminal (i.e., have no capabilities).
"""
_term_stream = sys.stdout
# Curses isn't available on all platforms
try:
import curses
except ImportError:
sys.stderr.write("INFO: no curses support: you won't see colors\n")
return
# If the stream isn't a tty, then assume it has no capabilities.
from . import config
if not _term_stream.isatty() and 'color-always' not in config.color.style:
return
# Check the terminal type. If we fail, then assume that the
# terminal has no capabilities.
try:
curses.setupterm()
except curses.error:
return
_lookup_caps()
def __init__(self):
ProgressTracker.__init__(self)
self.act_started = False
self.ind_started = False
self.last_print_time = 0
try:
import curses
if not os.isatty(sys.stdout.fileno()):
raise ProgressTrackerException()
curses.setupterm()
self.cr = curses.tigetstr("cr")
except KeyboardInterrupt:
raise
except:
if portable.ostype == "windows" and os.isatty(sys.stdout.fileno()):
self.cr = "\r"
else:
raise ProgressTrackerException()
self.dl_started = False
self.spinner = 0
self.spinner_chars = "/-\|"
self.curstrlen = 0
def init_colors(self, colors):
curses.setupterm()
initc = curses.tigetstr("initc")
for index, color in enumerate(colors):
red, green, blue = self.interpret_color(color)
command = curses.tparm(initc, self.palette_offset + index, red, green, blue)
print command,
def reinit(self, term_stream=None, color='auto'):
"""Reinitializes the :class:`Term`.
:param term_stream: the terminal stream that the
:class:`Term` should be initialized to use. If
*term_stream* is not given, :attr:`sys.stdout` is used.
:param color: when to colorize output. Valid values are
'always', 'auto', and 'never'. 'always' will use ANSI codes
to always colorize output, 'auto' will decide whether do
colorize depending on the terminal, and 'never' will never
colorize.
"""
self.__enabled = True
self.lines = 24
if color == 'always':
self.__forced_init()
return
# Output modes:
self.MODE = {
'bold' : '',
'blink' : '',
'dim' : '',
'reverse' : '',
'underline' : '',
'normal' : ''
}
# Colours
self.FG_COLOR = {
'black' : '',
'blue' : '',
'green' : '',
'cyan' : '',
'red' : '',
'magenta' : '',
'yellow' : '',
'white' : ''
}
self.BG_COLOR = {
'black' : '',
'blue' : '',
'green' : '',
'cyan' : '',
'red' : '',
'magenta' : '',
'yellow' : '',
'white' : ''
}
if color == 'never':
self.__enabled = False
return
assert color == 'auto'
# If the stream isn't a tty, then assume it has no capabilities.
if not term_stream:
term_stream = sys.stdout
if not term_stream.isatty():
self.__enabled = False
return
# Check the terminal type. If we fail, then assume that the
# terminal has no capabilities.
try:
curses.setupterm(fd=term_stream.fileno())
except Exception:
self.__enabled = False
return
self._ctigetstr = curses.tigetstr
self.lines = curses.tigetnum('lines')
# Look up string capabilities.
for cap_name in self.MODE:
mode = cap_name
if cap_name in self.__cap_names:
cap_name = self.__cap_names[cap_name]
self.MODE[mode] = self._tigetstr(cap_name)
# Colors
set_fg = self._tigetstr('setf').encode('utf-8')
if set_fg:
for (color, val) in self.__colors.items():
self.FG_COLOR[color] = curses.tparm(set_fg, val).decode() or ''
set_fg_ansi = self._tigetstr('setaf').encode('utf-8')
if set_fg_ansi:
for (color, val) in self.__ansi_colors.items():
fg_color = curses.tparm(set_fg_ansi, val).decode() or ''
self.FG_COLOR[color] = fg_color
set_bg = self._tigetstr('setb').encode('utf-8')
if set_bg:
for (color, val) in self.__colors.items():
self.BG_COLOR[color] = curses.tparm(set_bg, val).decode() or ''
set_bg_ansi = self._tigetstr('setab').encode('utf-8')
if set_bg_ansi:
for (color, val) in self.__ansi_colors.items():
bg_color = curses.tparm(set_bg_ansi, val).decode() or ''
self.BG_COLOR[color] = bg_color
CLIENT_PATH = os.path.normpath(os.path.join(os.getcwd(), __file__))
AI_PROCESS_PATH = os.path.join(os.path.dirname(CLIENT_PATH), 'dropblox_ai')
NUM_HTTP_RETRIES = 2 # number of times to retry if http connection fails
is_windows = platform.system() == "Windows"
# Printing utilities
# TODO(astaley): Consider vetting and using colorama module for windows support
# default to no colors
colorred = '{0}'
colorgrn = colorred
try:
import curses
curses.setupterm()
num_colors = curses.tigetnum('colors')
except Exception: # no term support (windows; piping to file; etc.)
pass
else:
if num_colors >= 8:
colorred = "\033[01;31m{0}\033[00m"
colorgrn = "\033[1;36m{0}\033[00m"
class Command(object):
def __init__(self, cmd, *args):
self.cmd = cmd
self.args = list(args)
def run(self, timeout):
def main(args=sys.argv):
"""
Parse command line arguments and view the log in the specified format.
"""
cli_command = ""
if sys.argv[0].endswith('gateone'):
cli_command = "termlog "
usage = '\t%prog {0}[options] <log file>'.format(cli_command)
parser = OptionParser(usage=usage, version=__version__)
parser.disable_interspersed_args()
parser.add_option("-f", "--flat",
dest="flat",
default=False,
action="store_true",
help="Display the log line-by-line in a syslog-like format."
)
parser.add_option("-p", "--playback",
dest="playback",
default=True,
action="store_false",
help=("Play back the log in a video-like fashion. This is the default "
"view.")
)
parser.add_option("--pretty",
dest="pretty",
default=True,
action="store_true",
help=("Preserve font and character renditions when displaying the log "
"in flat view (default).")
)
parser.add_option("--raw",
dest="raw",
default=False,
action="store_true",
help="Display control characters and escape sequences when viewing."
)
parser.add_option("--html",
dest="html",
default=False,
action="store_true",
help=(
"Render a given .golog as a self-contained HTML playback file "
"(to stdout).")
)
parser.add_option("--metadata",
dest="metadata",
default=False,
action="store_true",
help=( "Prints (to stdout) the metadata of the given .golog")
)
(options, args) = parser.parse_args(args=args)
if len(args) < 1:
print("ERROR: You must specify a log file to view.")
parser.print_help()
sys.exit(1)
log_path = args[0]
if not os.path.exists(log_path):
print("ERROR: %s does not exist" % log_path)
sys.exit(1)
sys_stdout = sys.stdout
if bytes != str: # Python 3
sys_stdout = sys.stdout.buffer
sys.stdout.flush() # Make sure it's empty before writing to the buffer
try:
if options.metadata:
import json
if curses and sys.stderr.isatty():
try:
curses.setupterm()
print(json.dumps(get_log_metadata(log_path), indent=4))
except Exception:
print(json.dumps(get_log_metadata(log_path)))
sys.exit(0)
elif options.flat:
flatten_log(
log_path,
sys_stdout,
preserve_renditions=options.pretty, show_esc=options.raw)
elif options.html:
result = render_html_playback(log_path)
print(result)
else:
playback_log(log_path, sys_stdout, show_esc=options.raw)
except (IOError, KeyboardInterrupt):
# Move the cursor to the bottom of the screen to ensure it isn't in the
# middle of the log playback output
rows, cols = get_terminal_size()
print("\x1b[%s;0H\n" % rows)
sys.exit(0)
def module_funcs(stdscr):
"Test module-level functions"
for func in [curses.baudrate, curses.beep, curses.can_change_color,
curses.cbreak, curses.def_prog_mode, curses.doupdate,
curses.filter, curses.flash, curses.flushinp,
curses.has_colors, curses.has_ic, curses.has_il,
curses.isendwin, curses.killchar, curses.longname,
curses.nocbreak, curses.noecho, curses.nonl,
curses.noqiflush, curses.noraw,
curses.reset_prog_mode, curses.termattrs,
curses.termname, curses.erasechar, curses.getsyx]:
func()
# Functions that actually need arguments
if curses.tigetstr("cnorm"):
curses.curs_set(1)
curses.delay_output(1)
curses.echo() ; curses.echo(1)
f = tempfile.TemporaryFile()
stdscr.putwin(f)
f.seek(0)
curses.getwin(f)
f.close()
curses.halfdelay(1)
curses.intrflush(1)
curses.meta(1)
curses.napms(100)
curses.newpad(50,50)
win = curses.newwin(5,5)
win = curses.newwin(5,5, 1,1)
curses.nl() ; curses.nl(1)
curses.putp('abc')
curses.qiflush()
curses.raw() ; curses.raw(1)
curses.setsyx(5,5)
curses.setupterm(fd=sys.__stdout__.fileno())
curses.tigetflag('hc')
curses.tigetnum('co')
curses.tigetstr('cr')
curses.tparm('cr')
curses.typeahead(sys.__stdin__.fileno())
curses.unctrl('a')
curses.ungetch('a')
curses.use_env(1)
# Functions only available on a few platforms
if curses.has_colors():
curses.start_color()
curses.init_pair(2, 1,1)
curses.color_content(1)
curses.color_pair(2)
curses.pair_content(curses.COLOR_PAIRS - 1)
curses.pair_number(0)
if hasattr(curses, 'use_default_colors'):
curses.use_default_colors()
if hasattr(curses, 'keyname'):
curses.keyname(13)
if hasattr(curses, 'has_key'):
curses.has_key(13)
if hasattr(curses, 'getmouse'):
curses.mousemask(curses.BUTTON1_PRESSED)
curses.mouseinterval(10)
def main():
curses.setupterm()
root_folder = '/home/vlad/172.16.10.10/NSNet_dataset/datasets/clean/'
folder_names_with_speech = [
'russian_speech', 'read_speech', 'french_data', 'german_speech',
'italian_speech', 'spanish_speech'
]
folder_name_for_balanced_speech = 'all_balanced_speech/'
root_folder, folder_names_with_speech, folder_name_for_balanced_speech = create_and_parse_args(
root_folder, folder_names_with_speech, folder_name_for_balanced_speech)
# Search .wav audio in specified folders
folder_info = {}
print("[i] Search .wav audio in '{}'...".format(root_folder))
for folder_name in folder_names_with_speech:
f_names_wavs = []
print("\n[i] Search .wav audio in '{}'... 0".format(folder_name))
for path, folder_names, f_names in os.walk(root_folder + folder_name):
for f_name in f_names:
if f_name.rfind('.wav') != -1:
if len(f_names_wavs) % 1000 == 0:
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print("[i] Search .wav audio in '{}'... {}".format(
folder_name, len(f_names_wavs)))
f_names_wavs.append(os.path.join(path, f_name))
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print("[i] Search .wav audio in '{}'... {}".format(
folder_name, len(f_names_wavs)))
f_names_wavs = [[f_name, os.path.getsize(f_name)]
for f_name in f_names_wavs]
total_size_gb = sum([f_size for f_name, f_size in f_names_wavs
]) / 1024 / 1024 / 1024
print('[i] Total size of found .wav audio {:.1f} Gb'.format(
total_size_gb))
folder_info[folder_name] = {
'number_files': len(f_names_wavs),
'wavs': f_names_wavs,
'size_gb': total_size_gb
}
# Trimming number of found .wav audio in each folder by the smallest folder of them
folder_name_with_min_size = min(
folder_info,
key=lambda folder_name: folder_info[folder_name]['size_gb'])
min_size_b = folder_info[folder_name_with_min_size][
'size_gb'] * 1024 * 1024 * 1024
print(
'\n[i] Trimming number of found .wav audio in each folder to {:.1f} Gb...'
.format(folder_info[folder_name_with_min_size]['size_gb']))
for folder_name in folder_info:
if folder_name != folder_name_with_min_size:
f_names_wavs = folder_info[folder_name]['wavs']
shuffle(f_names_wavs)
current_size = 0
for i, (f_name, f_size) in enumerate(f_names_wavs):
if current_size < min_size_b:
current_size += f_size
else:
del f_names_wavs[i:]
break
folder_info[folder_name]['number_files'] = len(f_names_wavs)
folder_info[folder_name]['wavs'] = f_names_wavs
folder_info[folder_name]['size_gb'] = sum(
[f_size
for f_name, f_size in f_names_wavs]) / 1024 / 1024 / 1024
print('[i] Folder information after trimming:')
for folder_name in folder_info:
print("\tfolder '{}': {} .wav files, {:.1f} Gb".format(
folder_name, folder_info[folder_name]['number_files'],
folder_info[folder_name]['size_gb']))
# Copying each selected/remaining .wav audio to a separate folder
if not os.path.exists(root_folder + folder_name_for_balanced_speech):
os.mkdir(root_folder + folder_name_for_balanced_speech)
else:
rmtree(root_folder + folder_name_for_balanced_speech,
ignore_errors=True)
os.mkdir(root_folder + folder_name_for_balanced_speech)
f_names_all_wavs = [
f_name_wav for f_name in folder_info
for f_name_wav, f_size_wav in folder_info[f_name]['wavs']
]
f_names_all_wavs = list(chain(f_names_all_wavs))
elapsed_time_per_file = 0.0
print("\n[i] Copying each remaining .wav audio to '{}'... 0 of {}".format(
folder_name_for_balanced_speech, len(f_names_all_wavs)))
for i, f_name_wav in enumerate(f_names_all_wavs):
if i % 10 == 0 or i == len(f_names_all_wavs) - 1:
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print(
"[i] Copying each remaining .wav audio to '{}'... {} of {}, {:.4f} s/file"
.format(folder_name_for_balanced_speech, i,
len(f_names_all_wavs), elapsed_time_per_file))
start_time = time.time()
f_name_wav_new = root_folder + folder_name_for_balanced_speech + f_name_wav[
f_name_wav.rfind('/') + 1:]
copyfile(f_name_wav, f_name_wav_new)
elapsed_time_per_file = time.time() - start_time
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print(
"[i] Copying each remaining .wav audio to '{}'... {} of {} "
.format(folder_name_for_balanced_speech, len(f_names_all_wavs),
len(f_names_all_wavs)))
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--process',
dest='procname',
help="A (sub)string to match against process names.")
parser.add_argument('-p',
'--pid',
dest='pid',
type=Validator.ValidateNonNegativeNumber,
help='Which pid to scan for.')
parser.add_argument('-d',
'--device',
dest='device',
help='Device serial to scan.')
parser.add_argument('-t',
'--timelimit',
dest='timelimit',
type=Validator.ValidateNonNegativeNumber,
help='How long to track memory in seconds.')
parser.add_argument('-f',
'--frequency',
dest='frequency',
default=0,
type=Validator.ValidateNonNegativeNumber,
help='How often to poll in seconds.')
parser.add_argument('-s',
'--diff-against-start',
dest='diff_against_start',
action='store_true',
help='Whether or not to always compare against the'
' original memory values for deltas.')
parser.add_argument('-b',
'--boring-output',
dest='dull_output',
action='store_true',
help='Whether or not to dull down the output.')
parser.add_argument('-k',
'--keep-results',
dest='no_overwrite',
action='store_true',
help='Keeps printing the results in a list instead of'
' overwriting the previous values.')
parser.add_argument('-g',
'--graph-file',
dest='graph_file',
type=Validator.ValidatePdfPath,
help='PDF file to save graph of memory stats to.')
parser.add_argument('-o',
'--text-file',
dest='text_file',
type=Validator.ValidatePath,
help='File to save memory tracking stats to.')
parser.add_argument('-m',
'--memory',
dest='show_mem',
action='store_true',
help='Whether or not to show memory stats. True by'
' default unless --n is specified.')
parser.add_argument('-n',
'--net',
dest='show_net',
action='store_true',
help='Whether or not to show network stats. False by'
' default.')
args = parser.parse_args()
# Add a basic filter to make sure we search for something.
if not args.procname and not args.pid:
args.procname = 'chrome'
# Make sure we show memory stats if nothing was specifically requested.
if not args.show_net and not args.show_mem:
args.show_mem = True
curses.setupterm()
printer = OutputBeautifier(not args.dull_output, not args.no_overwrite)
sys.stdout.write("Running... Hold CTRL-C to stop (or specify timeout).\n")
try:
last_time = time.time()
adb = None
old_snapshot = None
snapshots = []
while not args.timelimit or Timer.GetTimestamp() < float(
args.timelimit):
# Check if we need to track another device
device = DeviceHelper.GetDeviceToTrack(args.device)
if not device:
adb = None
elif not adb or device != str(adb):
#adb = adb_wrapper.AdbWrapper(device)
adb = device_utils.DeviceUtils(device)
old_snapshot = None
snapshots = []
try:
adb.EnableRoot()
except device_errors.CommandFailedError:
sys.stderr.write('Unable to run adb as root.\n')
sys.exit(1)
# Grab a snapshot if we have a device
snapshot = None
if adb:
pids = DeviceHelper.GetPidsToTrack(adb, args.pid,
args.procname)
snapshot = None
if pids:
snapshot = DeviceSnapshot(adb, pids, args.show_mem,
args.show_net)
if snapshot and snapshot.HasResults():
snapshots.append(snapshot)
printer.PrettyPrint(snapshot, old_snapshot, args.show_mem,
args.show_net)
# Transfer state for the next iteration and sleep
delay = max(1, args.frequency)
if snapshot:
delay = max(0, args.frequency - (time.time() - last_time))
time.sleep(delay)
last_time = time.time()
if not old_snapshot or not args.diff_against_start:
old_snapshot = snapshot
except KeyboardInterrupt:
pass
if args.graph_file:
printer.PrettyGraph(args.graph_file, snapshots)
if args.text_file:
printer.PrettyFile(args.text_file, snapshots, args.diff_against_start,
args.show_mem, args.show_net)
def test_terminal():
import readline, curses
curses.setupterm()
del readline
print('readline and curses OK!')
class Environment(object):
"""
Information about the execution context
(standard streams, config directory, etc).
By default, it represents the actual environment.
All of the attributes can be overwritten though, which
is used by the test suite to simulate various scenarios.
"""
is_windows = is_windows
config_dir = DEFAULT_CONFIG_DIR
stdin = sys.stdin
stdin_isatty = stdin.isatty()
stdin_encoding = None
stdout = sys.stdout
stdout_isatty = stdout.isatty()
stdout_encoding = None
stderr = sys.stderr
stderr_isatty = stderr.isatty()
colors = 256
if not is_windows:
if curses:
try:
curses.setupterm()
colors = curses.tigetnum('colors')
except curses.error:
pass
else:
# noinspection PyUnresolvedReferences
import colorama.initialise
stdout = colorama.initialise.wrap_stream(stdout,
convert=None,
strip=None,
autoreset=True,
wrap=True)
stderr = colorama.initialise.wrap_stream(stderr,
convert=None,
strip=None,
autoreset=True,
wrap=True)
del colorama
def __init__(self, **kwargs):
"""
Use keyword arguments to overwrite
any of the class attributes for this instance.
"""
assert all(hasattr(type(self), attr) for attr in kwargs.keys())
self.__dict__.update(**kwargs)
# Keyword arguments > stream.encoding > default utf8
if self.stdin_encoding is None:
self.stdin_encoding = getattr(self.stdin, 'encoding',
None) or 'utf8'
if self.stdout_encoding is None:
actual_stdout = self.stdout
if is_windows:
# noinspection PyUnresolvedReferences
from colorama import AnsiToWin32
if isinstance(self.stdout, AnsiToWin32):
actual_stdout = self.stdout.wrapped
self.stdout_encoding = getattr(actual_stdout, 'encoding',
None) or 'utf8'
@property
def config(self):
if not hasattr(self, '_config'):
self._config = Config(directory=self.config_dir)
if self._config.is_new():
self._config.save()
else:
self._config.load()
return self._config
def __str__(self):
defaults = dict(type(self).__dict__)
actual = dict(defaults)
actual.update(self.__dict__)
actual['config'] = self.config
return repr_dict_nice(
dict((key, value) for key, value in actual.items()
if not key.startswith('_')))
def __repr__(self):
return '<{0} {1}>'.format(type(self).__name__, str(self))
def main(argv):
if len(argv) < 2 or len(argv) > 3:
print("Usage: droidbox.py filename.apk <duration in seconds>")
sys.exit(1)
duration = 0
#Duration given?
if len(argv) == 3:
duration = int(argv[2])
apkName = sys.argv[1]
#APK existing?
if os.path.isfile(apkName) == False:
print("File %s not found" % argv[1])
sys.exit(1)
application = Application(apkName)
ret = application.processAPK()
#Error during the APK processing?
if (ret == 0):
print("Failed to analyze the APK. Terminate the analysis.")
sys.exit(1)
activities = application.getActivities()
mainActivity = application.getMainActivity()
packageName = application.getPackage()
recvsaction = application.getRecvsaction()
enfperm = application.getEnfperm()
#Get the hashes
hashes = application.getHashes()
curses.setupterm()
sys.stdout.write(curses.tigetstr("clear"))
sys.stdout.flush()
call(['adb', 'logcat', '-c'])
print " ____ __ ____"
print "/\ _`\ __ /\ \/\ _`\\"
print "\ \ \/\ \ _ __ ___ /\_\ \_\ \ \ \L\ \ ___ __ _"
print " \ \ \ \ \/\`'__\ __`\/\ \ /'_` \ \ _ <' / __`\/\ \/'\\"
print " \ \ \_\ \ \ \/\ \L\ \ \ \/\ \L\ \ \ \L\ \\ \L\ \/> </"
print " \ \____/\ \_\ \____/\ \_\ \___,_\ \____/ \____//\_/\_\\"
print " \/___/ \/_/\/___/ \/_/\/__,_ /\/___/ \/___/ \//\/_/"
#No Main acitvity found? Return an error
if mainActivity == None:
print("No activity to start. Terminate the analysis.")
sys.exit(1)
#No packages identified? Return an error
if packageName == None:
print("No package found. Terminate the analysis.")
sys.exit(1)
#Execute the application
ret = call([
'monkeyrunner', 'monkeyrunner.py', apkName, packageName, mainActivity
],
stderr=PIPE,
cwd=os.path.dirname(os.path.realpath(__file__)))
if (ret == 1):
print("Failed to execute the application.")
sys.exit(1)
print("Starting the activity %s..." % mainActivity)
#By default the application has not started
applicationStarted = 0
stringApplicationStarted = "Start proc %s" % packageName
#Open the adb logcat
adb = Popen(
["adb", "logcat", "DroidBox:W", "dalvikvm:W", "ActivityManager:I"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
#Wait for the application to start
while 1:
try:
logcatInput = adb.stdout.readline()
if not logcatInput:
raise Exception("We have lost the connection with ADB.")
#Application started?
if (stringApplicationStarted in logcatInput):
applicationStarted = 1
break
except:
break
if (applicationStarted == 0):
print("Analysis has not been done.")
#Kill ADB, otherwise it will never terminate
os.kill(adb.pid, signal.SIGTERM)
sys.exit(1)
print("Application started")
print("Analyzing the application during %s seconds..." %
(duration if (duration != 0) else "infinite time"))
count = CountingThread()
count.start()
timeStamp = time.time()
if duration:
signal.signal(signal.SIGALRM, interruptHandler)
signal.alarm(duration)
#Collect DroidBox logs
while 1:
try:
logcatInput = adb.stdout.readline()
if not logcatInput:
raise Exception("We have lost the connection with ADB.")
boxlog = logcatInput.split('DroidBox:')
if len(boxlog) > 1:
try:
load = json.loads(decode(boxlog[1]))
# DexClassLoader
if load.has_key('DexClassLoader'):
load['DexClassLoader']['type'] = 'dexload'
dexclass[time.time() -
timeStamp] = load['DexClassLoader']
count.increaseCount()
# service started
if load.has_key('ServiceStart'):
load['ServiceStart']['type'] = 'service'
servicestart[time.time() -
timeStamp] = load['ServiceStart']
count.increaseCount()
# received data from net
if load.has_key('RecvNet'):
host = load['RecvNet']['srchost']
port = load['RecvNet']['srcport']
recvnet[time.time() - timeStamp] = recvdata = {
'type': 'net read',
'host': host,
'port': port,
'data': load['RecvNet']['data']
}
count.increaseCount()
# fdaccess
if load.has_key('FdAccess'):
accessedfiles[load['FdAccess']['id']] = hexToStr(
load['FdAccess']['path'])
# file read or write
if load.has_key('FileRW'):
load['FileRW']['path'] = accessedfiles[load['FileRW']
['id']]
if load['FileRW']['operation'] == 'write':
load['FileRW']['type'] = 'file write'
else:
load['FileRW']['type'] = 'file read'
fdaccess[time.time() - timeStamp] = load['FileRW']
count.increaseCount()
# opened network connection log
if load.has_key('OpenNet'):
opennet[time.time() - timeStamp] = load['OpenNet']
count.increaseCount()
# closed socket
if load.has_key('CloseNet'):
closenet[time.time() - timeStamp] = load['CloseNet']
count.increaseCount()
# outgoing network activity log
if load.has_key('SendNet'):
load['SendNet']['type'] = 'net write'
sendnet[time.time() - timeStamp] = load['SendNet']
count.increaseCount()
# data leak log
if load.has_key('DataLeak'):
my_time = time.time() - timeStamp
load['DataLeak']['type'] = 'leak'
load['DataLeak']['tag'] = getTags(
int(load['DataLeak']['tag'], 16))
dataleaks[my_time] = load['DataLeak']
count.increaseCount()
if load['DataLeak']['sink'] == 'Network':
load['DataLeak']['type'] = 'net write'
sendnet[my_time] = load['DataLeak']
count.increaseCount()
elif load['DataLeak']['sink'] == 'File':
load['DataLeak']['path'] = accessedfiles[
load['DataLeak']['id']]
if load['DataLeak']['operation'] == 'write':
load['DataLeak']['type'] = 'file write'
else:
load['DataLeak']['type'] = 'file read'
fdaccess[my_time] = load['DataLeak']
count.increaseCount()
elif load['DataLeak']['sink'] == 'SMS':
load['DataLeak']['type'] = 'sms'
sendsms[my_time] = load['DataLeak']
count.increaseCount()
# sent sms log
if load.has_key('SendSMS'):
load['SendSMS']['type'] = 'sms'
sendsms[time.time() - timeStamp] = load['SendSMS']
count.increaseCount()
# phone call log
if load.has_key('PhoneCall'):
load['PhoneCall']['type'] = 'call'
phonecalls[time.time() - timeStamp] = load['PhoneCall']
count.increaseCount()
# crypto api usage log
if load.has_key('CryptoUsage'):
load['CryptoUsage']['type'] = 'crypto'
cryptousage[time.time() -
timeStamp] = load['CryptoUsage']
count.increaseCount()
except ValueError:
pass
except:
try:
count.stopCounting()
count.join()
finally:
break
#Kill ADB, otherwise it will never terminate
os.kill(adb.pid, signal.SIGTERM)
#Done? Store the objects in a dictionary, transform it in a JSON object and return it
output = dict()
#Sort the items by their key
output["dexclass"] = dexclass
output["servicestart"] = servicestart
output["recvnet"] = recvnet
output["opennet"] = opennet
output["sendnet"] = sendnet
output["closenet"] = closenet
output["accessedfiles"] = accessedfiles
output["dataleaks"] = dataleaks
output["fdaccess"] = fdaccess
output["sendsms"] = sendsms
output["phonecalls"] = phonecalls
output["cryptousage"] = cryptousage
output["recvsaction"] = recvsaction
output["enfperm"] = enfperm
output["hashes"] = hashes
output["apkName"] = apkName
print(json.dumps(output))
sys.exit(0)
def __init__(self, kind=None, stream=None, force_styling=False):
"""
Initialize the terminal.
:arg str kind: A terminal string as taken by :func:`curses.setupterm`.
Defaults to the value of the ``TERM`` environment variable.
.. note:: Terminals withing a single process must share a common
``kind``. See :obj:`_CUR_TERM`.
:arg file stream: A file-like object representing the Terminal output.
Defaults to the original value of :obj:`sys.__stdout__`, like
:func:`curses.initscr` does.
If ``stream`` is not a tty, empty Unicode strings are returned for
all capability values, so things like piping your program output to
a pipe or file does not emit terminal sequences.
:arg bool force_styling: Whether to force the emission of capabilities
even if :obj:`sys.__stdout__` does not seem to be connected to a
terminal. If you want to force styling to not happen, use
``force_styling=None``.
This comes in handy if users are trying to pipe your output through
something like ``less -r`` or build systems which support decoding
of terminal sequences.
"""
# pylint: disable=global-statement,too-many-branches
global _CUR_TERM
self._keyboard_fd = None
# Default stream is stdout, keyboard valid as stdin only when
# output stream is stdout or stderr and is a tty.
if stream is None:
stream = sys.__stdout__
if stream in (sys.__stdout__, sys.__stderr__):
self._keyboard_fd = sys.__stdin__.fileno()
# we assume our input stream to be line-buffered until either the
# cbreak of raw context manager methods are entered with an
# attached tty.
self._line_buffered = True
try:
stream_fd = (stream.fileno() if hasattr(stream, 'fileno')
and callable(stream.fileno) else None)
except io.UnsupportedOperation:
stream_fd = None
self._stream = stream
self._is_a_tty = stream_fd is not None and os.isatty(stream_fd)
self._does_styling = ((self.is_a_tty or force_styling)
and force_styling is not None)
# _keyboard_fd only non-None if both stdin and stdout is a tty.
self._keyboard_fd = (self._keyboard_fd
if self._keyboard_fd is not None and self.is_a_tty
and os.isatty(self._keyboard_fd) else None)
self._normal = None # cache normal attr, preventing recursive lookups
# The descriptor to direct terminal initialization sequences to.
self._init_descriptor = (stream_fd is None and sys.__stdout__.fileno()
or stream_fd)
self._kind = kind or os.environ.get('TERM', 'unknown')
if self.does_styling:
# Initialize curses (call setupterm).
#
# Make things like tigetstr() work. Explicit args make setupterm()
# work even when -s is passed to nosetests. Lean toward sending
# init sequences to the stream if it has a file descriptor, and
# send them to stdout as a fallback, since they have to go
# somewhere.
try:
curses.setupterm(self._kind, self._init_descriptor)
except curses.error as err:
warnings.warn('Failed to setupterm(kind={0!r}): {1}'.format(
self._kind, err))
self._kind = None
self._does_styling = False
else:
if _CUR_TERM is None or self._kind == _CUR_TERM:
_CUR_TERM = self._kind
else:
warnings.warn(
'A terminal of kind "%s" has been requested; due to an'
' internal python curses bug, terminal capabilities'
' for a terminal of kind "%s" will continue to be'
' returned for the remainder of this process.' % (
self._kind,
_CUR_TERM,
))
# initialize capabilities and terminal keycodes database
self.__init__capabilities()
self.__init__keycodes()
_need_flush = any(c.istty for _, c in _con.items())
# Unix and Windows/msys
else:
def _istty(file):
return file.isatty()
def _print_el(file, s):
print(file=file, end=s)
try:
import curses as _cu
_cu.setupterm()
_cols = _cu.tigetnum("colors")
_afstr = _cu.tigetstr("setaf")
_abstr = _cu.tigetstr("setab")
_colbold = _cu.tigetstr("bold").decode('ascii')
_colreset = _cu.tigetstr("sgr0").decode('ascii')
def _can_use(file):
return _cols and _cols >= 8
def _colors(file):
return min(256, _cols)
def _set_color(color, f):
if not color:
def main():
host = '127.0.0.1'
port = '5000'
data = None
type_operation = None
name_nn = None
if len(sys.argv) > 1:
if sys.argv[1].find(':') != -1: # Задание адреса сервера через host:port
if len(sys.argv) > 2:
if sys.argv[2] == 'list_nn': # Получить список имеющихся нейронных сетей и их адреса
name_nn = sys.argv[2]
elif sys.argv[2] == 'lenet': # Нейронная сеть LeNet
if len(sys.argv) > 3:
if sys.argv[3] == 'classify': # Классифицировать изображение с цифрой
if len(sys.argv) > 4: # Задать изображение для классификации
if sys.argv[4].find('.jpg') != -1 or sys.argv[4].find('.png') != -1 or sys.argv[4].find('.bmp') != -1 or sys.argv[4].find('.tiff') != -1:
img_path = sys.argv[4]
try:
with open(img_path, "rb") as fh:
data = fh.read()
except FileNotFoundError as e:
print('\n[E] ' + str(e) + '\n')
return
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[4] + "'. Введите help для помощи.\n")
return
type_operation = sys.argv[3]
elif sys.argv[3] == 'status': # Получить статус сети (точность классификации и дата последнего обучения)
type_operation = sys.argv[3]
elif sys.argv[3] == 'train': # Запустить обучение сети на наборе данных MNIST или предварительно созданном наборе данных
type_operation = sys.argv[3]
if len(sys.argv) > 4:
if sys.argv[4] == 'other':
data = 'other'
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[4] + "'. Введите help для помощи.\n")
return
else:
data = 'mnist'
elif sys.argv[3] == 'about': # Получить информацию о сети
type_operation = sys.argv[3]
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[3] + "'. Введите help для помощи.\n")
return
else:
print("\n[E] Ожидается ещё один параметр после '" + sys.argv[2] + "'. Введите help для помощи.\n")
return
name_nn = sys.argv[2]
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[2] + "'. Введите help для помощи.\n")
return
host = sys.argv[1][:sys.argv[1].find(':')]
port = sys.argv[1][sys.argv[1].find(':') + 1:]
elif sys.argv[1] == 'list_nn': # Получить список имеющихся нейронных сетей и их адреса
name_nn = sys.argv[1]
host = get_address_on_local_network()
elif sys.argv[1] == 'lenet': # Нейронная сеть LeNet
if len(sys.argv) > 2:
if sys.argv[2] == 'classify': # Классифицировать изображение с цифрой
if len(sys.argv) > 3: # Задать изображение для классификации
if sys.argv[3].find('.jpg') != -1 or sys.argv[3].find('.png') != -1 or sys.argv[3].find('.bmp') != -1 or sys.argv[3].find('.tiff') != -1:
img_path = sys.argv[3]
try:
with open(img_path, "rb") as fh:
data = fh.read()
except FileNotFoundError as e:
print('\n[E] ' + str(e) + '\n')
return
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[3] + "'. Введите help для помощи.\n")
return
type_operation = sys.argv[2]
elif sys.argv[2] == 'status': # Получить статус сети (точность классификации и дата последнего обучения)
type_operation = sys.argv[2]
elif sys.argv[2] == 'train': # Запустить обучение сети на наборе данных MNIST или предварительно созданном наборе данных
type_operation = sys.argv[2]
if len(sys.argv) > 3:
if sys.argv[3] == 'other':
data = 'other'
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[3] + "'. Введите help для помощи.\n")
return
else:
data = 'mnist'
elif sys.argv[2] == 'about': # Получить информацию о сети
type_operation = sys.argv[2]
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[2] + "'. Введите help для помощи.\n")
return
else:
print("\n[E] Ожидается ещё один параметр после '" + sys.argv[1] + "'. Введите help для помощи.\n")
return
name_nn = sys.argv[1]
host = get_address_on_local_network()
elif sys.argv[1] == 'help':
print('\nПоддерживаемые варианты работы:')
print('\tбез аргументов - запуск с автоопределением адреса машины в локальной сети и портом 5000')
print('\thost:port - запуск с подключением к host:port')
print('\thost:port list_nn - получить список имеющихся нейронных сетей и их адреса')
print('\thost:port lenet classify - классифицировать изображение с цифрой с помощью сети LeNet')
print('\thost:port lenet classify image.jpg - классифицировать image.jpg/.png/.bmp/.tiff с помощью сети LeNet')
print('\thost:port lenet status - получить статус сети LeNet')
print('\thost:port lenet about - получить информацию о сети LeNet')
print('\thost:port lenet train - запустить обучение сети LeNet на наборе данных MNIST')
print('\thost:port lenet train other - запустить обучение сети LeNet на предварительно созданном наборе данных')
print('\tlist_nn - получить список имеющихся нейронных сетей и их адреса')
print('\tlenet classify - классифицировать изображение с цифрой с помощью сети LeNet (host определяется автоматически, port = 5000)')
print('\tlenet classify image.jpg - классифицировать image.jpg/.png/.bmp/.tiff с помощью сети LeNet (host определяется автоматически, port = 5000)')
print('\tlenet status - получить статус сети LeNet (host определяется автоматически, port = 5000)')
print('\tlenet about - получить информацию о сети LeNet (host определяется автоматически, port = 5000)')
print('\tlenet train - запустить обучение сети LeNet на наборе данных MNIST (host определяется автоматически, port = 5000)')
print('\tlenet train other - запустить обучение сети LeNet на предварительно созданном наборе данных (host определяется автоматически, port = 5000)\n')
return
else:
print("\n[E] Неверный аргумент командной строки '" + sys.argv[1] + "'. Введите help для помощи.\n")
return
else: # Определение host автоматически
host = get_address_on_local_network()
if type_operation == None and name_nn == None: # Если не были переданы аргументы командной строки
curses.setupterm()
print('[i] Выберите вариант работы клиента:')
print('\t1. list_nn - получить список имеющихся нейронных сетей и их адреса')
print('\t2. lenet classify - классифицировать изображение с цифрой с помощью сети LeNet')
print('\t3. lenet status - получить статус сети LeNet')
print('\t4. lenet about - получить информацию о сети LeNet')
print('\t5. lenet train - запустить обучение сети LeNet на наборе данных MNIST')
print('\t6. lenet train other - запустить обучение сети LeNet на предварительно созданном наборе данных')
name_nn = 'lenet'
while True:
choice = input('Введите цифру: ')
if choice == '1':
name_nn = 'list_nn'
break
elif choice == '2':
type_operation = 'classify'
if data == None:
for_exit = True
while for_exit:
number_image = input('Введите номер изображения из директории image с цифрой для распознавания (0..9): ')
for digit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']:
if number_image == digit:
for_exit = False
break
if for_exit:
print('Необходимо ввести цифру от 0 до 9!')
# Загрузка изображения
img_path = 'images/' + number_image + '.jpg'
data = None
with open(img_path, 'rb') as f_image:
data = f_image.read()
break
elif choice == '3':
type_operation = 'status'
break
elif choice == '4':
type_operation = 'about'
break
elif choice == '5':
choice = input('Вы уверены?(д/н) ')
if choice == 'д' or choice == 'y':
type_operation = 'train'
data = 'mnist'
break
elif choice == '6':
choice = input('Вы уверены?(д/н) ')
if choice == 'д' or choice == 'y':
type_operation = 'train'
data = 'other'
break
else:
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
if data == None and type_operation == 'classify': # Если не был передан адрес изображения
for_exit = True
while for_exit:
number_image = input('Введите номер изображения из директории image с цифрой для распознавания (0..9): ')
for digit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']:
if number_image == digit:
for_exit = False
break
if for_exit:
print('Необходимо ввести цифру от 0 до 9!')
# Загрузка изображения
img_path = 'images/' + number_image + '.jpg'
data = None
with open(img_path, 'rb') as f_image:
data = f_image.read()
start_time = time.time()
try:
result = access_to_nn_server(host, port, name_nn, type_operation, data=data)
except requests.exceptions.RequestException as e:
print('\n[E] ' + str(e) + '\n')
return
end_time = time.time()
if isinstance(result, tuple):
print('Результат обработки запроса: точность классификации %s%%, дата последнего обучения %s' % (result[0], result[1]))
elif isinstance(result, list):
print('Результат обработки запроса: %s' % result)
elif result.find('[E]') != -1:
print(result)
else:
print('Результат запроса: ' + result)
print('Обработано за %.4f c' % (end_time - start_time))
def run(self, config, options, args, help=None):
if options.output:
output = StringIO()
global curses
if curses and config.progress_bar:
try:
curses.setupterm()
except:
curses = None
else:
output = sys.stdout
if not self.checks:
self.load_checks_from_options(options.checks)
self.load_bugs(options.bugfile)
self.load_false_positives(options.falsepositivesfile)
config.devhelp_dirname = options.devhelp_dirname
config.partial_build = False
module_set = jhbuild.moduleset.load(config)
if options.list_all_modules:
self.module_list = module_set.modules.values()
else:
self.module_list = module_set.get_module_list(
args or config.modules, config.skip)
results = {}
try:
cachedir = os.path.join(os.environ['XDG_CACHE_HOME'], 'jhbuild')
except KeyError:
cachedir = os.path.join(os.environ['HOME'], '.cache', 'jhbuild')
if options.cache:
try:
results = cPickle.load(
file(os.path.join(cachedir, options.cache)))
except:
pass
self.repeat_row_header = 0
if len(self.checks) > 4:
self.repeat_row_header = 1
for module_num, mod in enumerate(self.module_list):
if mod.type in ('meta', 'tarball'):
continue
if not mod.branch or not mod.branch.repository.__class__.__name__ in (
'SubversionRepository', 'GitRepository'):
if not mod.moduleset_name.startswith('gnome-external-deps'):
continue
if not os.path.exists(mod.branch.srcdir):
continue
tree_id = mod.branch.tree_id()
valid_cache = (tree_id and results.get(mod.name, {}).get('tree-id')
== tree_id)
if not mod.name in results:
results[mod.name] = {'results': {}}
results[mod.name]['tree-id'] = tree_id
r = results[mod.name]['results']
for check in self.checks:
if valid_cache and check.__name__ in r:
continue
try:
c = check(config, mod)
except ExcludedModuleException:
continue
if output != sys.stdout and config.progress_bar:
progress_percent = 1.0 * (module_num - 1) / len(
self.module_list)
msg = '%s: %s' % (mod.name, check.__name__)
self.display_status_line(progress_percent, module_num, msg)
try:
c.run()
except CouldNotPerformCheckException:
continue
except ExcludedModuleException:
continue
try:
c.fix_false_positive(
self.false_positives.get((mod.name, check.__name__)))
except ExcludedModuleException:
continue
r[check.__name__] = [c.status, c.complexity, c.result_comment]
if not os.path.exists(cachedir):
os.makedirs(cachedir)
if options.cache:
cPickle.dump(results,
file(os.path.join(cachedir, options.cache), 'w'))
print >> output, HTML_AT_TOP % {'title': self.title}
if self.page_intro:
print >> output, self.page_intro
print >> output, '<table>'
print >> output, '<thead>'
print >> output, '<tr><td></td>'
for check in self.checks:
print >> output, '<th>%s</th>' % check.__name__
print >> output, '<td></td></tr>'
if [x for x in self.checks if x.header_note]:
print >> output, '<tr><td></td>'
for check in self.checks:
print >> output, '<td>%s</td>' % (check.header_note or '')
print >> output, '</tr>'
print >> output, '</thead>'
print >> output, '<tbody>'
suites = []
for module_key, module in module_set.modules.items():
if not isinstance(module_set.get_module(module_key), MetaModule):
continue
if module_key.endswith('upcoming-deprecations'):
# mark deprecated modules as processed, so they don't show in "Others"
try:
metamodule = module_set.get_module(meta_key)
except KeyError:
continue
for module_name in metamodule.dependencies:
processed_modules[module_name] = True
else:
suites.append([module_key, module_key.replace('meta-', '')])
processed_modules = {'gnome-common': True}
not_other_module_names = []
for suite_key, suite_label in suites:
metamodule = module_set.get_module(suite_key)
module_names = [x for x in metamodule.dependencies if x in results]
if not module_names:
continue
print >> output, '<tr><td class="heading" colspan="%d">%s</td></tr>' % (
1 + len(self.checks) + self.repeat_row_header, suite_label)
for module_name in module_names:
if module_name in not_other_module_names:
continue
r = results[module_name].get('results')
print >> output, self.get_mod_line(module_name, r)
processed_modules[module_name] = True
not_other_module_names.extend(module_names)
external_deps = [x for x in results.keys() if \
x in [y.name for y in self.module_list] and \
not x in processed_modules and \
module_set.get_module(x).moduleset_name.startswith('gnome-external-deps')]
if external_deps:
print >> output, '<tr><td class="heading" colspan="%d">%s</td></tr>' % (
1 + len(self.checks) + self.repeat_row_header,
'External Dependencies')
for module_name in sorted(external_deps):
if not module_name in results:
continue
r = results[module_name].get('results')
try:
version = module_set.get_module(module_name).branch.version
except:
version = None
print >> output, self.get_mod_line(module_name,
r,
version_number=version)
other_module_names = [x for x in results.keys() if \
not x in processed_modules and not x in external_deps]
if other_module_names:
print >> output, '<tr><td class="heading" colspan="%d">%s</td></tr>' % (
1 + len(self.checks) + self.repeat_row_header, 'Others')
for module_name in sorted(other_module_names):
if not module_name in results:
continue
r = results[module_name].get('results')
print >> output, self.get_mod_line(module_name, r)
print >> output, '</tbody>'
print >> output, '<tfoot>'
print >> output, '<tr><td></td>'
for check in self.checks:
print >> output, '<th>%s</th>' % check.__name__
print >> output, '<td></td></tr>'
print >> output, self.get_stat_line(results, not_other_module_names)
print >> output, '</tfoot>'
print >> output, '</table>'
if (options.bugfile and options.bugfile.startswith('http://')) or \
(options.falsepositivesfile and options.falsepositivesfile.startswith('http://')):
print >> output, '<div id="data">'
print >> output, '<p>The following data sources are used:</p>'
print >> output, '<ul>'
if options.bugfile.startswith('http://'):
print >> output, ' <li><a href="%s">Bugs</a></li>' % options.bugfile
if options.falsepositivesfile.startswith('http://'):
print >> output, ' <li><a href="%s">False positives</a></li>' % options.falsepositivesfile
print >> output, '</ul>'
print >> output, '</div>'
print >> output, '<div id="footer">'
print >> output, 'Generated:', time.strftime('%Y-%m-%d %H:%M:%S %z')
print >> output, 'on ', socket.getfqdn()
print >> output, '</div>'
print >> output, '</body>'
print >> output, '</html>'
if output != sys.stdout:
file(options.output, 'w').write(output.getvalue())
if output != sys.stdout and config.progress_bar:
sys.stdout.write('\n')
sys.stdout.flush()
def main():
curses.setupterm()
f_name_zip_odict = 'odict.zip'
f_name_odict = 'zaliznyak.txt'
f_name_current_exception_dict = 'stressrnn/source_exception_dictionary.txt'
f_name_new_exception_dict = 'stressrnn/exception_dictionary.txt'
parser = argparse.ArgumentParser(description="Converting Zaliznyak's dictionary from http://odict.ru/ to the format of " + \
"exception dictionary and combining it with the current exception dictionary.")
parser.add_argument(
'-iz',
'--f_name_zip_odict',
type=str,
default=None,
help="Name of .zip archive with the Zaliznyak's dictionary")
parser.add_argument(
'-i',
'--f_name_odict',
type=str,
default=None,
help="Name of .txt file with the Zaliznyak's dictionary")
parser.add_argument(
'-ic',
'--f_name_current_exception_dict',
type=str,
default=None,
help=
"Name of .txt file with the current exception dictionary (it will be combined with the Zaliznyak's dictionary)"
)
parser.add_argument(
'-o',
'--f_name_new_exception_dict',
type=str,
default=None,
help="Name of .txt file to save the combined dictionary")
args = parser.parse_args()
if args.f_name_zip_odict and args.f_name_odict:
print(
"[W] 'f_name_zip_odict' and 'f_name_odict' are set simultaneously — the value from 'f_name_odict' will be used."
)
f_name_zip_odict = None
f_name_odict = args.f_name_odict
elif args.f_name_zip_odict and not args.f_name_odict:
f_name_zip_odict = args.f_name_zip_odict
elif not args.f_name_zip_odict and args.f_name_odict:
f_name_zip_odict = None
f_name_odict = args.f_name_odict
if args.f_name_current_exception_dict:
f_name_current_exception_dict = args.f_name_current_exception_dict
if args.f_name_new_exception_dict:
f_name_new_exception_dict = args.f_name_new_exception_dict
# Unpacking archive with the dictionary to the same folder, where the archive is located
start_time = time.time()
if f_name_zip_odict:
print("[i] Unpacking '{}'...".format(f_name_zip_odict))
with zipfile.ZipFile(f_name_zip_odict, 'r') as zip_odict:
zip_odict.extractall(os.path.dirname(f_name_zip_odict))
f_name_odict = zip_odict.namelist()[0]
print(
"[i] Loading Zaliznyak's dictionary from '{}'...".format(f_name_odict))
zaliznyak_dict = []
with open(f_name_odict, 'r') as f_odict:
zaliznyak_dict = f_odict.readlines()
zaliznyak_dict[0] = zaliznyak_dict[0].replace('\ufeff', '')
print('[i] Loaded {} values'.format(len(zaliznyak_dict)))
print(
"[i] Converting Zaliznyak's dictionary to the format of exception dictionary..."
)
for i, word in enumerate(zaliznyak_dict):
word = word.replace('\n', '').lower().split(' ')
if not word[0] or count_number_of_vowels(word[0]) == 0:
zaliznyak_dict[i] = ''
continue
word, stress_index = [subword for subword in word if subword][:2]
if stress_index.find(',') != -1 or stress_index.find('.') != -1:
zaliznyak_dict[i] = ''
continue
if word[0] == '-':
word = word[1:]
j = 0
while j < len(word):
if SAME_LETTERS_EN_RU.get(word[j]):
word = word.replace(word[j], SAME_LETTERS_EN_RU[word[j]])
j += 1
zaliznyak_dict[i] = word[:int(stress_index)] + '+' + word[
int(stress_index):] + '\n'
zaliznyak_dict = [word for word in zaliznyak_dict if word]
print('[i] After converting, there are {} values left'.format(
len(zaliznyak_dict)))
print("[i] Loading current exception dictionary from '{}'...".format(
f_name_current_exception_dict))
current_exception_dict = []
with open(f_name_current_exception_dict, 'r') as f_exception_dict:
current_exception_dict = f_exception_dict.readlines()
current_exception_dict[-1] += '\n'
print('[i] Loaded {} values'.format(len(current_exception_dict)))
print('[i] Combining dictionaries... 0 of {}'.format(
len(current_exception_dict)))
zaliznyak_dict_without_stresses = [
word.replace('+', '') for word in zaliznyak_dict
]
for i, word in enumerate(current_exception_dict):
if i % 1000 == 0 or i == len(current_exception_dict) - 1:
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print('[i] Combining dictionaries... {} of {}'.format(
i, len(current_exception_dict)))
if word not in zaliznyak_dict and word.replace(
'+', '') in zaliznyak_dict_without_stresses:
zaliznyak_dict[zaliznyak_dict_without_stresses.index(
word.replace('+', ''))] = word
zaliznyak_dict = current_exception_dict + zaliznyak_dict
zaliznyak_dict = sorted(list(set(zaliznyak_dict)))
print("[i] Saving {} values in '{}'...".format(len(zaliznyak_dict),
f_name_new_exception_dict))
with open(f_name_new_exception_dict, 'w') as f_new_exception_dict:
f_new_exception_dict.writelines(zaliznyak_dict)
if f_name_zip_odict:
os.remove(f_name_odict)
print('[i] Done in {:.2f} second(-s)'.format(time.time() - start_time))
def main(username, sharer, wlist, num):
"""Create UDFs, folders and files for the given user using a wordlist."""
user = get_storage_user(None, username=utf2unicode(username))
sharer = get_storage_user(None, username=utf2unicode(sharer))
folders = [user.root]
with open(wlist) as f:
names = [utf2unicode(i.strip()) for i in f.readlines() if i.strip()]
sample = random.sample(names, num)
if sys.stdout.isatty():
import curses
curses.setupterm()
cols = curses.tigetnum('cols') - 1
progbar = makeStatBar(cols, cols - 2)
home = curses.tigetstr('cr')
def progress(ll):
"""progress bar writer."""
sys.stdout.write(
home + progbar((cols - 2) * (num - len(ll)) / num))
sys.stdout.flush()
else:
def progress(ll): return
# UDF
udf = user.make_udf('~/abundant-files')
folders.append(user.volume(udf.id).get_node(udf.root_id))
# some UDFs
for i in range(num / 100):
progress(sample)
folders.append(make_udf(user, sample))
for i in range(num / 4):
progress(sample)
name = sample.pop()
folders.append(random.choice(folders).make_subdirectory(name))
sh_folders = [sharer.root]
for i in range(num / 10):
progress(sample)
sh_folders.append(make_udf(sharer, sample))
for i in range(num / 10):
progress(sample)
name = sample.pop()
sh_folders.append(random.choice(sh_folders).make_subdirectory(name))
for i in range(num / 20):
progress(sample)
name = sample.pop()
filename = 'shared by ' + sharer.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(sh_folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(user.id, folder.name, readonly)
user.get_share(share.id).accept()
for i in random.sample(folders, len(folders) / 4):
progress(sample)
name = sample.pop()
filename = 'shared by ' + user.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(sharer.id, folder.name, readonly)
sharer.get_share(share.id).accept()
for i in range(num / 20):
progress(sample)
name = sample.pop()
random.choice(folders).make_file(name)
fake_hash = Factory().get_fake_hash()
while sample:
progress(sample)
name = sample.pop()
random.choice(folders).make_file_with_content(
name, fake_hash, 12345, 100, 10000, uuid.uuid4(), 'image/tiff')
if sys.stdout.isatty():
sys.stdout.write(home + curses.tigetstr('el'))
def main(args=None):
uids = {}
gids = {}
modedata = (
(stat.S_IRUSR, "-r"),
(stat.S_IWUSR, "-w"),
(stat.S_IXUSR, "-x"),
(stat.S_IRGRP, "-r"),
(stat.S_IWGRP, "-w"),
(stat.S_IXGRP, "-x"),
(stat.S_IROTH, "-r"),
(stat.S_IWOTH, "-w"),
(stat.S_IXOTH, "-x"),
)
curses.setupterm()
width = curses.tigetnum('cols')
def rpad(s, l):
meas = str(s)
if not isinstance(s, (str, astyle.Text)):
s = str(s)
if len(meas) < l:
size = l - len(meas)
psize = len(args.padding)
repeats = (size + psize - 1) // psize
padding = (args.padding * repeats)[-size:]
return astyle.style_default(s, style_pad(padding))
return s
def lpad(s, l):
meas = str(s)
if not isinstance(s, (str, astyle.Text)):
s = str(s)
if len(meas) < l:
size = l - len(meas)
psize = len(args.padding)
repeats = (size + psize - 1) // psize
padding = (args.padding * repeats)[:size]
return astyle.style_default(style_pad(padding), s)
return s
def findcolcount(urls):
def width4cols(numcols):
cols = [0] * numcols
rows = (len(urls) + numcols - 1) // numcols
for (i, (u, su)) in enumerate(urls):
cols[i // rows] = max(cols[i // rows], len(su))
return (sum(cols) + (numcols - 1) * args.spacing, rows, cols)
numcols = len(urls)
if numcols:
while True:
(s, rows, cols) = width4cols(numcols)
if s <= width or numcols == 1:
return (rows, cols)
numcols -= 1
else:
return (0, 0)
def printone(url):
if args.long:
sep = style_pad(args.separator)
stat = url.stat()
owner = url.owner()
group = url.group()
mtime = datetime.datetime.fromtimestamp(
stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S")
mode = "".join(
[text[bool(stat.st_mode & bit)] for (bit, text) in modedata])
size = stat.st_size
if args.human:
s = "BKMGTP"
for c in s:
if size < 2048:
if c == "B":
size = str(int(size))
else:
size = astyle.style_default(
str(int(size)), style_sizeunit(c))
break
size /= 1024.
stdout.write(mode, sep, rpad(owner, 8), sep, rpad(group, 8), sep,
lpad(size, 5 if args.human else 12), sep,
lpad(stat.st_nlink, 3), sep, mtime, sep)
if url.isdir():
stdout.writeln(style_dir(str(url)))
else:
stdout.writeln(style_file(str(url)))
def printblock(url, urls):
if url is not None:
stdout.writeln(style_dir(str(url)), ":")
(rows, cols) = findcolcount(urls)
for i in range(rows):
for (j, w) in enumerate(cols):
index = i + j * rows
try:
(u, su) = urls[index]
except IndexError:
pass
else:
if u.isdir():
su = style_dir(su)
else:
su = style_file(su)
if index + rows < len(urls):
su = rpad(su, w + args.spacing)
stdout.write(su)
stdout.writeln()
def printall(base, url):
if url.isdir():
if url.path.segments[-1]:
url.path.segments.append("")
if not args.long and not args.one:
if args.recursive:
urls = [(url / child, str(child))
for child in url.files(include=args.include,
exclude=args.exclude,
ignorecase=args.ignorecase)]
if urls:
printblock(url, urls)
for child in url.dirs():
if url_.matchpatterns(child.path[-2], enterdir,
skipdir):
printall(base, url / child)
else:
urls = [
(url / child, str(child))
for child in url.listdir(include=args.include,
exclude=args.exclude,
ignorecase=args.ignorecase)
]
printblock(None, urls)
else:
for child in url.listdir(include=args.include,
exclude=args.exclude,
ignorecase=args.ignorecase):
child = url / child
isdir = child.isdir()
if not args.recursive or isdir: # For files the print call is done by the recursive call to ``printall``
printone(child)
if args.recursive and (not isdir or url_.matchpatterns(
child.path[-2], enterdir, skipdir)):
printall(base, child)
else:
printone(url)
p = argparse.ArgumentParser(
description="List the content of one or more URLs",
epilog=
"For more info see http://www.livinglogic.de/Python/scripts_uls.html")
p.add_argument("urls",
metavar="url",
help="URLs to be listed (default: current dir)",
nargs="*",
default=[url_.Dir("./", scheme=None)],
type=url_.URL)
p.add_argument("-c",
"--color",
dest="color",
help="Color output (default: %(default)s)",
default="auto",
choices=("yes", "no", "auto"))
p.add_argument("-1",
"--one",
dest="one",
help="One entry per line? (default: %(default)s)",
action=misc.FlagAction,
default=False)
p.add_argument("-l",
"--long",
dest="long",
help="Long format? (default: %(default)s)",
action=misc.FlagAction,
default=False)
p.add_argument("-s",
"--human-readable-sizes",
dest="human",
help="Human readable file sizes? (default: %(default)s)",
action=misc.FlagAction,
default=False)
p.add_argument("-r",
"--recursive",
dest="recursive",
help="Recursive listing? (default: %(default)s)",
action=misc.FlagAction,
default=False)
p.add_argument("-w",
"--spacing",
dest="spacing",
metavar="INTEGER",
help="Space between columns (default: %(default)s)",
type=int,
default=3)
p.add_argument(
"-P",
"--padding",
dest="padding",
metavar="CHARS",
help="Characters used for column padding (default: %(default)s)",
default=" ",
type=str)
p.add_argument(
"-S",
"--separator",
dest="separator",
metavar="CHARS",
help=
"Characters used for separating columns in long format (default: %(default)s)",
default=" ",
type=str)
p.add_argument("-i",
"--include",
dest="include",
metavar="PATTERN",
help="Include only URLs matching PATTERN",
action="append")
p.add_argument("-e",
"--exclude",
dest="exclude",
metavar="PATTERN",
help="Exclude URLs matching PATTERN",
action="append")
p.add_argument("--enterdir",
dest="enterdir",
metavar="PATTERN",
help="Only enter directories matching PATTERN",
action="append")
p.add_argument("--skipdir",
dest="skipdir",
metavar="PATTERN",
help="Skip directories matching PATTERN",
action="append")
p.add_argument(
"--ignorecase",
dest="ignorecase",
help="Perform case-insensitive name matching? (default: %(default)s)",
action=misc.FlagAction,
default=False)
args = p.parse_args(args)
if args.color == "yes":
color = True
elif args.color == "no":
color = False
else:
color = None
stdout = astyle.Stream(sys.stdout, color)
stderr = astyle.Stream(sys.stderr, color)
enterdir = url_.compilepattern(args.enterdir, ignorecase=args.ignorecase)
skipdir = url_.compilepattern(args.skipdir, ignorecase=args.ignorecase)
with url_.Context():
for u in args.urls:
printall(u, u)
def print_help(commands):
"""
Tornado's options.print_help() function with a few minor changes:
* Help text is not hard wrapped (why did the Tornado devs do that? Ugh).
* It includes information about Gate One 'commands'.
* It only prints to stdout.
"""
import textwrap, fcntl, termios, struct
renditions = False
try:
import curses
if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
renditions = True
except Exception:
renditions = False
except ImportError:
pass
def bold(text):
if renditions:
return "\x1b[1m%s\x1b[0m" % text
return text
print("Usage: %s [OPTIONS]" % sys.argv[0])
print(bold("\nOptions:\n"))
rows, columns, hp, wp = struct.unpack(
'HHHH',
fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0)))
by_group = {}
for option in options._options.values():
by_group.setdefault(option.group_name, []).append(option)
for filename, o in sorted(by_group.items()):
if filename:
print(bold("\n%s options:\n" % os.path.normpath(filename)))
o.sort(key=lambda option: option.name)
for option in o:
prefix = option.name
if option.metavar:
prefix += "=" + option.metavar
description = option.help or ""
if option.default is not None and option.default != '':
description += " (default %s)" % option.default
lines = textwrap.wrap(description, columns - 35)
if len(prefix) > 30 or len(lines) == 0:
lines.insert(0, '')
print(" --%-30s %s" % (prefix, lines[0]))
for line in lines[1:]:
print("%-34s %s" % (' ', line))
print(bold("\nCommands:\n"))
print(" Usage: %s <command> [OPTIONS]\n" % sys.argv[0])
commands_description = _(
"GateOne supports many different CLI 'commands' which can be used "
"to invoke special functionality provided by plugins and applications "
"(and application's plugins). Each command can have it's own options "
"and most will have a --help function of their own.")
lines = textwrap.wrap(commands_description, columns)
for line in lines:
print("%s %s" % (' ', line))
print("")
for module, command_dict in commands.items():
print(bold("Commands provided by '%s':\n" % module))
for command, details in sorted(command_dict.items()):
print(" %-32s %s" % (command, details['description']))
print("")
print(bold("Example command usage:\n"))
print(" %s termlog --help" % sys.argv[0])
print("") # The oh-so-important whitespace before the prompt
sys.exit(1)
def main():
# install a signal handler so the script can stop safely
def signal_handler(signum, frame):
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
if tty_fd:
os.close(tty_fd)
if SDL_WasInit(SDL_INIT_JOYSTICK) == SDL_INIT_JOYSTICK:
SDL_QuitSubSystem(SDL_INIT_JOYSTICK)
SDL_Quit()
LOG.debug(f'{sys.argv[0]} exiting cleanly')
sys.exit(0)
debug_flag, hex_chars = parse_arguments(sys.argv)
if debug_flag:
LOG.setLevel(logging.DEBUG)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
# daemonize after signal handlers are registered
if os.fork():
os._exit(0)
try:
tty_fd = os.open('/dev/tty', os.O_WRONLY)
except IOError:
LOG.error('Unable to open /dev/tty', file=sys.stderr)
sys.exit(1)
curses.setupterm()
mapped_chars = [
get_hex_chars(code) for code in hex_chars
if get_hex_chars(code) is not None
]
def_buttons = [
'left', 'right', 'up', 'down', 'a', 'b', 'x', 'y', 'pageup', 'pagedown'
]
joy_map = {}
# add for each button the mapped keycode, based on the arguments received
for i, btn in enumerate(def_buttons):
if i < len(mapped_chars):
joy_map[btn] = mapped_chars[i]
menu_swap = ra_btn_swap_config()
# if button A is <enter> and menu_swap_ok_cancel_buttons is true, swap buttons A and B functions
if menu_swap \
and 'a' in joy_map.keys() \
and 'b' in joy_map.keys() \
and joy_map['a'] == '\n':
joy_map['a'] = joy_map['b']
joy_map['b'] = '\n'
# tell SDL that we don't want to grab and lock the keyboard
os.environ['SDL_INPUT_LINUX_KEEP_KBD'] = '1'
# disable the HIDAPI joystick driver in SDL
if not (SDL_USE_HIDAPI):
os.environ['SDL_JOYSTICK_HIDAPI'] = '0'
# tell SDL to not add any signal handlers for TERM/INT
os.environ['SDL_NO_SIGNAL_HANDLERS'] = '1'
configs = get_all_ra_config(def_buttons)
if SDL_Init(SDL_INIT_JOYSTICK) < 0:
LOG.error(f'Error in SDL_Init: {SDL_GetError()}')
exit(2)
if LOG.isEnabledFor(logging.DEBUG):
sdl_ver = version.SDL_version()
version.SDL_GetVersion(byref(sdl_ver))
wrapper_version = '.'.join(str(i) for i in version_info)
LOG.debug(
f'Using SDL Version {sdl_ver.major}.{sdl_ver.minor}.{sdl_ver.patch}, PySDL2 version {wrapper_version}'
)
if joystick.SDL_NumJoysticks() < 1:
LOG.debug(f'No available joystick devices found on startup')
event_loop(configs, joy_map, tty_fd)
SDL_QuitSubSystem(SDL_INIT_JOYSTICK)
SDL_Quit()
return 0
def execute_droidbox(argv, logs_directory):
sendsms = {}
phonecalls = {}
cryptousage = {}
dexclass = {}
dataleaks = {}
opennet = {}
sendnet = {}
recvnet = {}
closenet = {}
fdaccess = {}
servicestart = {}
accessedfiles = {}
if len(argv) < 2 or len(argv) > 3:
print("Usage: droidbox.py filename.apk <duration in seconds>")
sys.exit(1)
duration = 0
# Duration given?
if len(argv) == 3:
duration = int(argv[2])
apkName = argv[1]
# APK existing?
if os.path.isfile(apkName) == False:
print("File %s not found" % argv[1])
sys.exit(1)
application = Application(apkName)
ret = application.processAPK()
# Error during the APK processing?
if (ret == 0):
print("Failed to analyze the APK. Terminate the analysis.")
# sys.exit(1)
return "", ""
activities = application.getActivities()
mainActivity = application.getMainActivity()
packageName = application.getPackage()
recvsaction = application.getRecvActions()
enfperm = application.getEnfperm()
# Get the hashes
hashes = application.getHashes()
curses.setupterm()
sys.stdout.write(curses.tigetstr("clear"))
sys.stdout.flush()
call(['adb', 'logcat', '-c'])
print " ____ __ ____"
print "/\ _`\ __ /\ \/\ _`\\"
print "\ \ \/\ \ _ __ ___ /\_\ \_\ \ \ \L\ \ ___ __ _"
print " \ \ \ \ \/\`'__\ __`\/\ \ /'_` \ \ _ <' / __`\/\ \/'\\"
print " \ \ \_\ \ \ \/\ \L\ \ \ \/\ \L\ \ \ \L\ \\ \L\ \/> </"
print " \ \____/\ \_\ \____/\ \_\ \___,_\ \____/ \____//\_/\_\\"
print " \/___/ \/_/\/___/ \/_/\/__,_ /\/___/ \/___/ \//\/_/"
# No Main acitvity found? Return an error
if mainActivity is None:
print("No activity to start. Terminate the analysis.")
return "", ""
# No packages identified? Return an error
if packageName is None:
print("No package found. Terminate the analysis.")
return "", ""
PID_ZYGOTE = ""
output_ps = subprocess.Popen(['adb', 'shell', 'ps'],
stdout=subprocess.PIPE)
lines = output_ps.stdout.readlines()
for line in lines:
if "zygote" in line:
PID_ZYGOTE = line.split()[1]
break
p_mount = subprocess.Popen(
["adb", "shell", "mount", "-o", "rw,remount", "rootfs", "/"])
p_mount.wait()
file_execute_strace = open("executeStraceIn.sh", "w")
file_execute_strace.write("strace -p " + str(PID_ZYGOTE) +
" -f -ttF -o /sdcard/" + apkName.split("/")[-1] +
"_strace_monkey.txt")
file_execute_strace.close()
p_push = subprocess.Popen(
["adb", "push", "executeStraceIn.sh", "/sdcard/"])
p_push.wait()
# Execute the application
ret = call([
'monkeyrunner', 'monkeyrunner.py', apkName, packageName, mainActivity
],
stderr=PIPE,
cwd=os.path.dirname(os.path.realpath(__file__)))
if (ret == 1):
print("Failed to execute the application.")
return "", ""
print("Starting the activity %s..." % mainActivity)
# By default the application has not started
applicationStarted = 0
stringApplicationStarted = "Start proc %s" % packageName
# Open the adb logcat
adb = Popen(
["adb", "logcat", "DroidBox:W", "dalvikvm:W", "ActivityManager:I"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
strace_process = subprocess.Popen(
["adb", "shell", "sh", "./sdcard/executeStraceIn.sh"],
stdout=subprocess.PIPE)
# Wait for the application to start
starting_time = time.time()
while 1:
time.sleep(0.01)
# int "IN LOOP"
current_time = time.time()
if current_time - starting_time > 400:
print "MAX TIME TO START ACTIVITY REACHED. Stopping..."
return "", ""
try:
logcatInput = adb.stdout.readline()
if not logcatInput:
raise Exception("We have lost the connection with ADB.")
# Application started?
if stringApplicationStarted in logcatInput:
# print "Application started..."
applicationStarted = 1
break
except:
break
if (applicationStarted == 0):
print("Analysis has not been done.")
# Kill ADB, otherwise it will never terminate
os.kill(adb.pid, signal.SIGTERM)
sys.exit(1)
print("Application started")
print("Analyzing the application during %s seconds..." %
(duration if (duration != 0) else "infinite time"))
print "APP LAUNCHED " + apkName + "/-- " + packageName
count = CountingThread()
count.start()
timeStamp = time.time()
if duration:
signal.signal(signal.SIGALRM, interruptHandler)
signal.alarm(duration)
output_log = ""
# Collect DroidBox logs
while 1:
try:
logcatInput = adb.stdout.readline()
output_log += logcatInput
if not logcatInput:
raise Exception("We have lost the connection with ADB.")
boxlog = logcatInput.split('DroidBox:')
if len(boxlog) > 1:
try:
load = json.loads(decode(boxlog[1]))
# DexClassLoader
if load.has_key('DexClassLoader'):
load['DexClassLoader']['type'] = 'dexload'
dexclass[time.time() -
timeStamp] = load['DexClassLoader']
count.increaseCount()
# service started
if load.has_key('ServiceStart'):
load['ServiceStart']['type'] = 'service'
servicestart[time.time() -
timeStamp] = load['ServiceStart']
count.increaseCount()
# received data from net
if load.has_key('RecvNet'):
host = load['RecvNet']['srchost']
port = load['RecvNet']['srcport']
recvnet[time.time() - timeStamp] = recvdata = {
'type': 'net read',
'host': host,
'port': port,
'data': load['RecvNet']['data']
}
count.increaseCount()
# fdaccess
if load.has_key('FdAccess'):
accessedfiles[load['FdAccess']['id']] = hexToStr(
load['FdAccess']['path'])
# file read or write
if load.has_key('FileRW'):
load['FileRW']['path'] = accessedfiles[load['FileRW']
['id']]
if load['FileRW']['operation'] == 'write':
load['FileRW']['type'] = 'file write'
else:
load['FileRW']['type'] = 'file read'
fdaccess[time.time() - timeStamp] = load['FileRW']
count.increaseCount()
# opened network connection log
if load.has_key('OpenNet'):
opennet[time.time() - timeStamp] = load['OpenNet']
count.increaseCount()
# closed socket
if load.has_key('CloseNet'):
closenet[time.time() - timeStamp] = load['CloseNet']
count.increaseCount()
# outgoing network activity log
if load.has_key('SendNet'):
load['SendNet']['type'] = 'net write'
sendnet[time.time() - timeStamp] = load['SendNet']
count.increaseCount()
# data leak log
if load.has_key('DataLeak'):
my_time = time.time() - timeStamp
load['DataLeak']['type'] = 'leak'
load['DataLeak']['tag'] = getTags(
int(load['DataLeak']['tag'], 16))
dataleaks[my_time] = load['DataLeak']
count.increaseCount()
if load['DataLeak']['sink'] == 'Network':
load['DataLeak']['type'] = 'net write'
sendnet[my_time] = load['DataLeak']
count.increaseCount()
elif load['DataLeak']['sink'] == 'File':
load['DataLeak']['path'] = accessedfiles[
load['DataLeak']['id']]
if load['DataLeak']['operation'] == 'write':
load['DataLeak']['type'] = 'file write'
else:
load['DataLeak']['type'] = 'file read'
fdaccess[my_time] = load['DataLeak']
count.increaseCount()
elif load['DataLeak']['sink'] == 'SMS':
load['DataLeak']['type'] = 'sms'
sendsms[my_time] = load['DataLeak']
count.increaseCount()
# sent sms log
if load.has_key('SendSMS'):
load['SendSMS']['type'] = 'sms'
sendsms[time.time() - timeStamp] = load['SendSMS']
count.increaseCount()
# phone call log
if load.has_key('PhoneCall'):
load['PhoneCall']['type'] = 'call'
phonecalls[time.time() - timeStamp] = load['PhoneCall']
count.increaseCount()
# crypto api usage log
if load.has_key('CryptoUsage'):
load['CryptoUsage']['type'] = 'crypto'
cryptousage[time.time() -
timeStamp] = load['CryptoUsage']
count.increaseCount()
except ValueError:
pass
except:
try:
count.stopCounting()
count.join()
finally:
break
executionTime = time.time() - timeStamp
PID_STRACE = ""
output_ps = subprocess.Popen(['adb', 'shell', 'ps'],
stdout=subprocess.PIPE)
lines = output_ps.stdout.readlines()
for line in lines:
if "strace" in line:
PID_STRACE = line.split()[1]
break
# Killing strace
subprocess.Popen(["adb", "shell", "kill", "-9", PID_STRACE])
time.sleep(1)
subprocess_pull = subprocess.Popen([
"adb", "pull",
"/sdcard/" + apkName.split("/")[-1] + "_strace_monkey.txt"
])
subprocess_pull.wait()
shutil.move(
apkName.split("/")[-1] + "_strace_monkey.txt",
logs_directory + "strace_" + apkName.split("/")[-1] + ".txt")
subprocess_pull.wait()
# Kill ADB, otherwise it will never terminate
os.kill(adb.pid, signal.SIGTERM)
# Done? Store the objects in a dictionary, transform it in a JSON object and return it
output = dict()
# Sort the items by their key
output["dexclass"] = dexclass
output["servicestart"] = servicestart
output["recvnet"] = recvnet
output["opennet"] = opennet
output["sendnet"] = sendnet
output["closenet"] = closenet
output["accessedfiles"] = accessedfiles
output["dataleaks"] = dataleaks
output["fdaccess"] = fdaccess
output["sendsms"] = sendsms
output["phonecalls"] = phonecalls
output["cryptousage"] = cryptousage
output["recvsaction"] = recvsaction
output["enfperm"] = enfperm
output["hashes"] = hashes
output["apkName"] = apkName
output["executionTime"] = executionTime
output_json = json.dumps(output,
sort_keys=True,
indent=4,
separators=(',', ': '))
output = None
return output_log, output_json
def __init__(self):
"""Initialize curses."""
curses.setupterm()
def __init__(self, project=None, inp=None):
"""
Launch the Brownie console.
Arguments
---------
project : `Project`, optional
Active Brownie project to include in the console's local namespace.
inp : Input, optional
Input object to be passed to the prompt_toolkit `PromptSession`.
Required for unit testing to avoid conflicts with Pytest.
"""
console_settings = CONFIG.settings["console"]
locals_dict = dict((i, getattr(brownie, i)) for i in brownie.__all__)
locals_dict["dir"] = self._dir
if project:
project._update_and_register(locals_dict)
# only make GUI available if Tkinter is installed
try:
Gui = importlib.import_module("brownie._gui").Gui
locals_dict["Gui"] = Gui
except ModuleNotFoundError:
pass
# prepare lexer and formatter
self.lexer = PythonLexer()
fmt_name = "terminal"
try:
import curses
curses.setupterm()
if curses.tigetnum("colors") == 256:
fmt_name = "terminal256"
except Exception:
# if curses won't import we are probably using Windows
pass
self.formatter = get_formatter_by_name(
fmt_name, style=console_settings["color_style"])
# create prompt session object
history_file = str(_get_data_folder().joinpath(".history").absolute())
kwargs = {}
if console_settings["show_colors"]:
kwargs.update(
lexer=PygmentsLexer(PythonLexer),
style=style_from_pygments_cls(
get_style_by_name(console_settings["color_style"])),
include_default_pygments_style=False,
)
if console_settings["auto_suggest"]:
kwargs["auto_suggest"] = TestAutoSuggest(locals_dict)
if console_settings["completions"]:
kwargs["completer"] = ConsoleCompleter(locals_dict)
self.prompt_session = PromptSession(history=SanitizedFileHistory(
history_file, locals_dict),
input=inp,
**kwargs)
if console_settings["auto_suggest"]:
# remove the builting binding for auto-suggest acceptance
key_bindings = self.prompt_session.app.key_bindings
accept_binding = key_bindings.get_bindings_for_keys(("right", ))[0]
key_bindings._bindings2.remove(accept_binding.handler)
super().__init__(locals_dict)
def init():
"""
Initialize attributes with appropriate values for the current terminal.
`_term_stream` is the stream that will be used for terminal
output; if this stream is not a tty, then the terminal is
assumed to be a dumb terminal (i.e., have no capabilities).
"""
def _tigetstr(cap_name):
import curses
cap = curses.tigetstr(cap_name) or ''
# String capabilities can include "delays" of the form "$<2>".
# For any modern terminal, we should be able to just ignore
# these, so strip them out.
# terminof(5) states that:
# A "/" suffix indicates that the padding is mandatory and forces a
# delay of the given number of milliseconds even on devices for which
# xon is present to indicate flow control.
# So let's respect that. But not the optional ones.
cap = re.sub(r'\$<\d+>[*]?', '', cap)
# To switch back to "NORMAL", we use sgr0, which resets "everything" to defaults.
# That on some terminals includes ending "alternate character set mode".
# Which is usually done by appending '\017'. Unfortunately, less -R
# does not eat that, but shows it as an annoying inverse '^O'
# Instead of falling back to less -r, which would have other implications as well,
# strip off that '\017': we don't use the alternative character set,
# so we won't need to reset it either.
if cap_name == 'sgr0':
cap = re.sub(r'\017$', '', cap)
return cap
_term_stream = sys.stdout
# Curses isn't available on all platforms
try:
import curses
except:
sys.stderr.write("INFO: no curses support: you won't see colors\n")
return
# If the stream isn't a tty, then assume it has no capabilities.
from . import config
if not _term_stream.isatty() and 'color-always' not in config.color.style:
return
# Check the terminal type. If we fail, then assume that the
# terminal has no capabilities.
try:
curses.setupterm()
except:
return
# Look up numeric capabilities.
colors.COLS = curses.tigetnum('cols')
colors.LINES = curses.tigetnum('lines')
# Look up string capabilities.
for capability in _STRING_CAPABILITIES:
(attrib, cap_name) = capability.split('=')
setattr(colors, attrib, _tigetstr(cap_name) or '')
# Colors
set_fg = _tigetstr('setf')
if set_fg:
for i, color in zip(range(len(_COLORS)), _COLORS):
setattr(colors, color, curses.tparm(set_fg, i) or '')
set_fg_ansi = _tigetstr('setaf')
if set_fg_ansi:
for i, color in zip(range(len(_ANSICOLORS)), _ANSICOLORS):
setattr(colors, color, curses.tparm(set_fg_ansi, i) or '')
set_bg = _tigetstr('setb')
if set_bg:
for i, color in zip(range(len(_COLORS)), _COLORS):
setattr(colors, 'BG_' + color, curses.tparm(set_bg, i) or '')
set_bg_ansi = _tigetstr('setab')
if set_bg_ansi:
for i, color in zip(range(len(_ANSICOLORS)), _ANSICOLORS):
setattr(colors, 'BG_' + color, curses.tparm(set_bg_ansi, i) or '')
def func(args):
curses.setupterm()
sys.stdout.write(curses.tigetstr("clear"))
sys.stdout.flush()
def clear():
print('\n' * 2500)
curses.setupterm()
e3 = curses.tigetstr('E3') or b''
clear_screen_seq = curses.tigetstr('clear') or b''
os.write(sys.stdout.fileno(), e3 + clear_screen_seq)
def main(argv):
"""Main program."""
# Environment
if sys.version_info < (2, 7):
stderr('%s: need Python 2.7 or later' % argv[0])
stderr('your python is %s' % sys.version)
return 1
# Defaults
cfg = Options()
cfg.basedir = os.path.join(os.path.dirname(argv[0]), 'src')
cfg.basedir = os.path.abspath(cfg.basedir)
# Figure out terminal size
try:
import curses
except ImportError:
pass
else:
try:
curses.setupterm()
cols = curses.tigetnum('cols')
if cols > 0:
cfg.screen_width = cols
except (curses.error, TypeError):
# tigetnum() is broken in PyPy3 and raises TypeError
pass
# Option processing
opts, args = getopt.gnu_getopt(argv[1:], 'hvpqufw', [
'list-files', 'list-tests', 'list-hooks', 'level=', 'all-levels',
'coverage'
])
for k, v in opts:
if k == '-h':
print(__doc__)
return 0
elif k == '-v':
cfg.verbosity += 1
cfg.quiet = False
elif k == '-p':
cfg.progress = True
cfg.quiet = False
elif k == '-q':
cfg.verbosity = 0
cfg.progress = False
cfg.quiet = True
elif k == '-u':
cfg.unit_tests = True
elif k == '-f':
cfg.functional_tests = True
elif k == '-w':
cfg.warn_omitted = True
elif k == '--list-files':
cfg.list_files = True
cfg.run_tests = False
elif k == '--list-tests':
cfg.list_tests = True
cfg.run_tests = False
elif k == '--list-hooks':
cfg.list_hooks = True
cfg.run_tests = False
elif k == '--coverage':
cfg.coverage = True
elif k == '--level':
try:
cfg.level = int(v)
except ValueError:
stderr('%s: invalid level: %s' % (argv[0], v))
stderr('run %s -h for help')
return 1
elif k == '--all-levels':
cfg.level = None
else:
stderr('%s: invalid option: %s' % (argv[0], k))
stderr('run %s -h for help')
return 1
if args:
cfg.pathname_regex = args[0]
if len(args) > 1:
cfg.test_regex = args[1]
if len(args) > 2:
stderr('%s: too many arguments: %s' % (argv[0], args[2]))
stderr('run %s -h for help')
return 1
if not cfg.unit_tests and not cfg.functional_tests:
cfg.unit_tests = True
# Set up the python path
# sys.path.append(cfg.basedir)
sys.path.insert(0, cfg.basedir)
# Set up tracing before we start importing things
cov = None
if cfg.run_tests and cfg.coverage:
from coverage import coverage
# load the coverage file from same directory
cov_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'.coveragerc')
if not os.path.isfile(cov_cfg):
cov_cfg = None
cov = coverage(config_file=cov_cfg)
# Finding and importing
test_files = get_test_files(cfg)
# Determine whether using inpace module or from site-packages
try:
filename = os.path.splitext(test_files[0])[0]
modname = filename[len(cfg.basedir):].replace(os.path.sep, '.')
if modname.startswith('.'):
modname = modname[1:]
mod = __import__(modname)
del mod
del sys.modules[modname]
except ImportError:
if cfg.verbosity:
print("Using package from site-packages.")
else:
if cfg.verbosity:
print("Using inplace built. {mpath}".format(mpath=mod.__file__))
if cov is not None:
cov.start()
if cfg.list_tests or cfg.run_tests:
test_cases = get_test_cases(test_files, cfg, cov=cov)
if cfg.list_hooks or cfg.run_tests:
test_hooks = get_test_hooks(test_files, cfg, cov=cov)
# Configure the logging module
import logging
logging.basicConfig()
logging.root.setLevel(logging.CRITICAL)
# Running
success = True
if cfg.list_files:
baselen = len(cfg.basedir) + 1
print("\n".join([fn[baselen:] for fn in test_files]))
if cfg.list_tests:
print("\n".join([test.id() for test in test_cases]))
if cfg.list_hooks:
print("\n".join([str(hook) for hook in test_hooks]))
if cfg.run_tests:
runner = CustomTestRunner(cfg, test_hooks)
suite = unittest.TestSuite()
suite.addTests(test_cases)
if cov is not None:
cov.start()
run_result = runner.run(suite)
if cov is not None:
cov.stop()
success = run_result.wasSuccessful()
del run_result
if cov is not None:
traced_file_types = ('.py', '.pyx', '.pxi', '.pxd')
modules = []
def add_file(_, path, files):
if 'tests' in os.path.relpath(path, cfg.basedir).split(os.sep):
return
for filename in files:
if filename.endswith(traced_file_types):
modules.append(os.path.join(path, filename))
if cfg.follow_symlinks:
walker = walk_with_symlinks
else:
walker = os.path.walk
walker(os.path.abspath(cfg.basedir), add_file, None)
try:
cov.xml_report(modules, outfile='coverage.xml')
if cfg.coverdir:
cov.html_report(modules, directory=cfg.coverdir)
finally:
# test runs can take a while, so at least try to print something
cov.report()
# That's all
if success:
return 0
else:
return 1
def get_termstr(action):
global _term_setup
if not _term_setup:
curses.setupterm()
_term_setup = True
return curses.tigetstr(CODES[action]).decode('utf-8')
def main(screen):
logger = logging.getLogger('snake')
file_log_handler = logging.FileHandler('log')
logger.addHandler(file_log_handler)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_log_handler.setFormatter(formatter)
logger.setLevel('DEBUG')
logger.info('Game Started')
height, width = screen.getmaxyx()
start_screen(height, width)
curses.curs_set(0)
box = [[3, 3], [height - 3, width - 3]]
textpad.rectangle(screen, box[0][0], box[0][1], box[1][0], box[1][1])
screen.nodelay(1)
screen.timeout(100)
pos_x = width / 4
pos_y = height / 2
# Snake start position
snake = [[pos_y, pos_x], [pos_y, pos_x - 1], [pos_y, pos_x - 2]]
food = [height / 2, width / 3]
screen.addch(int(food[0]), int(food[1]), curses.ACS_PI)
screen.scrollok(0)
key = curses.KEY_RIGHT
score = 0
while True:
next_key = screen.getch()
key = key if next_key == -1 else next_key
if collision(snake, box):
msg = "GAME OVER!"
screen.addstr(height // 2, width // 2 - len(msg) // 2, msg)
screen.nodelay(0)
screen.getch()
logger.info('Game Over!')
break
new_head = [int(snake[0][0]), int(snake[0][1])]
if key == curses.KEY_DOWN:
new_head[0] += 1
if key == curses.KEY_UP:
new_head[0] -= 1
if key == curses.KEY_LEFT:
new_head[1] -= 1
if key == curses.KEY_RIGHT:
new_head[1] += 1
snake.insert(0, new_head)
# Score
scoreboard = "Score: {} Food: [{}, {}] Snake {}".format(
score, int(food[0]), int(food[1]), snake[0])
screen.addstr(1, width // 2 - len(scoreboard) // 2, scoreboard)
# If snake find the food
if snake[0] == food:
logger.info('Food ate @ [{}, {}]'.format(food[0], food[1]))
# Create a new food
food = new_food(snake, box)
screen.addch(int(food[0]), int(food[1]), curses.ACS_PI)
# increase snake length
snake.append([pos_y, pos_x - len(snake)])
# Update score
score += 1
scoreboard = "Score: {} Food: [{}, {}] Snake {}".format(
score, int(food[0]), int(food[1]), snake[0])
screen.addstr(1, width // 2 - len(scoreboard) // 2, scoreboard)
# Increase speed
screen.timeout(100 - (len(snake) // 3) % 90)
else:
tail = snake.pop()
try:
screen.addch(int(tail[0]), int(tail[1]), ' ')
except:
logger.error(screen)
pass
try:
screen.addch(int(snake[0][0]), int(snake[0][1]),
curses.ACS_CKBOARD)
except:
logger.error("2")
logger.error(curses.setupterm())
pass
class Environment:
"""
Information about the execution context
(standard streams, config directory, etc).
By default, it represents the actual environment.
All of the attributes can be overwritten though, which
is used by the test suite to simulate various scenarios.
"""
is_windows: bool = is_windows
config_dir: Path = DEFAULT_CONFIG_DIR
stdin: Optional[IO] = sys.stdin # `None` when closed fd (#791)
stdin_isatty: bool = stdin.isatty() if stdin else False
stdin_encoding: str = None
stdout: IO = sys.stdout
stdout_isatty: bool = stdout.isatty()
stdout_encoding: str = None
stderr: IO = sys.stderr
stderr_isatty: bool = stderr.isatty()
colors = 256
program_name: str = 'http'
if not is_windows:
if curses:
try:
curses.setupterm()
colors = curses.tigetnum('colors')
except curses.error:
pass
else:
# noinspection PyUnresolvedReferences
import colorama.initialise
stdout = colorama.initialise.wrap_stream(
stdout, convert=None, strip=None,
autoreset=True, wrap=True
)
stderr = colorama.initialise.wrap_stream(
stderr, convert=None, strip=None,
autoreset=True, wrap=True
)
del colorama
def __init__(self, devnull=None, **kwargs):
"""
Use keyword arguments to overwrite
any of the class attributes for this instance.
"""
assert all(hasattr(type(self), attr) for attr in kwargs.keys())
self.__dict__.update(**kwargs)
# The original STDERR unaffected by --quiet’ing.
self._orig_stderr = self.stderr
self._devnull = devnull
# Keyword arguments > stream.encoding > default UTF-8
if self.stdin and self.stdin_encoding is None:
self.stdin_encoding = getattr(
self.stdin, 'encoding', None) or UTF8
if self.stdout_encoding is None:
actual_stdout = self.stdout
if is_windows:
# noinspection PyUnresolvedReferences
from colorama import AnsiToWin32
if isinstance(self.stdout, AnsiToWin32):
# noinspection PyUnresolvedReferences
actual_stdout = self.stdout.wrapped
self.stdout_encoding = getattr(
actual_stdout, 'encoding', None) or UTF8
self.quiet = kwargs.pop('quiet', 0)
def __str__(self):
defaults = dict(type(self).__dict__)
actual = dict(defaults)
actual.update(self.__dict__)
actual['config'] = self.config
return repr_dict({
key: value
for key, value in actual.items()
if not key.startswith('_')
})
def __repr__(self):
return f'<{type(self).__name__} {self}>'
_config: Config = None
@property
def config(self) -> Config:
config = self._config
if not config:
self._config = config = Config(directory=self.config_dir)
if not config.is_new():
try:
config.load()
except ConfigFileError as e:
self.log_error(e, level='warning')
return config
@property
def devnull(self) -> IO:
if self._devnull is None:
self._devnull = open(os.devnull, 'w+')
return self._devnull
@contextmanager
def as_silent(self) -> Iterator[None]:
original_stdout = self.stdout
original_stderr = self.stderr
try:
self.stdout = self.devnull
self.stderr = self.devnull
yield
finally:
self.stdout = original_stdout
self.stderr = original_stderr
def log_error(self, msg: str, level: Levels = Levels.ERROR) -> None:
if self.stdout_isatty and self.quiet >= DISPLAY_THRESHOLDS[level]:
stderr = self.stderr # Not directly /dev/null, since stderr might be mocked
else:
stderr = self._orig_stderr
stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n')
def apply_warnings_filter(self) -> None:
if self.quiet >= DISPLAY_THRESHOLDS[Levels.WARNING]:
warnings.simplefilter("ignore")
def main(argv):
"""Main program."""
# Environment
if sys.version_info < (2, 3):
print >> sys.stderr, '%s: need Python 2.3 or later' % argv[0]
print >> sys.stderr, 'your python is %s' % sys.version
return 1
# Defaults
cfg = Options()
cfg.basedir = os.path.join(os.path.split(__file__)[0], '..', 'src')
cfg.basedir = os.path.abspath(cfg.basedir)
# Figure out terminal size
try:
import curses
except ImportError:
pass
else:
try:
curses.setupterm()
cols = curses.tigetnum('cols')
if cols > 0:
cfg.screen_width = cols
except curses.error:
pass
# Option processing
opts, args = getopt.gnu_getopt(argv[1:], 'hvpqufw', [
'list-files', 'list-tests', 'list-hooks', 'level=', 'all-levels',
'coverage'
])
for k, v in opts:
if k == '-h':
print __doc__
return 0
elif k == '-v':
cfg.verbosity += 1
cfg.quiet = False
elif k == '-p':
cfg.progress = True
cfg.quiet = False
elif k == '-q':
cfg.verbosity = 0
cfg.progress = False
cfg.quiet = True
elif k == '-u':
cfg.unit_tests = True
elif k == '-f':
cfg.functional_tests = True
elif k == '-w':
cfg.warn_omitted = True
elif k == '--list-files':
cfg.list_files = True
cfg.run_tests = False
elif k == '--list-tests':
cfg.list_tests = True
cfg.run_tests = False
elif k == '--list-hooks':
cfg.list_hooks = True
cfg.run_tests = False
elif k == '--coverage':
cfg.coverage = True
elif k == '--level':
try:
cfg.level = int(v)
except ValueError:
print >> sys.stderr, '%s: invalid level: %s' % (argv[0], v)
print >> sys.stderr, 'run %s -h for help'
return 1
elif k == '--all-levels':
cfg.level = None
else:
print >> sys.stderr, '%s: invalid option: %s' % (argv[0], k)
print >> sys.stderr, 'run %s -h for help'
return 1
if args:
cfg.pathname_regex = args[0]
if len(args) > 1:
cfg.test_regex = args[1]
if len(args) > 2:
print >> sys.stderr, '%s: too many arguments: %s' % (argv[0], args[2])
print >> sys.stderr, 'run %s -h for help'
return 1
if not cfg.unit_tests and not cfg.functional_tests:
cfg.unit_tests = True
# Set up the python path
sys.path[0] = cfg.basedir
# Set up tracing before we start importing things
tracer = None
if cfg.run_tests and cfg.coverage:
import trace
# trace.py in Python 2.3.1 is buggy:
# 1) Despite sys.prefix being in ignoredirs, a lot of system-wide
# modules are included in the coverage reports
# 2) Some module file names do not have the first two characters,
# and in general the prefix used seems to be arbitrary
# These bugs are fixed in src/trace.py which should be in PYTHONPATH
# before the official one.
ignoremods = ['test']
ignoredirs = [sys.prefix, sys.exec_prefix]
tracer = trace.Trace(count=True,
trace=False,
ignoremods=ignoremods,
ignoredirs=ignoredirs)
# Finding and importing
test_files = get_test_files(cfg)
if cfg.list_tests or cfg.run_tests:
test_cases = get_test_cases(test_files, cfg, tracer=tracer)
if cfg.list_hooks or cfg.run_tests:
test_hooks = get_test_hooks(test_files, cfg, tracer=tracer)
# Configure the logging module
import logging
logging.basicConfig()
logging.root.setLevel(logging.CRITICAL)
# Running
success = True
if cfg.list_files:
baselen = len(cfg.basedir) + 1
print "\n".join([fn[baselen:] for fn in test_files])
if cfg.list_tests:
print "\n".join([test.id() for test in test_cases])
if cfg.list_hooks:
print "\n".join([str(hook) for hook in test_hooks])
if cfg.run_tests:
runner = CustomTestRunner(cfg, test_hooks)
suite = unittest.TestSuite()
suite.addTests(test_cases)
if tracer is not None:
success = tracer.runfunc(runner.run, suite).wasSuccessful()
results = tracer.results()
results.write_results(show_missing=True, coverdir=cfg.coverdir)
else:
success = runner.run(suite).wasSuccessful()
# That's all
if success:
return 0
else:
return 1
if hasattr(curses, 'resizeterm'):
lines, cols = curses.LINES, curses.COLS
curses.resizeterm(lines - 1, cols + 1)
if curses.LINES != lines - 1 or curses.COLS != cols + 1:
raise RuntimeError, "Expected resizeterm to update LINES and COLS"
def main(stdscr):
curses.savetty()
try:
module_funcs(stdscr)
window_funcs(stdscr)
test_userptr_without_set(stdscr)
test_resize_term(stdscr)
finally:
curses.resetty()
if __name__ == '__main__':
curses.wrapper(main)
unit_tests()
else:
# testing setupterm() inside initscr/endwin
# causes terminal breakage
curses.setupterm(fd=sys.__stdout__.fileno())
try:
stdscr = curses.initscr()
main(stdscr)
finally:
curses.endwin()
unit_tests()
