def main(argv):
parser = OptionParser(
conflict_handler='resolve') # Intelligently resolve switch collisions
parser.set_defaults(verbose=False,
filters_to_run=[],
packages=BuiltInPackages)
# If any library loading switches (-l) are given, collect their names and remove them from argv
argv, user_defined_libraries = filter_library_switches(argv)
argv, autoloaded_libraries = filter_autoload_paths(argv)
# Load built-in filters (those under BuiltInPackages)
# Load user-requested filters (passed by -l on the command line)
all_libraries = BuiltInPackages + user_defined_libraries + autoloaded_libraries
tracer = TraceCore(libraries=all_libraries)
# For each available filter, allow it to be invoked with switches on the command line
for filter in tracer.available_filters_dict.values():
add_filter_to_optparser(parser, filter)
# Load built-in optparse switches
register_builtin_switches(parser)
if len(argv) <= 1:
parser.print_help()
sys.exit(1)
# Perform option parse, check for user-requested filter classes
opts, remaining_args = parser.parse_args(argv)
# Done with parser
parser.destroy()
if opts.debug:
config.set(debug=True)
# Set verbose switch if given on command line
tracer.verbose = opts.verbose
tracer.break_on_exception = opts.break_on_exception
# Set override Reader if given on command line
tracer.reader_class_name = opts.reader_class_name
# Take remaining arguments as input file names
files = remaining_args[1:] # remaining_args[0] seems to be sys.argv[0]
# If switch -L was passed, dump out all available filter names and quit
if opts.do_list_filters:
tracer.list_filters()
sys.exit(0)
# Run requested filters
try:
tracer.run(opts.filters_to_run, files)
except RuntimeError, e:
err('RuntimeError: %s', e)
sys.exit(1)
def __init__(self, pager_path=None, files=None, verbose=True, clear_history=False):
HistorySavingDefaultShell.__init__(self, clear_history=clear_history)
self.tracer = TraceCore(libraries=BuiltInPackages, verbose=verbose, reader_class_name=config.cmd_reader_class)
self.prompt = "trace> "
self.files = files or []
self.last_exception = None
self._verbose = verbose
if pager_path:
self.pager_path = pager_path
self.output_file = PagerFilename
else:
self.pager_path = self.output_file = None
def main(argv):
parser = OptionParser(conflict_handler='resolve') # Intelligently resolve switch collisions
parser.set_defaults(verbose=False, filters_to_run=[], packages=BuiltInPackages)
# If any library loading switches (-l) are given, collect their names and remove them from argv
argv, user_defined_libraries = filter_library_switches(argv)
argv, autoloaded_libraries = filter_autoload_paths(argv)
# Load built-in filters (those under BuiltInPackages)
# Load user-requested filters (passed by -l on the command line)
all_libraries = BuiltInPackages + user_defined_libraries + autoloaded_libraries
tracer = TraceCore(libraries=all_libraries)
# For each available filter, allow it to be invoked with switches on the command line
for filter in tracer.available_filters_dict.values(): add_filter_to_optparser(parser, filter)
# Load built-in optparse switches
register_builtin_switches(parser)
if len(argv) <= 1:
parser.print_help()
sys.exit(1)
# Perform option parse, check for user-requested filter classes
opts, remaining_args = parser.parse_args(argv)
# Done with parser
parser.destroy()
if opts.debug:
config.set(debug=True)
# Set verbose switch if given on command line
tracer.verbose = opts.verbose
tracer.break_on_exception = opts.break_on_exception
# Set override Reader if given on command line
tracer.reader_class_name = opts.reader_class_name
# Take remaining arguments as input file names
files = remaining_args[1:] # remaining_args[0] seems to be sys.argv[0]
# If switch -L was passed, dump out all available filter names and quit
if opts.do_list_filters:
tracer.list_filters()
sys.exit(0)
# Run requested filters
try:
tracer.run(opts.filters_to_run, files)
except RuntimeError, e:
err('RuntimeError: %s', e)
sys.exit(1)
class Shell(HistorySavingDefaultShell):
'''A shell interface to trace functionality.'''
def __init__(self, pager_path=None, files=None, verbose=True, clear_history=False):
HistorySavingDefaultShell.__init__(self, clear_history=clear_history)
self.tracer = TraceCore(libraries=BuiltInPackages, verbose=verbose, reader_class_name=config.cmd_reader_class)
self.prompt = "trace> "
self.files = files or []
self.last_exception = None
self._verbose = verbose
if pager_path:
self.pager_path = pager_path
self.output_file = PagerFilename
else:
self.pager_path = self.output_file = None
def get_verbose(self): return self._verbose
def set_verbose(self, is_verbose):
self._verbose = self.tracer.verbose = is_verbose
verbose = property(get_verbose, set_verbose)
def do_quiet(self, args):
'''Produces less output during processing.'''
if self._verbose:
self.set_verbose(False)
msg("Will generate less output.")
def do_verbose(self, args):
'''Produces more output during processing.'''
if not self._verbose:
self.set_verbose(True)
msg("Will generate more output.")
def preloop(self):
'''Executed before the command loop is entered.'''
DefaultShell.preloop(self)
# remove '-' (for option completion) and a number of shell separators (for path completion)
# from the set of completer delimiters
readline.set_completer_delims(re.sub(r'[-/*~\\]', '', readline.get_completer_delims()))
def precmd(self, line):
'''Executed before every command is interpreted.'''
cleaned_line = line.strip()
self._hist.append(cleaned_line)
return cleaned_line
def do_load(self, args):
'''Loads filter modules given as package names.'''
modules = args.split()
old_modules = set(self.tracer.available_filters_dict.keys())
self.tracer.add_modules(modules)
modules = set(self.tracer.available_filters_dict.keys())
added_modules = modules.difference(old_modules)
if added_modules:
msg("%s modules added:", len(added_modules))
for module in added_modules:
msg("\t%s", module)
else:
msg("No modules added.")
@options([ make_option('-l', '--long', action='store_true', dest='long', help='Show detailed summary', default=False),
make_option('-s', '--sort-by', action='store', type='choice', choices=['name', 'module', 'opt', 'long-opt'],
dest='sort_key', help='Display filters in a given sorted order', default='name')])
def do_list(self, args, opts):
"""Lists all loaded filters."""
self.tracer.list_filters(long=opts.long, filter_sort_key=opts.sort_key)
def do_with(self, args):
'''Changes or displays the working set.'''
args = args.split()
if args:
self.files = []
for arg in args:
# So that if arg is not a glob, it won't get removed
# because it doesn't exist on the fs:
globbed = list(flatten(glob.glob(arg)))
if globbed:
self.files += globbed
else:
self.files.append(arg)
msg("Working set is: " + list_preview(self.files))
def get_filter_by_switch(self, switch_name):
'''Retrieves the filter object based on its short or long form switch name.'''
is_option_long_name = switch_name.startswith('--')
for filter in self.tracer.available_filters_dict.values():
if is_option_long_name:
if filter.long_opt == switch_name[2:]:
return filter.__name__
else:
if filter.opt == switch_name[1:]:
return filter.__name__
err("No filter with switch %s found.", switch_name)
return None
def do_run(self, args):
'''Runs the given filter with the given arguments on the working set.'''
args = args.split()
if not args: return
filter_name = args.pop(0)
if not filter_name: # no filter with the requested switch was found
return
if filter_name.startswith('--'): # a long option name was given
filter_name = self.get_filter_by_switch(filter_name)
elif filter_name.startswith('-'): # a short option name was given
filter_name = self.get_filter_by_switch(filter_name)
# Special case: for a one-arg filter any number of arguments are treated
# as a single argument.
if (filter_name in self.tracer and
get_argcount_for_method(self.tracer[filter_name].__init__) == 1):
filter_args = (' '.join(args), )
elif args is not None:
filter_args = args
else:
filter_args = None
def action():
self.tracer.run( [(filter_name, filter_args)], self.files )
print
self.redirecting_stdout(action, filter_name, filter_args)
def do_runmany(self, args):
# XXX: HACK HACK HACK HACK
filters = args.split()
if not args: return
def action():
self.tracer.run( [( filter_name, () ) for filter_name in filters], self.files )
print
self.redirecting_stdout(action, '|'.join(filters), ())
def do_backtrace(self, args):
'''Displays the exception backtrace for the last failed filter.'''
if self.last_exception:
sys.excepthook(*self.last_exception)
do_bt = do_backtrace
def create_pager_pipe(self):
if not self.pager_path:
raise RuntimeError('No pager was given.')
if os.path.basename(self.pager_path) == 'less':
params = (self.pager_path, '-R', '-')
else:
params = (self.pager_path, '-')
return subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=None, stderr=None,
shell=False)
def do_into(self, args):
'''Sets or displays the destination for filter output. The special filename
StdoutFilename will redirect filter output to the console.'''
def print_output_destination():
if self.output_file is None:
msg("Filter output will be sent to the console.")
elif self.output_file == PagerFilename:
msg("Filter output will be paged with %s.", self.pager_path)
else:
msg("Filter output will be redirected to: %s", self.output_file)
args = args.split(' ', 1)
output_file = args[0]
if output_file:
if output_file == StdoutFilename:
self.output_file = None
else: # we will treat the value PagerFilename specially
self.output_file = output_file
print_output_destination() # report on output destination in any case
def redirecting_stdout(self, action, filter_name, filter_args):
'''Calls the given _action_ with stdout temporarily redirected to _self.output_file_ or
a pager program.'''
try:
old_stdout = sys.stdout
pipe = None
if self.output_file == PagerFilename:
pipe = self.create_pager_pipe()
sys.stdout = pipe.stdin
elif self.output_file:
sys.stdout = open(self.output_file, 'w')
action()
if self.output_file == PagerFilename:
if pipe:
sys.stdout.close() # Signal EOF
pipe.wait()
except KeyboardInterrupt:
if pipe:
pipe.stdin.close()
pipe.wait()
msg("\nFilter run %s halted by user.", filter_run_name(filter_name, filter_args))
except Exception, e:
if pipe:
pipe.stdin.close()
pipe.wait()
msg("Filter run %s halted by framework:", filter_run_name(filter_name, filter_args))
# TODO: Exception no longer has a message field in Python 3.0, produces a DeprecationWarning
# under Python 2.7
msg("\t%s (%s)", e.message, e.__class__.__name__)
self.last_exception = sys.exc_info()
finally:
'''Yields slices from _L_ of length _K_ each.'''
if K <= 0: raise ValueError('required: k > 0')
i = 0
while i < len(L):
yield L[i:i+K]
i += K
def segmented_corpora(topdir, N):
'''Divides the files under _topdir_ into N groups of approximately equal size.'''
def all_files():
file_list = (f for f in glob.glob(os.path.join(topdir, '**'))
if os.path.isfile(f))
for file in file_list:
reader = GuessReader(file)
L = list(reader)
for i in xrange(1,len(L)+1):
yield file + ':%d' % i
files = list(all_files())
return as_slices(files, int(math.ceil(len(files)/N)))
if __name__ == '__main__':
trace = TraceCore(['apps.sanity'], verbose=False)
topdir = 'cn'
N = 10
for files in segmented_corpora(topdir, 10):
print '%s...%s' % (files[0], files[-1])
trace.run( [('SanityChecks', None)], files )
i = 0
while i < len(L):
yield L[i:i + K]
i += K
def segmented_corpora(topdir, N):
'''Divides the files under _topdir_ into N groups of approximately equal size.'''
def all_files():
file_list = (f for f in glob.glob(os.path.join(topdir, '**'))
if os.path.isfile(f))
for file in file_list:
reader = GuessReader(file)
L = list(reader)
for i in xrange(1, len(L) + 1):
yield file + ':%d' % i
files = list(all_files())
return as_slices(files, int(math.ceil(len(files) / N)))
if __name__ == '__main__':
trace = TraceCore(['apps.sanity'], verbose=False)
topdir = 'cn'
N = 10
for files in segmented_corpora(topdir, 10):
print '%s...%s' % (files[0], files[-1])
trace.run([('SanityChecks', None)], files)