def _all_parser(args): outfile = args.outfile files = _get_files_from_directories(args.directories, args.recursive, lambda v: 'xml' in v) if runtime.verbose: range_over = ProgressBar(files, which_tick=True) else: range_over = iter(files) sha_to_ids = {} for f in range_over: try: triples = paragraphs_to_sha(open(f, 'r'), keep_paragraphs=True) for triple in triples: sha_to_ids.setdefault(triple[2], [])\ .append((triple[0], triple[1], triple[3])) except Exception, e: sys.stderr.write("Error with %s: %s\n" % (f, str(e)))
def _shas_parser(args): """ Globs the files from the given directory and outputs the """ outfile = csv.writer(args.outfile) files = _get_files_from_directories(args.directories, args.recursive, lambda v: 'xml' in v) if runtime.verbose: range_over = ProgressBar(files, which_tick=True) else: range_over = iter(files) for f in range_over: try: to_write = paragraphs_to_sha(open(f, 'r')) outfile.writerows(to_write) except Exception, e: sys.stderr.write("Error with %s: %s\n" % (f, str(e)))