def _all_parser(args):
outfile = args.outfile
files = _get_files_from_directories(args.directories,
args.recursive,
lambda v: 'xml' in v)
if runtime.verbose:
range_over = ProgressBar(files, which_tick=True)
else:
range_over = iter(files)
sha_to_ids = {}
for f in range_over:
try:
triples = paragraphs_to_sha(open(f, 'r'), keep_paragraphs=True)
for triple in triples:
sha_to_ids.setdefault(triple[2], [])\
.append((triple[0], triple[1], triple[3]))
except Exception, e:
sys.stderr.write("Error with %s: %s\n" % (f, str(e)))
def _shas_parser(args):
"""
Globs the files from the given directory and outputs the
"""
outfile = csv.writer(args.outfile)
files = _get_files_from_directories(args.directories,
args.recursive,
lambda v: 'xml' in v)
if runtime.verbose:
range_over = ProgressBar(files, which_tick=True)
else:
range_over = iter(files)
for f in range_over:
try:
to_write = paragraphs_to_sha(open(f, 'r'))
outfile.writerows(to_write)
except Exception, e:
sys.stderr.write("Error with %s: %s\n" % (f, str(e)))
