def analyzeTask( inputFilePath ):
""" Read a .txt file, anylize it and store the resulti into a .bulk and/or .json file.
If the analysis fails, an according .json.failed and/or .bulk.failed file is created with the erorr message captured.
Return 0 on sucess and -1 on failure.
"""
print 'working on %s' % repr( inputFilePath )
txtDone=False
jsonDone=False
bulkDone=False
try:
frame = cleanedFrame( pagesToFrame( iteratePageText(inputFilePath) ) )
if args.txt:
cleanedText = '\n'.join(frame['text']).encode('utf-8')
saveText( cleanedText, mkOutputPath( inputFilePath, '.txt', args.txt ) )
txtDone=True
if args.json or args.bulk:
split( frame )
content = analyzeContent( frame )
if args.json: saveJson( content, mkOutputPath( inputFilePath, '.json', args.json ) )
jsonDone=True
if args.bulk: saveBulk( content, mkOutputPath( inputFilePath, '.bulk', args.bulk), index_key='id' )
bulkDone=True
except Exception as e:
from traceback import print_exc
if args.txt and not txtDone:
with open( mkOutputPath( inputFilePath, ".txt.failed", args.txt ), 'wt' ) as outputFile:
print_exc( None, outputFile )
if args.json and not jsonDone:
with open( mkOutputPath( inputFilePath, ".json.failed", args.txt ), 'wt' ) as outputFile:
print_exc( None, outputFile )
if args.bulk and not bulkDone:
with open( mkOutputPath( inputFilePath, ".bulk.failed", args.txt ), 'wt' ) as outputFile:
print_exc( None, outputFile )
print 'error working on %s' % repr( inputFilePath )
return -1
print 'done working on %s' % repr( inputFilePath )
return 0
import argparse
from saver import mkOutputPath, autoSave
from text_extractor import iteratePageText
from pagetext_cleaner import cleanedFrame, pagesToFrame
argparser = argparse.ArgumentParser(
description='Convert a pdf file to a txt file.',
usage='%(prog)s [options] file [file ...]',
)
argparser.add_argument( 'files', action='store', nargs='+',
help='the json files to be parsed' )
argparser.add_argument( '-o', '--output', action='store',
help='save the parsed files to directory OUTPUT' )
args = argparser.parse_args()
for ifpath in args.files:
frame = cleanedFrame( pagesToFrame( iteratePageText(ifpath) ) )
split( frame )
for sectionType, sectionFrame in frame.groupby( 'section' ):
with open( mkOutputPath( ifpath, '.'+sectionType+'.txt', args.output ), 'wt' ) as ofile:
for row in sectionFrame.itertuples():
ofile.write( "%s\n" % (row[5].encode('utf-8')) )
frame.to_pickle( mkOutputPath( ifpath, '.phase0.pkl', args.output ) )
