def main(reprocess=False): client = MongoClient('mongodb://localhost:27017') db = client.get_database('extractor') collection = db.reports for report in collection.find(): if report.has_key('filename'): print >> sys.stderr, 'Cleaning {}'.format(report['filename']) else: print >> sys.stderr, 'Cleaning {}'.format(report['message_id']) if not report.has_key('ast'): print ' -> Nothing to clean' continue else: if report.has_key('flat_ast'): if reprocess == False: print ' -> Already Clean' continue else: print ' -> Recleaning' else: print ' -> Cleaning' ast = report['ast'] report['flat_ast'] = flatten(ast) collection.save(report)