def main(**kwargs):
"""Entry point for the program."""
# set up logging
logger = logging.getLogger('formMailer')
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=kwargs['logfile'],
filemode='a')
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
starttime = datetime.datetime.now()
logger.info('Begin processing data...')
logger.debug('Current run inputs: {0}'.format(locals()))
collector = collector()
recFilter = recordFilter(collector)
db = dbInserter.dbInserter()
# build the first record individually, to create the necessary schema
rec = recFilter.next()
db.insertRec(rec, firstrec=True)
# loop over the rest of the records and insert them
try:
for c, rec in enumerate(recFilter):
db.insertRec(rec)
if not (c % 5000) and (c > 0):
logger.info('Added {0} total records to session so far'.format(c))
if not ((c+1) % 30000):
db.flush()
db.flush()
db.updatesAndViews()
db.commit()
except Exception, e:
logger.error(str(e))
db.rollback()
def setUp(self):
self.recordFilter = recordFilter()
