def getProcessedSet(self): # TODO: refactor - same code is in scrapeforum, move out to a DB object config = Configurator() db = config.getDb() connectionString = config.getDbConnectionString() engine = create_engine(connectionString, echo=(db['echo'] == 'True')) Session = sessionmaker(bind=engine) session = Session() # TODO - end return session.query(LatestTopic).filter_by(batch_id=self.batch_id)
def __init__(self): try: # TODO: refactor - same code is in scrapeforum, move out to a DB object config = Configurator() db = config.getDb() connectionString = config.getDbConnectionString() engine = create_engine(connectionString, echo=(db['echo'] == 'True')) Session = sessionmaker(bind=engine) session = Session() # TODO - end # TODO add filter for 'success = 1' self.batch_id = session.query(func.max(LatestTopic.batch_id).label("max_batch_id")).one().max_batch_id except: # TODO: report error raise finally: engine.dispose()
try: config = Configurator() db = config.getDb() URL = config.getURL()["startURL"] logger.debug("URL is set to " + URL) except KeyError: logger.fatal("Unable to read configuration from '../config/config.ini'. Make sure you are running the script from 'scrapeforum' folder.") # TODO decide if it is needed or it's too much spam # Send notification email about start Mailer(constants.EMAIL_TYPE_NOTIFICATION).send("Scraping started", "Scraping started for " + URL + " at " + datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')) # Connect to database try: # Set up DB connection connectionString = config.getDbConnectionString() logger.debug("Trying to connect to DB with connectionString: " + connectionString) engine = create_engine(connectionString, echo=(db['echo'] == 'True')) # Create missing tables Batch.metadata.create_all(engine) LatestTopic.metadata.create_all(engine) except: errorStr = "Failed to connect to database. Connection string: " + connectionString + ". Exiting!" logger.fatal(errorStr) Mailer(constants.EMAIL_TYPE_ERROR).send("Fatal error - failed to connect to DB", errorStr) os._exit(1) try: # Set up and open a session