def setupProcess():
""" setup the db connection for the worker """
engine = sqlalchemy.create_engine(sqlalchemy_url, echo=sqlalchemy_echo)
model.maker = sessionmaker(autoflush=False, autocommit=False, extension=model.MySessionExtension())
model.DBSession = scoped_session(model.maker)
model.init_model(engine)
model.metadata.create_all(engine)
sys.exit("Could not read crawler.ini")
sqlalchemy_url = config.get('main', 'sqlalchemy.url')
sqlalchemy_echo = config.getboolean('main', 'sqlalchemy.echo')
debug_mode = config.getboolean('main', 'debug')
FORMAT = "%(asctime)-15s %(message)s"
if debug_mode:
logging.basicConfig(level=logging.DEBUG, format=FORMAT)
else:
logging.basicConfig(level=logging.INFO, format=FORMAT)
engine = sqlalchemy.create_engine(sqlalchemy_url)
model.maker = sessionmaker(autoflush=False, autocommit=False, extension=model.MySessionExtension())
model.DBSession = scoped_session(model.maker)
model.init_model(engine)
model.metadata.create_all(engine)
conn = engine.connect()
checkConn = engine.connect()
checkTrans = checkConn.begin()
for item in conn.execute( select([share.id],) ):
logging.debug("checking id: %i" % item[0])
checkItem(item[0], checkConn)
logging.info("valid Shares: %i invalid Shares: %i valid Hosts: %i invalid Hosts: %i" % (len(validShares), len(invalidShares), len(validHosts), len(invalidHosts)) )
logging.info("deleting invalid db entries")
for id in invalidShares:
logging.debug("deleting share id: %i" % id)
checkConn.execute( delete(share.__table__, share.id==id) )
checkTrans.commit()
def setup_db():
"""Method used to build a database"""
engine = config['pylons.app_globals'].sa_engine
model.init_model(engine)
model.metadata.create_all(engine)
