def db_worker(cls, manager):
b = cls(manager)
settings = manager.settings
drop = settings.get('SQLALCHEMYBACKEND_DROP_ALL_TABLES')
clear_content = settings.get('SQLALCHEMYBACKEND_CLEAR_CONTENT')
inspector = Inspector.from_engine(b.engine)
metadata_m = b.models['MetadataModel']
queue_m = b.models['QueueModel']
if drop:
existing = inspector.get_table_names()
if metadata_m.__table__.name in existing:
metadata_m.__table__.drop(bind=b.engine)
if queue_m.__table__.name in existing:
queue_m.__table__.drop(bind=b.engine)
metadata_m.__table__.create(bind=b.engine)
queue_m.__table__.create(bind=b.engine)
if clear_content:
session = b.session_cls()
session.execute(metadata_m.__table__.delete())
session.execute(queue_m.__table__.delete())
session.close()
b._metadata = Metadata(b.session_cls, metadata_m,
settings.get('SQLALCHEMYBACKEND_CACHE_SIZE'))
b._queue = Queue(b.session_cls, queue_m, b.partitioner)
return b
def __init__(self, manager):
self.manager = manager
settings = manager.settings
engine = settings.get('SQLALCHEMYBACKEND_ENGINE')
engine_echo = settings.get('SQLALCHEMYBACKEND_ENGINE_ECHO')
drop_all_tables = settings.get('SQLALCHEMYBACKEND_DROP_ALL_TABLES')
clear_content = settings.get('SQLALCHEMYBACKEND_CLEAR_CONTENT')
models = settings.get('SQLALCHEMYBACKEND_MODELS')
self.engine = create_engine(engine, echo=engine_echo)
self.models = dict([(name, load_object(klass))
for name, klass in models.items()])
if drop_all_tables:
DeclarativeBase.metadata.drop_all(self.engine)
DeclarativeBase.metadata.create_all(self.engine)
self.session_cls = sessionmaker()
self.session_cls.configure(bind=self.engine)
if clear_content:
session = self.session_cls()
for name, table in DeclarativeBase.metadata.tables.items():
session.execute(table.delete())
session.close()
self._metadata = Metadata(self.session_cls,
self.models['MetadataModel'],
settings.get('SQLALCHEMYBACKEND_CACHE_SIZE'))
self._states = States(self.session_cls, self.models['StateModel'],
settings.get('STATE_CACHE_SIZE_LIMIT'))
self._queue = self._create_queue(settings)
def db_worker(cls, manager):
b = cls(manager)
settings = manager.settings
drop = settings.get('SQLALCHEMYBACKEND_DROP_ALL_TABLES')
clear_content = settings.get('SQLALCHEMYBACKEND_CLEAR_CONTENT')
metadata_m = b.models['MetadataModel']
queue_m = b.models['QueueModel']
if drop:
metadata_m.__table__.drop(bind=b.engine)
queue_m.__table__.drop(bind=b.engine)
metadata_m.__table__.create(bind=b.engine)
queue_m.__table__.create(bind=b.engine)
if clear_content:
session = b.session_cls()
session.execute(metadata_m.__table__.delete())
session.execute(queue_m.__table__.delete())
session.close()
b._metadata = Metadata(b.session_cls, metadata_m,
settings.get('SQLALCHEMYBACKEND_CACHE_SIZE'))
b._queue = Queue(b.session_cls, queue_m,
settings.get('SPIDER_FEED_PARTITIONS'))
return b
def _init_db_worker(self, manager):
settings = manager.settings
drop = settings.get('SQLALCHEMYBACKEND_DROP_ALL_TABLES')
clear_content = settings.get('SQLALCHEMYBACKEND_CLEAR_CONTENT')
metadata_m = self.models['MetadataModel']
queue_m = self.models['QueueModel']
self.check_and_create_tables(drop, clear_content, (metadata_m, queue_m))
self._metadata = Metadata(self.session_cls, metadata_m,
settings.get('SQLALCHEMYBACKEND_CACHE_SIZE'))
self._queue = WlwQueue(self.session_cls, queue_m, settings.get('SPIDER_FEED_PARTITIONS'))