def db_worker(cls, manager):
b = cls(manager)
settings = manager.settings
drop = settings.get('SQLALCHEMYBACKEND_DROP_ALL_TABLES')
clear_content = settings.get('SQLALCHEMYBACKEND_CLEAR_CONTENT')
inspector = Inspector.from_engine(b.engine)
metadata_m = b.models['MetadataModel']
queue_m = b.models['QueueModel']
if drop:
existing = inspector.get_table_names()
if metadata_m.__table__.name in existing:
metadata_m.__table__.drop(bind=b.engine)
if queue_m.__table__.name in existing:
queue_m.__table__.drop(bind=b.engine)
metadata_m.__table__.create(bind=b.engine)
queue_m.__table__.create(bind=b.engine)
if clear_content:
session = b.session_cls()
session.execute(metadata_m.__table__.delete())
session.execute(queue_m.__table__.delete())
session.close()
b._metadata = Metadata(b.session_cls, metadata_m,
settings.get('SQLALCHEMYBACKEND_CACHE_SIZE'))
b._queue = Queue(b.session_cls, queue_m, b.partitioner)
return b
def db_worker(cls, manager):
b = cls(manager)
settings = manager.settings
drop = settings.get('SQLALCHEMYBACKEND_DROP_ALL_TABLES')
clear_content = settings.get('SQLALCHEMYBACKEND_CLEAR_CONTENT')
metadata_m = b.models['MetadataModel']
queue_m = b.models['QueueModel']
if drop:
metadata_m.__table__.drop(bind=b.engine)
queue_m.__table__.drop(bind=b.engine)
metadata_m.__table__.create(bind=b.engine)
queue_m.__table__.create(bind=b.engine)
if clear_content:
session = b.session_cls()
session.execute(metadata_m.__table__.delete())
session.execute(queue_m.__table__.delete())
session.close()
b._metadata = Metadata(b.session_cls, metadata_m,
settings.get('SQLALCHEMYBACKEND_CACHE_SIZE'))
b._queue = Queue(b.session_cls, queue_m,
settings.get('SPIDER_FEED_PARTITIONS'))
return b
def _init_db_worker(self, manager):
settings = manager.settings
drop = settings.get('SQLALCHEMYBACKEND_DROP_ALL_TABLES')
clear_content = settings.get('SQLALCHEMYBACKEND_CLEAR_CONTENT')
metadata_m = self.models['MetadataModel']
queue_m = self.models['QueueModel']
self.check_and_create_tables(drop, clear_content, (metadata_m, queue_m))
self._metadata = Metadata(self.session_cls, metadata_m,
settings.get('SQLALCHEMYBACKEND_CACHE_SIZE'))
self._queue = Queue(self.session_cls, queue_m, settings.get('SPIDER_FEED_PARTITIONS'))
def _create_queue(self, settings):
return Queue(self.session_cls, self.models['QueueModel'],
self.partitioner)
def _create_queue(self, settings):
return Queue(self.session_cls,
self.models['QueueModel'],
self.partitioner,
ordering='created_desc')
def _create_queue(self, settings):
return Queue(self.session_cls, self.models['QueueModel'],
settings.get('SPIDER_FEED_PARTITIONS'))