class RDBMSScoreBackend(SQLAlchemyBackend): component_name = 'RDBMS Score-based backend.' def __init__(self, manager): super(RDBMSScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy( super(RDBMSScoreBackend, self).get_next_requests, manager.logger.manager.debug) self.logger = getLogger('onetera.backends.RDBMSScoreBackend') def _get_order_by(self, query): return query.order_by(self.page_model.score.desc()) def _create_page(self, obj): db_page = super(RDBMSScoreBackend, self)._create_page(obj) db_page.score = obj.meta['scrapy_meta']['score'] return db_page def cleanup(self): for name, table in Base.metadata.tables.items(): self.session.execute(table.delete()) def get_next_requests(self, max_n_requests, **kwargs): return self._requests_buffer.get_next_requests(max_n_requests, **kwargs)
class MemoryScoreBackend(MemoryBaseBackend): component_name = 'Score-based memory backend' def __init__(self, manager): super(MemoryScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy(super(MemoryScoreBackend, self).get_next_requests, manager.logger.manager.debug) def _compare_pages(self, first, second): return cmp(second.meta['scrapy_meta']['score'], first.meta['scrapy_meta']['score']) def get_next_requests(self, max_n_requests, **kwargs): return self._requests_buffer.get_next_requests(max_n_requests, **kwargs) def cleanup(self): self.requests.clear() self.heap.heap = []
class MemoryScoreBackend(MemoryBaseBackend): component_name = 'Score-based memory backend' def __init__(self, manager): super(MemoryScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy( super(MemoryScoreBackend, self).get_next_requests, manager.logger.manager.debug) def _compare_pages(self, first, second): return cmp(second.meta['scrapy_meta']['score'], first.meta['scrapy_meta']['score']) def get_next_requests(self, max_n_requests, **kwargs): return self._requests_buffer.get_next_requests(max_n_requests, **kwargs) def cleanup(self): self.requests.clear() self.heap.heap = []
class RDBMSScoreBackend(SQLAlchemyBackend): component_name = 'RDBMS Score-based backend.' def __init__(self, manager): super(RDBMSScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy(super(RDBMSScoreBackend, self).get_next_requests, manager.logger.manager.debug) self.logger = getLogger('onetera.backends.RDBMSScoreBackend') def _get_order_by(self, query): return query.order_by(self.page_model.score.desc()) def _create_page(self, obj): db_page = super(RDBMSScoreBackend, self)._create_page(obj) db_page.score = obj.meta['scrapy_meta']['score'] return db_page def cleanup(self): for name, table in Base.metadata.tables.items(): self.session.execute(table.delete()) def get_next_requests(self, max_n_requests, **kwargs): return self._requests_buffer.get_next_requests(max_n_requests, **kwargs)
def __init__(self, manager): super(RDBMSScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy( super(RDBMSScoreBackend, self).get_next_requests, manager.logger.manager.debug) self.logger = getLogger('onetera.backends.RDBMSScoreBackend')
def __init__(self, manager): super(MemoryScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy( super(MemoryScoreBackend, self).get_next_requests, manager.logger.manager.debug)
def __init__(self, manager): super(RDBMSScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy(super(RDBMSScoreBackend, self).get_next_requests, manager.logger.manager.debug) self.logger = getLogger('onetera.backends.RDBMSScoreBackend')
def __init__(self, manager): super(MemoryScoreBackend, self).__init__(manager) self._requests_buffer = OverusedBufferScrapy(super(MemoryScoreBackend, self).get_next_requests, manager.logger.manager.debug)