class QuarantinePipeline(object):
@classmethod
def from_crawler(cls, crawler):
ext = cls(crawler)
return ext
def __init__(self, *args, **kwargs):
self.processed_items = 0
self.db = QuarantineDatabase()
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
def spider_opened(self, spider):
self.db.create_conn(spider)
def spider_closed(self, spider):
spider.log('Closing scrape for spider: {}'.format(spider.publisher))
def process_item(self, item, spider):
self.db.save_item(item)
return item
def __init__(self, *args, **kwargs):
self.processed_items = 0
self.db = QuarantineDatabase()
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
