class MongoDBExportPipeline(object): def __init__(self): self.exporter = MongoDBExporter(); @classmethod def from_crawler(cls, crawler): pipeline = cls() crawler.signals.connect(pipeline.spider_opened, signals.spider_opened) crawler.signals.connect(pipeline.spider_closed, signals.spider_closed) return pipeline def spider_opened(self, spider): self.exporter.start_exporting() def spider_closed(self, spider): self.exporter.finish_exporting() def process_item(self, item, spider): if not item is None: self.exporter.export_item(item) return item
def __init__(self): self.exporter = MongoDBExporter();