class XmlExportPipeline(object): def __init__(self): dispatcher.connect(self.spider_opened, signals.spider_opened) dispatcher.connect(self.spider_closed, signals.spider_closed) self.count = 0 self.open_new_file() def spider_opened(self, spider): pass def spider_closed(self, spider): self.exporter.finish_exporting() self.file.close() def open_new_file(self): timestampt = time.time() self.file = open('/tmp/%s_products.xml' % timestampt, 'w+b') self.exporter = MyXmlItemExporter(self.file) self.exporter.start_exporting() def process_item(self, item, spider): self.count+=1 if self.count>10: self.exporter.finish_exporting() self.file.close() self.open_new_file() self.count = 0 self.exporter.export_item(item) return item
def open_new_file(self): timestampt = time.time() self.file = open('/tmp/%s_products.xml' % timestampt, 'w+b') self.exporter = MyXmlItemExporter(self.file) self.exporter.start_exporting()