def open_spider(self, spider): print('Custom export opened') # Opening file in binary-write mode file = open(self.file_name, 'wb') self.file_handle = file # Creating a FanItemExporter object and initiating export self.exporter = MarshalItemExporter(file) self.exporter.start_exporting()
def test_nonstring_types_item(self): item = self._get_nonstring_types_item() item.pop('time') # datetime is not marshallable fp = tempfile.TemporaryFile() ie = MarshalItemExporter(fp) ie.start_exporting() ie.export_item(item) ie.finish_exporting() fp.seek(0) self.assertEqual(marshal.load(fp), item)
def test_nonstring_types_item(self): item = self._get_nonstring_types_item() item.pop('time') # datetime is not marshallable fp = tempfile.TemporaryFile() ie = MarshalItemExporter(fp) ie.start_exporting() ie.export_item(item) ie.finish_exporting() fp.seek(0) self.assertEqual(marshal.load(fp), item)
def exporter_for_format(feed_format, f): if feed_format == 'csv': return CsvItemExporter(f) elif feed_format == 'xml': return XmlItemExporter(f) elif feed_format == 'json': return JsonItemExporter(f) elif feed_format == 'jsonlines': return JsonLinesItemExporter(f) elif feed_format == 'pickle': return PickleItemExporter(f) elif feed_format == 'marshal': return MarshalItemExporter(f) else: raise ValueError( 'Export format {} is not supported'.format(feed_format))
class MarshalExporterPipeline(object): def __init__(self, file_name): # Storing output filename self.file_name = file_name # Creating a file handle and setting it to None self.file_handle = None @classmethod def from_crawler(cls, crawler): return cls( file_name=crawler.settings.get('MARSHAL_PATH'), ) def open_spider(self, spider): print('Custom export opened') # Opening file in binary-write mode file = open(self.file_name, 'wb') self.file_handle = file # Creating a FanItemExporter object and initiating export self.exporter = MarshalItemExporter(file) self.exporter.start_exporting() def close_spider(self, spider): print('Custom Exporter closed') # Ending the export to file from FanItemExport object self.exporter.finish_exporting() # Closing the opened output file self.file_handle.close() def process_item(self, item, spider): # passing the item to FanItemExporter object for expoting to file self.exporter.export_item(item) return item
def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return MarshalItemExporter(self.output, **kwargs)