def test_nonstring_types_item(self): item = self._get_nonstring_types_item() item.pop('time') # datetime is not marshallable fp = tempfile.TemporaryFile() ie = MarshalItemExporter(fp) ie.start_exporting() ie.export_item(item) ie.finish_exporting() fp.seek(0) self.assertEqual(marshal.load(fp), item)
def test_nonstring_types_item(self): item = self._get_nonstring_types_item() item.pop('time') # datetime is not marshallable fp = tempfile.TemporaryFile() ie = MarshalItemExporter(fp) ie.start_exporting() ie.export_item(item) ie.finish_exporting() fp.seek(0) self.assertEqual(marshal.load(fp), item)
class MarshalExporterPipeline(object): def __init__(self, file_name): # Storing output filename self.file_name = file_name # Creating a file handle and setting it to None self.file_handle = None @classmethod def from_crawler(cls, crawler): return cls( file_name=crawler.settings.get('MARSHAL_PATH'), ) def open_spider(self, spider): print('Custom export opened') # Opening file in binary-write mode file = open(self.file_name, 'wb') self.file_handle = file # Creating a FanItemExporter object and initiating export self.exporter = MarshalItemExporter(file) self.exporter.start_exporting() def close_spider(self, spider): print('Custom Exporter closed') # Ending the export to file from FanItemExport object self.exporter.finish_exporting() # Closing the opened output file self.file_handle.close() def process_item(self, item, spider): # passing the item to FanItemExporter object for expoting to file self.exporter.export_item(item) return item