def __init__(self):
self.file_name = "class.pickle"
self.file_path = join(dirname(dirname(abspath(__file__))),
self.file_name)
self.file = open(self.file_path, 'wb')
self.exporter = PickleItemExporter(self.file)
self.exporter.start_exporting()
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
fp = BytesIO()
ie = PickleItemExporter(fp)
ie.start_exporting()
ie.export_item(item)
ie.finish_exporting()
self.assertEqual(pickle.loads(fp.getvalue()), item)
def open_spider(self, spider):
print('Custom export opened')
# Opening file in binary-write mode
file = open(self.file_name, 'wb')
self.file_handle = file
# Creating a FanItemExporter object and initiating export
self.exporter = PickleItemExporter(file)
self.exporter.start_exporting()
def test_export_multiple_items(self):
i1 = TestItem(name='hello', age='world')
i2 = TestItem(name='bye', age='world')
f = BytesIO()
ie = PickleItemExporter(f)
ie.start_exporting()
ie.export_item(i1)
ie.export_item(i2)
ie.finish_exporting()
f.seek(0)
self.assertEqual(pickle.load(f), i1)
self.assertEqual(pickle.load(f), i2)
def test_export_multiple_items(self):
i1 = self.item_class(name="hello", age="world")
i2 = self.item_class(name="bye", age="world")
f = BytesIO()
ie = PickleItemExporter(f)
ie.start_exporting()
ie.export_item(i1)
ie.export_item(i2)
ie.finish_exporting()
f.seek(0)
self.assertEqual(self.item_class(**pickle.load(f)), i1)
self.assertEqual(self.item_class(**pickle.load(f)), i2)
def exporter_for_format(feed_format, f):
if feed_format == 'csv':
return CsvItemExporter(f)
elif feed_format == 'xml':
return XmlItemExporter(f)
elif feed_format == 'json':
return JsonItemExporter(f)
elif feed_format == 'jsonlines':
return JsonLinesItemExporter(f)
elif feed_format == 'pickle':
return PickleItemExporter(f)
elif feed_format == 'marshal':
return MarshalItemExporter(f)
else:
raise ValueError(
'Export format {} is not supported'.format(feed_format))
def _get_exporter(self, **kwargs):
return PickleItemExporter(self.output, **kwargs)
def spider_opened(self, spider):
file = open('%s.pickle' % spider.name, 'w+b')
self.files[spider] = file
self.exporter = PickleItemExporter(file)
self.exporter.start_exporting()
