def test_export_multiple_items(self):
i1 = TestItem(name='hello', age='world')
i2 = TestItem(name='bye', age='world')
f = StringIO()
ie = PickleItemExporter(f)
ie.start_exporting()
ie.export_item(i1)
ie.export_item(i2)
ie.finish_exporting()
f.reset()
self.assertEqual(pickle.load(f), i1)
self.assertEqual(pickle.load(f), i2)
def test_export_multiple_items(self):
i1 = TestItem(name='hello', age='world')
i2 = TestItem(name='bye', age='world')
f = StringIO()
ie = PickleItemExporter(f)
ie.start_exporting()
ie.export_item(i1)
ie.export_item(i2)
ie.finish_exporting()
f.reset()
self.assertEqual(pickle.load(f), i1)
self.assertEqual(pickle.load(f), i2)
class SocketExportPipeline(object): def __init__ (self, IP, Port): self.IP = IP self.Port = Port self.filepath = "/tmp/pickler.dat" self.socket = socket.socket() def spider_opened (self): self.file = open(self.filepath, "w") self.socket.connect((self.IP, self.Port)) self.pickler = PickleItemExporter(self.file) self.pickler.start_exporting() def process_item (self, item): if item: self.pickler.export_item(item) def spider_closed(): self.file.close() pickledfile = open(self.filepath, "r") for line in pickledfile: self.socket.send(line) self.socket.close() pickledfile.close()
class SocketExportPipeline(object):
def __init__(self, IP, Port):
self.IP = IP
self.Port = Port
self.filepath = "/tmp/pickler.dat"
self.socket = socket.socket()
def spider_opened(self):
self.file = open(self.filepath, "w")
self.socket.connect((self.IP, self.Port))
self.pickler = PickleItemExporter(self.file)
self.pickler.start_exporting()
def process_item(self, item):
if item:
self.pickler.export_item(item)
def spider_closed():
self.file.close()
pickledfile = open(self.filepath, "r")
for line in pickledfile:
self.socket.send(line)
self.socket.close()
pickledfile.close()
def _get_exporter(self, **kwargs):
return PickleItemExporter(self.output, **kwargs)
def spider_opened (self): self.file = open(self.filepath, "w") self.socket.connect((self.IP, self.Port)) self.pickler = PickleItemExporter(self.file) self.pickler.start_exporting()
def spider_opened(self):
self.file = open(self.filepath, "w")
self.socket.connect((self.IP, self.Port))
self.pickler = PickleItemExporter(self.file)
self.pickler.start_exporting()