def test_export_multiple_items(self): i1 = TestItem(name='hello', age='world') i2 = TestItem(name='bye', age='world') f = StringIO() ie = PickleItemExporter(f) ie.start_exporting() ie.export_item(i1) ie.export_item(i2) ie.finish_exporting() f.reset() self.assertEqual(pickle.load(f), i1) self.assertEqual(pickle.load(f), i2)
def test_export_multiple_items(self): i1 = TestItem(name='hello', age='world') i2 = TestItem(name='bye', age='world') f = StringIO() ie = PickleItemExporter(f) ie.start_exporting() ie.export_item(i1) ie.export_item(i2) ie.finish_exporting() f.reset() self.assertEqual(pickle.load(f), i1) self.assertEqual(pickle.load(f), i2)
class SocketExportPipeline(object): def __init__ (self, IP, Port): self.IP = IP self.Port = Port self.filepath = "/tmp/pickler.dat" self.socket = socket.socket() def spider_opened (self): self.file = open(self.filepath, "w") self.socket.connect((self.IP, self.Port)) self.pickler = PickleItemExporter(self.file) self.pickler.start_exporting() def process_item (self, item): if item: self.pickler.export_item(item) def spider_closed(): self.file.close() pickledfile = open(self.filepath, "r") for line in pickledfile: self.socket.send(line) self.socket.close() pickledfile.close()
class SocketExportPipeline(object): def __init__(self, IP, Port): self.IP = IP self.Port = Port self.filepath = "/tmp/pickler.dat" self.socket = socket.socket() def spider_opened(self): self.file = open(self.filepath, "w") self.socket.connect((self.IP, self.Port)) self.pickler = PickleItemExporter(self.file) self.pickler.start_exporting() def process_item(self, item): if item: self.pickler.export_item(item) def spider_closed(): self.file.close() pickledfile = open(self.filepath, "r") for line in pickledfile: self.socket.send(line) self.socket.close() pickledfile.close()
def _get_exporter(self, **kwargs): return PickleItemExporter(self.output, **kwargs)
def spider_opened (self): self.file = open(self.filepath, "w") self.socket.connect((self.IP, self.Port)) self.pickler = PickleItemExporter(self.file) self.pickler.start_exporting()
def spider_opened(self): self.file = open(self.filepath, "w") self.socket.connect((self.IP, self.Port)) self.pickler = PickleItemExporter(self.file) self.pickler.start_exporting()