def _process_result(self, crawler, items):
for item in items:
# proccess item as data item
if isinstance(item, Item):
# process item by pipes
for pipe in self.pipelines:
item = pipe.process(crawler, item)
# item filtered - stop pipe processing
if not item:
break
# yield item as request
if isinstance(item, BaseHttpRequest) or isstring(item):
yield item
def start(self, crawler):
if isstring(self.output_file):
if PY3:
self.csvfile = codecs.open(
self.output_file, 'w', encoding='utf-8'
)
else:
self.csvfile = open(self.output_file, 'w')
self._need_close = True
else:
self.csvfile = self.output_file
if PY3:
self.writer = csv.writer(
self.csvfile, *self._csv_args, **self._csv_kwargs
)
else:
self.writer = UnicodeCsvWriter(
self.csvfile, *self._csv_args, **self._csv_kwargs
)
def _encode_item(self, data):
if isstring(data):
return data.encode('utf-8')
return data
