def _process_result(self, crawler, items): for item in items: # proccess item as data item if isinstance(item, Item): # process item by pipes for pipe in self.pipelines: item = pipe.process(crawler, item) # item filtered - stop pipe processing if not item: break # yield item as request if isinstance(item, BaseHttpRequest) or isstring(item): yield item
def start(self, crawler): if isstring(self.output_file): if PY3: self.csvfile = codecs.open( self.output_file, 'w', encoding='utf-8' ) else: self.csvfile = open(self.output_file, 'w') self._need_close = True else: self.csvfile = self.output_file if PY3: self.writer = csv.writer( self.csvfile, *self._csv_args, **self._csv_kwargs ) else: self.writer = UnicodeCsvWriter( self.csvfile, *self._csv_args, **self._csv_kwargs )
def _encode_item(self, data): if isstring(data): return data.encode('utf-8') return data