def process_module(self, stream):
"""extracts encoding from the stream and decodes each line, so that
international text's length is properly calculated.
"""
line_reader = stream.readline
if sys.version_info < (3, 0):
data = stream.read()
if not is_ascii(data)[0]:
encoding = guess_encoding(data)
if encoding is not None:
line_reader = lambda: stream.readline().decode(encoding,
'replace')
del data
stream.seek(0)
self.process_tokens(tokenize.generate_tokens(line_reader))
def process_module(self, stream):
"""extracts encoding from the stream and
decodes each line, so that international
text's lenght properly calculated.
"""
data = stream.read()
line_generator = stream.readline
ascii, lineno = is_ascii(data)
if not ascii:
encoding = guess_encoding(data)
if encoding is not None:
line_generator = lambda: stream.readline().decode(encoding,
'replace')
del data
stream.seek(0)
self.process_tokens(tokenize.generate_tokens(line_generator))
