def distillTxt(rstream, wstream, meta):
""" Similar interface to distill() for text/plain media type
@params rstream - the input stream
@params wstream - the output stream (meta data + content of rstream, utf8 encoded)
@params meta - meta data like uri and timestamp is supplied by caller.
No title or description defined for plain text.
@returns - 0 means accepted. Otherwise a tuple of reason code and an explanation string.
"""
first_block = rstream.read(8192)
rstream.seek(0)
result = preparse_filter(first_block, meta)
if result:
return result
encoding, source = encode_tools.determineEncodingLenient(meta, '')
Reader = encode_tools.getreader(encoding, source)
reader = Reader(rstream, 'replace')
writer = codecs.getwriter('utf8')(wstream,'replace')
meta['encoding'] = '%s [%s]' % (encoding, source)
writeHeader(writer, meta)
shutil.copyfileobj(reader, writer)
return 0
def distill(rstream, wstream, meta):
""" Parse the HTML doc rstream. Determine its character encoding.
Return buf and fill in meta.
Use heuristic to determine if it should be indexed.
Return False if not.
@params rstream - the input stream
@params wstream - the distilled output stream (utf8 encoded)
@params meta - Add the title, description and keywords fields while parsing.
Also add encoding (for diagnosis)
Other meta data like uri and timestamp is supplied by caller.
@returns - 0 means accepted. Otherwise a tuple of reason code and an explanation string.
"""
first_block = rstream.read(32768)
rstream.seek(0) # network stream would not support seek!?
result = preparse_filter(first_block, meta)
if result:
return result
encoding, source = encode_tools.determineEncodingLenient(meta, first_block)
Reader = encode_tools.getreader(encoding, source)
reader = Reader(rstream, 'replace')
writer = codecs.getwriter('utf8')(wstream,'replace')
meta['encoding'] = '%s [%s]' % (encoding, source)
formatter = Formatter(writer)
try:
has_html, has_frameset, has_common_tag = process(reader, formatter, meta)
except sgmllib.SGMLParseError, e:
return (PARSE_ERROR, 'SGMLParseError: %s' % str(e)) # SGMLParseError
def test_getreader(self):
Reader = encode_tools.getreader('latin-1')
fp = StringIO.StringIO('mam\xe1')
reader = Reader(fp)
udata = reader.read()
expect = u'mam\xe1'
self.assertEqual(udata, expect)
self.assertEqual(type(udata), type(expect))
def test_getreader_invalid(self):
try:
Reader = encode_tools.getreader('bad')
except UnicodeError, e:
self.assert_(e.args[0].find('bad') > 0)
