def distill(rstream, wstream, meta):
""" Parse the HTML doc rstream. Determine its character encoding.
Return buf and fill in meta.
Use heuristic to determine if it should be indexed.
Return False if not.
@params rstream - the input stream
@params wstream - the distilled output stream (utf8 encoded)
@params meta - Add the title, description and keywords fields while parsing.
Also add encoding (for diagnosis)
Other meta data like uri and timestamp is supplied by caller.
@returns - 0 means accepted. Otherwise a tuple of reason code and an explanation string.
"""
first_block = rstream.read(32768)
rstream.seek(0) # network stream would not support seek!?
result = preparse_filter(first_block, meta)
if result:
return result
encoding, source = encode_tools.determineEncodingLenient(meta, first_block)
Reader = encode_tools.getreader(encoding, source)
reader = Reader(rstream, 'replace')
writer = codecs.getwriter('utf8')(wstream,'replace')
meta['encoding'] = '%s [%s]' % (encoding, source)
formatter = Formatter(writer)
try:
has_html, has_frameset, has_common_tag = process(reader, formatter, meta)
except sgmllib.SGMLParseError, e:
return (PARSE_ERROR, 'SGMLParseError: %s' % str(e)) # SGMLParseError
def distillTxt(rstream, wstream, meta):
""" Similar interface to distill() for text/plain media type
@params rstream - the input stream
@params wstream - the output stream (meta data + content of rstream, utf8 encoded)
@params meta - meta data like uri and timestamp is supplied by caller.
No title or description defined for plain text.
@returns - 0 means accepted. Otherwise a tuple of reason code and an explanation string.
"""
first_block = rstream.read(8192)
rstream.seek(0)
result = preparse_filter(first_block, meta)
if result:
return result
encoding, source = encode_tools.determineEncodingLenient(meta, '')
Reader = encode_tools.getreader(encoding, source)
reader = Reader(rstream, 'replace')
writer = codecs.getwriter('utf8')(wstream,'replace')
meta['encoding'] = '%s [%s]' % (encoding, source)
writeHeader(writer, meta)
shutil.copyfileobj(reader, writer)
return 0
def test_determine_lenient(self):
result = encode_tools.determineEncoding(
{'content-type': 'text/html; charset=bad' },
'')
self.assertEqual(('bad',encode_tools.HTTP_CONTENT_TYPE), result) # controlled test
result = encode_tools.determineEncodingLenient(
{'content-type': 'text/html; charset=bad' },
'')
self.assertEqual(('iso-8859-1',encode_tools.DEFAULT), result) # return default instead of bad