def distillTxt(rstream, wstream, meta): """ Similar interface to distill() for text/plain media type @params rstream - the input stream @params wstream - the output stream (meta data + content of rstream, utf8 encoded) @params meta - meta data like uri and timestamp is supplied by caller. No title or description defined for plain text. @returns - 0 means accepted. Otherwise a tuple of reason code and an explanation string. """ first_block = rstream.read(8192) rstream.seek(0) result = preparse_filter(first_block, meta) if result: return result encoding, source = encode_tools.determineEncodingLenient(meta, '') Reader = encode_tools.getreader(encoding, source) reader = Reader(rstream, 'replace') writer = codecs.getwriter('utf8')(wstream,'replace') meta['encoding'] = '%s [%s]' % (encoding, source) writeHeader(writer, meta) shutil.copyfileobj(reader, writer) return 0
def distill(rstream, wstream, meta): """ Parse the HTML doc rstream. Determine its character encoding. Return buf and fill in meta. Use heuristic to determine if it should be indexed. Return False if not. @params rstream - the input stream @params wstream - the distilled output stream (utf8 encoded) @params meta - Add the title, description and keywords fields while parsing. Also add encoding (for diagnosis) Other meta data like uri and timestamp is supplied by caller. @returns - 0 means accepted. Otherwise a tuple of reason code and an explanation string. """ first_block = rstream.read(32768) rstream.seek(0) # network stream would not support seek!? result = preparse_filter(first_block, meta) if result: return result encoding, source = encode_tools.determineEncodingLenient(meta, first_block) Reader = encode_tools.getreader(encoding, source) reader = Reader(rstream, 'replace') writer = codecs.getwriter('utf8')(wstream,'replace') meta['encoding'] = '%s [%s]' % (encoding, source) formatter = Formatter(writer) try: has_html, has_frameset, has_common_tag = process(reader, formatter, meta) except sgmllib.SGMLParseError, e: return (PARSE_ERROR, 'SGMLParseError: %s' % str(e)) # SGMLParseError
def test_getreader(self): Reader = encode_tools.getreader('latin-1') fp = StringIO.StringIO('mam\xe1') reader = Reader(fp) udata = reader.read() expect = u'mam\xe1' self.assertEqual(udata, expect) self.assertEqual(type(udata), type(expect))
def test_getreader_invalid(self): try: Reader = encode_tools.getreader('bad') except UnicodeError, e: self.assert_(e.args[0].find('bad') > 0)