def email_strip_html(html_content): """Strip html tags from html_content, trying to respect formatting.""" html_content = RE_SPACES.sub(' ', html_content) html_content = RE_NEWLINES.sub('\n', html_content) html_content = RE_HTML_TAGS.sub('', html_content) html_content = html_content.split('\n') out = StringIO() out_format = AbstractFormatter(DumbWriter(out)) for row in html_content: out_format.add_flowing_data(row) out_format.end_paragraph(1) return out.getvalue()
def parse_html(self, html): from StringIO import StringIO from formatter import (AbstractFormatter, DumbWriter) from htmllib import HTMLParser _html = re.sub(self.notrans_tag, r" \1 ", html) buf = StringIO() p = HTMLParser(AbstractFormatter(DumbWriter(buf))) p.feed(_html) _sub = re.sub(self.whitespaces, " ", buf.getvalue()) # FIXME: how can zerowidth be removed more simply? _sub = re.sub(self.zerowidth, "", _sub) _sub = re.sub(self.colon, r"\1", _sub) return _sub
def parseAndGetLinks(self, html_string): try: self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(html_string) self.parser.close() links = [] for eachLink in self.parser.anchorlist: if eachLink[:4] != "http" and find(eachLink, "://") == -1: eachLink = urljoin(self.base_url, eachLink) links.append(eachLink) return links except IOError: return []
def test(): import sys file = 'test.html' if sys.argv[1:]: file = sys.argv[1] fp = open(file, 'r') data = fp.read() fp.close() from formatter import DumbWriter, AbstractFormatter w = DumbWriter() f = AbstractFormatter(w) p = HTMLParser(f) p.feed(data) p.close()
def get_plain_from_html(html): """extract plain text from html >>> test_html = "<div><h1>Hey<h1><p>This is some text</p></div>" >>> get_plain_from_html(test_html) '\\nHey\\n\\nThis is some text' """ from htmllib import HTMLParser # import here to avoid high startup cost textout = StringIO() formtext = AbstractFormatter(DumbWriter(textout)) parser = HTMLParser(formtext) parser.feed(html) parser.close() return textout.getvalue()
def collectURLSFromPage(page): """ This returns a list of URLS that come from a certain page. Useful for spiders. It takes just a string as an argument. """ resultList = [] if page == "": #nothing to parse, so nothing to return return resultList #print "Doing form parser" if page.count("<form") > 0: otherlist = daveFormParse(page) for key in otherlist: resultList.append(key) pass #DEBUG #return resultList #print "Doing RAW Parser" spamList = rawParse(page) for key in spamList: resultList.append(key) pass #This needs to be documented somehow, but I have no idea what it does try: parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) parser.feed(page) parser.close() except: #print "DEBUG: Caught an exception trying to parse that html file." #print "(Not sure why this happens - you'll have to crawl this page manually)" return resultList #print "Adding HTML Parser data" for key in parser.anchorlist: resultList.append(key) pass return resultList
def parseAndGetLinks(self): # parse HTML, save links self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) r = self.download() if r: print '________' try: try: s = r.read(50000) except socket.error as e: print "***************************socket error***************************", e return [] self.parser.feed(s) print '------------------' r.close() print '***************************' except HTMLParseError: print 'get links error\n' return [] self.parser.close() return self.parser.anchorlist
def collectURLSFromPage(page): resultList = [] #print "Doing form parser" if page.count("<form") > 0: otherlist = daveFormParse(page) for key in otherlist: resultList.append(key) pass #DEBUG #return resultList #print "Doing RAW Parser" spamList = rawParse(page) for key in spamList: resultList.append(key) pass #the whole "AbstractFormater()" line is a bunch of crap I copied #That needs to be documented somehow, but I have no idea what it does try: parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) parser.feed(page) parser.close() except: #print "DEBUG: Caught an exception trying to parse that html file." #print "(Not sure why this happens - you'll have to crawl this page manually)" return resultList #print "Adding HTML Parser data" for key in parser.anchorlist: resultList.append(key) pass return resultList
def insert_read_only_node(c, p, name): if name == "": name = g.app.gui.runOpenFileDialog( c, title="Open", filetypes=[("All files", "*")], ) c.setHeadString(p, "@read-only %s" % name) c.redraw() parse = urlparse(name) try: if parse[0] == 'ftp': f = FTPurl(name) # FTP URL elif parse[0] == 'http': f = urlopen(name) # HTTP URL else: f = open(name, "r") # local file g.es("..." + name) new = f.read() f.close() except IOError: # as msg: # g.es("error reading %s: %s" % (name, msg)) # g.es("...not found: " + name) c.setBodyString(p, "") # Clear the body text. return True # Mark the node as changed. else: ext = os.path.splitext(parse[2])[1] if ext.lower() in ['.htm', '.html']: #@+<< convert HTML to text >> #@+node:edream.110203113231.895: *3* << convert HTML to text >> fh = StringIO() fmt = AbstractFormatter(DumbWriter(fh)) # the parser stores parsed data into fh (file-like handle) ### pylint: disable=too-many-function-args parser = HTMLParser(fmt) # send the HTML text to the parser parser.feed(new) parser.close() # now replace the old string with the parsed text new = fh.getvalue() fh.close() # finally, get the list of hyperlinks and append to the end of the text ### pylint: disable=no-member hyperlinks = parser.anchorlist numlinks = len(hyperlinks) if numlinks > 0: hyperlist = ['\n\n--Hyperlink list follows--'] for i in range(numlinks): hyperlist.append("\n[%d]: %s" % (i + 1, hyperlinks[i])) # 3/26/03: was i. new = new + ''.join(hyperlist) #@-<< convert HTML to text >> previous = p.b c.setBodyString(p, new) changed = (g.toUnicode(new) != g.toUnicode(previous)) if changed and previous != "": g.es("changed: %s" % name) # A real change. return changed
def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(urlopen(self.url).read()) self.parser.close() return self.parser.anchorlist
def html2text(html): output = StringIO() writer = DumbWriter(output) p = HTMLParser(AbstractFormatter(writer)) p.feed(toText(html)) return toText(output.getvalue())
def __init__(self, gc): Coloring.__init__(self, gc, "help") self.heading = self.printer("heading", attr="bold") self.wrap = AbstractFormatter(DumbWriter())
def parseAndGetLinks(self): # parse HTML, save links self.parser = HTMLParser(AbstractFormatter(\ DumbWriter(StringIO())))
def __init__(self, name, data): f = AbstractFormatter(DumbWriter(open(name, 'w'), 100)) HTMLParser.__init__(self, f) self.feed(data) self.close()
def parseAndGetLinks(self):#分析页面获得url self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist #锚链接列表
def __init__(self, gc): Coloring.__init__(self, gc, 'help') self.heading = self.printer('heading', attr='bold') self.wrap = AbstractFormatter(DumbWriter())
def parseAndGetLinks(self): # parse HTML, save links self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
#!/usr/bin/env python # Zastosuj ten plik wraz z plikiem "naglowki.html" from formatter import AbstractFormatter, DumbWriter from htmllib import HTMLParser class HeadingParser(HTMLParser): def start_h1(self, tag): print "Znalaz³em H1" writer = DumbWriter() formatter = AbstractFormatter(writer) parser = HeadingParser(formatter) parser.feed(open('naglownik.html').read()) parser.close() print "Koniec analizy"