class Retriever(object): def __init__(self,url): self.url=url self.file=self.filename(url) def filename(self,url,deffile='index.html'): parsedurl=urlparse(url,'http:',0) path=parsedurl[1]+parsedurl[2] #weibo.com+/gothack ext=splitext(path) #weibo.com/gothack , '' #split by . if ext[1]=='': if path[-1]=='/': path+=deffile else: path+='/'+deffile ldir=dirname(path) #weibo.com #before the last / if sep != '/': #default value is / ldir=replce(ldir,'/',sep) #replace / with sep #(string,old,new) if not isdir(ldir): if exists(ldir):unlink(ldir) makedirs(ldir) return path def download(self): try: retval=urlretrieve(self.url,self.file) except IOError: retval=('***ERROR: invalid URL "%s"' %self.url,) return retval def parseAndGetLinks(self): self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def create_plaintext_message(self, text): """ Create a plain-text-message by parsing the html and attaching links as endnotes Modified from EasyNewsletter/content/ENLIssue.py """ # This reflows text which we don't want, but it creates # parser.anchorlist which we do want. textout = StringIO.StringIO() formtext = formatter.AbstractFormatter(formatter.DumbWriter(textout)) parser = HTMLParser(formtext) parser.feed(text) parser.close() # append the anchorlist at the bottom of a message # to keep the message readable. counter = 0 anchorlist = "\n\n" + '----' + "\n\n" for item in parser.anchorlist: counter += 1 anchorlist += "[%d] %s\n" % (counter, item) # This reflows text: # text = textout.getvalue() + anchorlist # This just strips tags, no reflow text = html.fromstring(text).text_content() text += anchorlist del textout, formtext, parser, anchorlist return text
class Retriever(object): # download Web pages def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile='index.htm'): parsedurl = urlparse(url, 'http:', 0) path = parsedurl[1] + parsedurl[2] ext = splitext(path) if ext[1] == '': # no file, use default if path[-1] == '/': path += deffile else: path += '/' + deffile ldir = dirname(path) # local directory if sep != '/': # os-indep. path separator ldir = replace(ldir, '/', sep) if not isdir(ldir): # create archive dir if nec. if exists(ldir): unlink(ldir) makedirs(ldir) return path def download(self): # download Web page try: retval = urlretrieve(self.url, self.file) except IOError: retval = ('*** ERROR: invalid URL "%s"' % self.url, ) return retval def parseAndGetLinks(self): # parse HTML, save links self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
class Retriever(object): #下载网页类 def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile="index.htm"): parsedurl = urlparse(url, "http:", 0) #解析路径 path = parsedurl[1] + parsedurl[2] ext = splitext(path) if ext[1] == "": #如果没有文件,使用默认 if path[-1] == "/": path += deffile else: path += "/" + deffile ldir = dirname(path) #本地目录 if sep != "/": ldir = replace(ldir, "/", sep) if not isdir(ldir): #如果没有目录,创建一个 if exists(ldir): unlink(ldir) makedirs(ldir) return path def download(self): # 下载网页 try: retval = urlretrieve(self.url, self.file) except IOError: retval = ('***Error: invalid URL: "%s"' % self.url, ) return retval def parseAndGetLinks(self): #解析HTML,保存链接 self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def __init__(self, funFormatter, objHere): self.objH1 = None self.objH2 = None self.objH3 = None self.dodHelp = GetDOD(objHere, "E3Help") HTMLParser.__init__(self, funFormatter)
class Retrive(object): def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile='index.php'): parsedurl = urlparse(url,'http:', 0) path = parsedurl[1] + parsedurl[2] ext = splitext(path) if ext[1] == '': if path[-1] == '/': path += deffile else: path += '/' + deffile ldir = dirname(path) if sep != '/': ldir = replace(ldir, '/', sep) if not isdir(ldir): if exists(ldir): unlink(ldir) makedirs(ldir) return path def download(self): try: retval = urlretrieve(self.url, self.file) except IOError: retval = 'error' return retval def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter): HTMLParser.__init__(self, fmt) self.result = "" self.open_tags = [] # A list of the only tags allowed. Be careful adding to this. Adding # "script," for example, would not be smart. 'img' is out by default # because of the danger of IMG embedded commands, and/or web bugs. self.permitted_tags = ['a', 'b', 'blockquote', 'br', 'i', 'li', 'ol', 'ul', 'p', 'cite'] # A list of tags that require no closing tag. self.requires_no_close = ['img', 'br'] # A dictionary showing the only attributes allowed for particular tags. # If a tag is not listed here, it is allowed no attributes. Adding # "on" tags, like "onhover," would not be smart. Also be very careful # of "background" and "style." self.allowed_attributes = \ {'a':['href', 'title'], 'img':['src', 'alt'], 'blockquote':['type']} # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ['http', 'https', 'ftp']
def __init__( self, permitted_tags=['a', 'b', 'blockquote', 'br/', 'i', 'li', 'ol', 'ul', 'p', 'cite', 'code', 'pre', 'img/',], allowed_attributes={ 'a': ['href', 'title'], 'img': ['src', 'alt'], 'blockquote': ['type']}, fmt=AbstractFormatter, strip_disallowed=False ): HTMLParser.__init__(self, fmt) self.result = '' self.open_tags = [] self.permitted_tags = [i for i in permitted_tags if i[-1] != '/'] self.requires_no_close = [i[:-1] for i in permitted_tags if i[-1] == '/'] self.permitted_tags += self.requires_no_close self.allowed_attributes = allowed_attributes # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ['http', 'https', 'ftp'] #to strip or escape disallowed tags? self.strip_disallowed = strip_disallowed self.in_disallowed = False
def __init__(self, fmt = AbstractFormatter): HTMLParser.__init__(self, fmt) self.result = "" self.open_tags = [] # A list of forbidden tags. self.forbidden_tags = ['script', 'embed', 'iframe', 'frame' ] # A list of tags that require no closing tag. self.requires_no_close = ['img', 'br'] # A dictionary showing the only attributes allowed for particular tags. # If a tag is not listed here, it is allowed no attributes. Adding # "on" tags, like "onhover," would not be smart. Also be very careful # of "background" and "style." self.allowed_attributes =\ {'a':['href','title','target','style'], 'img':['src','alt','border','style'], 'blockquote':['type','style'], 'table': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'tbody': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'tr': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'td': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'div': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'span': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], } # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ['http','https','ftp']
def parseAndGetLinks(self): """StringIO是从内存中读取数据 DumbWriter将事件流转换为存文本文档 AbstractFormatter 类进行格式化 """ self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def parseAndGetLinks(self): # 创建一个基本的HTML解释器,可能需要单独一篇文章来说这句 self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) # 解析html文件,获取所有的连接(带有href的) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def get_plain_from_html(html): textout = StringIO() formtext = AbstractFormatter(DumbWriter(textout)) parser = HTMLParser(formtext) parser.feed(html) parser.close() return textout.getvalue()
def create_plaintext_message(self, text): """ Create a plain-text-message by parsing the html and attaching links as endnotes """ plain_text_maxcols = 72 textout = cStringIO.StringIO() formtext = formatter.AbstractFormatter(formatter.DumbWriter( textout, plain_text_maxcols)) parser = HTMLParser(formtext) parser.feed(text) parser.close() # append the anchorlist at the bottom of a message # to keep the message readable. counter = 0 anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n" for item in parser.anchorlist: counter += 1 if item.startswith('https://'): new_item = item.replace('https://', 'http://') else: new_item = item anchorlist += "[%d] %s\n" % (counter, new_item) text = textout.getvalue() + anchorlist del textout, formtext, parser, anchorlist return text
class Retriever(object): # download Web pages def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile='index.htm'): parsedurl = urlparse(url, 'http:', 0) ## parse path path = parsedurl[1] + parsedurl[2] ext = splitext(path) if ext[1] == '': # no file, use default if path[-1] == '/': path += deffile else: path += '/' + deffile ldir = dirname(path) # local directory if sep != '/': # os-indep. path separator ldir = replace(ldir, '/', sep) if not isdir(ldir): # create archive dir if nec. if exists(ldir): unlink(ldir) makedirs(ldir) return path def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def create_plaintext_message(message): """ Create clean plain text version of email message Parse the html and remove style and javacript tags and then create a plain-text-message by parsing the html and attaching links as endnotes """ cleaner = Cleaner() cleaner.javascript = True cleaner.style = True cleaner.kill_tags = ['style'] doc = message.decode('utf-8', 'ignore') to_clean = lxml.html.fromstring(doc) cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean)) plain_text_maxcols = 72 textout = cStringIO.StringIO() formtext = formatter.AbstractFormatter(formatter.DumbWriter( textout, plain_text_maxcols)) parser = HTMLParser(formtext) parser.feed(cleaned_msg) parser.close() # append the anchorlist at the bottom of a message # to keep the message readable. counter = 0 anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n" for item in parser.anchorlist: counter += 1 if item.startswith('https://'): new_item = item.replace('https://', 'http://') else: new_item = item anchorlist += "[%d] %s\n" % (counter, new_item) text = textout.getvalue() + anchorlist del textout, formtext, parser, anchorlist return text
class Retriever(object):#下载网页类 def __init__(self,url): self.url = url self.file = self.filename(url) def filename(self,url,deffile ="index.htm"): parsedurl = urlparse(url,"http:",0) #解析路径 path = parsedurl[1] + parsedurl[2] ext = splitext(path) if ext[1] == "": #如果没有文件,使用默认 if path[-1] == "/": path += deffile else: path += "/" + deffile ldir = dirname(path) #本地目录 if sep != "/": ldir = replace(ldir,"/",sep) if not isdir(ldir): #如果没有目录,创建一个 if exists(ldir):unlink(ldir) makedirs(ldir) return path def download(self):# 下载网页 try: retval = urlretrieve(self.url,self.file) except IOError: retval = ('***Error: invalid URL: "%s"' % self.url,) return retval def parseAndGetLinks(self): #解析HTML,保存链接 self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def OpenURL(self,url): from htmllib import HTMLParser import formatter self.url = url m = re.match('http://([^/]+)(/\S*)\s*', url) if m: host = m.groups()[0] path = m.groups()[1] else: m = re.match('http://(\S+)\s*', url) if not m: # Invalid URL self.logprint("Invalid or unsupported URL: %s" % (url)) return host = m.groups()[0] path = '' f = self.RetrieveAsFile(host,path) if not f: self.logprint("Could not open %s" % (url)) return self.logprint("Receiving data...") data = f.read() tmp = open('hangman_dict.txt','w') fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp)) p = HTMLParser(fmt) self.logprint("Parsing data...") p.feed(data) p.close() tmp.close()
class Retriever(object): # download Web pages def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile = 'index.htm'): parsedurl = urlparse(url, 'http:', 0) # parse path path = parsedurl[1] + parsedurl[2] ext = splitext(path) if ext[1] == '': # no file, use default if path[-1] == '/': path += deffile else: path += '/' + deffile ldir = dirname(path) # local directory if sep != '/': ldir = replace(ldir, '/', sep) if not isdir(ldir): if exists(ldir): unlink(ldir) makedirs(ldir) return path def download(self): try: retval = urlretrieve(self.url, self.file) except IOError: retval = ('*** ERROR: invalid URL "%s"' % self.url) return retval def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def parseAndGetLinks(self): # pars HTML, save links self.parser = HTMLParser(AbstractFormatter( \ DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() print self.parser return self.parser.anchorlist
class Restriever(object): def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile = 'index.htm'): parsedurl = urlparse(url, 'http:', 0) path = parsedurl[1] + parsedurl[2] ext = splitext(path) if ext[1] == '': if path[-1] == '/': path += deffile else: path += '/' + deffile ldir = dirname(path) if sep != '/': ldir = replace(ldir, '/', sep) if not isdir(ldir): if exists(ldir) : unlink(ldir) makedirs(ldir) return path def download(self): try: retval = urlretrieve(self.url, self.file) except IOError: retval = ('*** ERROR: invalid URL "%s"' % self.url) return retval def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter): HTMLParser.__init__(self, fmt) self.result = "" self.open_tags = [] # A list of forbidden tags. self.forbidden_tags = ["script", "embed", "iframe", "frame"] # A list of tags that require no closing tag. self.requires_no_close = ["img", "br"] # A dictionary showing the only attributes allowed for particular tags. # If a tag is not listed here, it is allowed no attributes. Adding # "on" tags, like "onhover," would not be smart. Also be very careful # of "background" and "style." self.allowed_attributes = { "a": ["href", "title", "target", "style"], "img": ["src", "alt", "border", "style"], "blockquote": ["type", "style"], "font": ["size", "face"], "h5": ["style"], "h4": ["style"], "h3": ["style"], "h2": ["style"], "h1": ["style"], "table": ["border", "width", "height", "style", "align", "bgcolor"], "tbody": ["border", "width", "height", "style", "align", "bgcolor"], "tr": ["border", "width", "height", "style", "align", "bgcolor"], "td": ["border", "width", "height", "style", "align", "bgcolor"], "div": ["border", "width", "height", "style", "align", "bgcolor"], "span": ["border", "width", "height", "style", "align", "bgcolor"], } # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ["http", "https", "ftp"]
def reset(self): self._data_is_title = False self._data_is_comment = False self.output = None self._current = None self._container = [] HTMLParser.reset(self)
class Retriever(object): def __init__(self,url): self.url=url self.file=self.filename(url) def filename(self,url,deffile='index.html'): """ 生成下载连接和文件名 """ full_url = "" if url.endswith(DOM_SUFFIX): full_url = url + '/' else: full_url = url parsedurl=urlparse(full_url,'http:',0) path=parsedurl[1]+parsedurl[2] ext=splitext(path) if ext[1]=='': if path[-1]=='/': path+=deffile else: path+='/'+deffile ldir=dirname(path) if sep!='/': ldir=replace(ldir,'/',sep) path=replace(path,'/',sep) if not isdir(ldir): if exists(ldir): #unlink(ldir) pass else: makedirs(ldir) print path return path def download(self): """ 下载文件 """ try: retval=urlretrieve(self.url,self.file) except IOError: retval=('***ERROR :invalid URL "%s"' %self.url,) return retval def parseAndGetLinks(self): """ 获取网页中的链接 """ #print 'Get Html Links from file:%s' % self.file #self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO))) self.parser=HTMLParser(NullFormatter()) #self.parser.feed(open(self.file).read()) try: self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist except: print self.file + " error !" return []
def __init__( self, permitted_tags=[ 'a', 'b', 'blockquote', 'br/', 'i', 'li', 'ol', 'ul', 'p', 'cite', 'code', 'pre', 'img/', ], allowed_attributes={'a': ['href', 'title'], 'img': ['src', 'alt' ], 'blockquote': ['type']}, fmt=AbstractFormatter, ): HTMLParser.__init__(self, fmt) self.result = '' self.open_tags = [] self.permitted_tags = [i for i in permitted_tags if i[-1] != '/'] self.requires_no_close = [i[:-1] for i in permitted_tags if i[-1] == '/'] self.permitted_tags += self.requires_no_close self.allowed_attributes = allowed_attributes # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ['http', 'https', 'ftp']
class Retriever(htmllib.HTMLParser): #download Web Pages def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile='index.htm'): parsedurl = urlparse(url,'http:',0) #parse path path = parsedurl[1] + parsedurl[2] text = splitext(path) if text[1]=='': #its not file use default if text[-1] == '/': path = path + deffile else: path = path + '/' + deffile print "PATH:%s" % path dir = dirname(path) if not isdir(dir): #create new archieve dir if necessary if exists(dir): unlink(dir) makedirs(dir) return path def download(self): #download web pages try: retval = urlretrieve(self.url,self.file) except IOError: retval =('***ERROR: invalid URL "%s"' % self.url) return retval def parseAndGetLinks(self): #Parse HTML self.parser = HTMLParser(AbstractFormatter(\ DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter): HTMLParser.__init__(self, fmt) self.result = "" self.open_tags = [] # A list of the only tags allowed. Be careful adding to this. Adding # "script," for example, would not be smart. 'img' is out by default # because of the danger of IMG embedded commands, and/or web bugs. self.permitted_tags = [ 'a', 'b', 'blockquote', 'br', 'i', 'li', 'ol', 'ul', 'p', 'cite' ] # A list of tags that require no closing tag. self.requires_no_close = ['img', 'br'] # A dictionary showing the only attributes allowed for particular tags. # If a tag is not listed here, it is allowed no attributes. Adding # "on" tags, like "onhover," would not be smart. Also be very careful # of "background" and "style." self.allowed_attributes = \ {'a':['href','title'], 'img':['src','alt'], 'blockquote':['type']} # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ['http', 'https', 'ftp']
def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) try: self.parser.feed(open(self.file).read()) self.parser.close() except IOError: pass return self.parser.anchorlist
def html2text(htmldata): # patch htmldata htmldata = htmldata.replace("<br/>", "<br>") fmt = HTMLtoTextFormatter() prs = HTMLParser(fmt) prs.feed(htmldata) return fmt.getText()
def __init__(self, formatter, path, output): HTMLParser.__init__(self, formatter) self.path = path # relative path self.ft = output # output file self.indent = 0 # number of tabs for pretty printing of files self.proc = False # True when actively processing, else False # (headers, footers, etc) # XXX This shouldn't need to be a stack -- anchors shouldn't nest. # XXX See SF bug <http://www.python.org/sf/546579>. self.hrefstack = [] # stack of hrefs from anchor begins
class Retriever(object): def __init__(self,url): self.url = url #parse HTML ,save links def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(urlopen(self.url).read()) self.parser.close() return self.parser.anchorlist
def parse_links(self): """fetch all links from page """ f = open(self.save_file, 'r') data = f.read() f.close() parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO()))) parser.feed(data) parser.close() return parser.anchorlist
def parse_links(self): 'Parse out the links found in downloaded HTML file' f = open(self.file, 'r') data = f.read() f.close() parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter( cStringIO.StringIO()))) parser.feed(data) parser.close() return parser.anchorlist
def __init__(self, formatter=AbstractFormatter(DumbWriter())): HTMLParser.__init__(self,formatter) self.intoTheBox=False #ingresso/uscita dall'ambiente del box self.intoTheTitle=None #ingresso/uscita dalla riga titolo self.intoTheNews=None #ingresso/uscita dal testo della notizia self.effectiveTitle=False #tag <span> del titolo self.effectiveDate=False #tag <span> della data self.effectiveAuth=False #tag <div> dell'autore self.writable=False #testo scrivibile self.cookedText="" #testo xml risultante
def parse_links(self): """ Parse out the links found is downloaded HTML file""" f = open(self.file, "r") data = f.read() f.close() parser = HTMLParser( formatter.AbstractFormatter( formatter.DumbWriter(cStringIO.StringIO()))) parser.feed(data) return parser.anchorlist
def parseAndGetLinks(self): '''解析html页面,获取页面中的链接,并保存链接''' self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) #使用HTMLParser的方法进行处理 , StringIO是从内存中读取数据,DumbWriter将事件流转换为存文本文档。 self.parser.feed(open(self.file).read()) #将self.file文件打开,并一次性读入上面定的文件中 self.parser.close() print 'self.parser.anchorlist --> ', self.parser.anchorlist return self.parser.anchorlist #anchorlist 记录href 地址
def parse_links(self): f = open(self.file, "r") data = f.read() f.close() parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO()))) # parser = MyHTMLParser() parser.feed(data) parser.close() # 没有在模块中找到 anchorlist属性 # 返回页面中所有的锚点,href # 该属性在2.6后就被弃用了,如果想用可以通过自定义parser继承HTMLParser来实现,参见evernote.或者用第三方库beautifulsoup return parser.anchorlist
def parseAndGetLinks(self, html_string): try: self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(html_string) self.parser.close() links = [] for eachLink in self.parser.anchorlist: if eachLink[:4] != "http" and find(eachLink, "://") == -1: eachLink = urljoin(self.base_url, eachLink) links.append(eachLink) return links except IOError: return []
def parse_html(self, html): from StringIO import StringIO from formatter import (AbstractFormatter, DumbWriter) from htmllib import HTMLParser _html = re.sub(self.notrans_tag, r" \1 ", html) buf = StringIO() p = HTMLParser(AbstractFormatter(DumbWriter(buf))) p.feed(_html) _sub = re.sub(self.whitespaces, " ", buf.getvalue()) # FIXME: how can zerowidth be removed more simply? _sub = re.sub(self.zerowidth, "", _sub) _sub = re.sub(self.colon, r"\1", _sub) return _sub
def __init__(self, formatter=AbstractFormatter(DumbWriter())): #HTMLParser.__init__(self) HTMLParser.__init__(self,formatter) self.intoTheAnchor=False #ingresso/uscita dai tag anchor self.intoTheDate=False #ingresso/uscita dal box data self.intoTheTitle=False #ingresso/uscita dal box titolo self.intoTheNews=False #ingresso/uscita dal box notizia self.writable=False #testo scrivibile self.maxNews=10 #totale notizie visualizzate self.cookedText="" #testo xml risultante self.gotTitle=False #flag indicante la reale presenza del titolo self.pseudoTitle="" #titolo fake (se mancante)
def parse(self, url): self.base = "" self.href = "" m = re.compile(".*/").match(url) if m != None: self.base = m.string[m.start(0):m.end(0)] result = urlfetch.fetch(url, headers = {'Cache-Control' : 'max-age=30', 'Pragma' : 'no-cache'} ) if result.status_code == 200: logging.debug(str(result.status_code) + " OK " + url) HTMLParser.feed(self, result.content) HTMLParser.close(self) else: logging.error(str(result.status_code) + " NG " + url)
class Retriever(object):#download web page def __init__(self, url): self.url = url self.file = self.filename(url) #local filename ,directory def filename(self, url, deffile = "index.htm"): parseurl = urlparse(url, 'http:', 0) path = parseurl[1] + parseurl[2] #将路径转换为一个元组,如果为目录则第二个元素为空,如果文件则第二个元素为文件扩展名 #path = "D:/pycharmProjects/PythonWebApp/weblearning/Crawl.py" #print splitext(path) #('D:/pycharmProjects/PythonWebApp/weblearning/Crawl', '.py') ext = splitext(path) if ext[1] == '':#no file use default #tuple[-index],倒数第index个元素 if path[-1] == '/': path += deffile else: path += '/' + deffile #获取path的dir, # path=D:/pycharmProjects/PythonWebApp/webLearning # dir(path)=D:/pycharmProjects/PythonWebApp ldir = dirname(path)#local directory if sep != '/': #os-indep. path separator ldir = replace(ldir, '/', sep) if not isdir(ldir): #create archive dir if nec. #如果存在文件,则删除 if exists(ldir): unlink(ldir) makedirs(ldir) return path def download(self): #urlretrieve()返回一个2元组,(filename,mime_hdrs) try: retval = urlretrieve(self.url, self.file) except IOError: retval = ('*** ERROR: invalid URL "%s"' %self.url,) print 'erro,invalid url' return retval def parseAndGetLinks(self):#parse HTML , save links self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
class Retriever(object): def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url, deffile='index.htm'): # 使用http协议解析url地址,将url解析为这样的五元组:(scheme, netloc, path, query, fragment) parsedurl = urlparse(url, 'http:', 0) # 将主机地址(netloc)和路径(path)合并起来作为存储文件的路径名 path = parsedurl[1] + parsedurl[2] # splitext将path分割为路径与后缀名,如(/path/to/file, txt) ext = splitext(path) # 如果后缀名为空,则给当前path添加默认的名字index.htm if ext[1] == '': # 若path结尾有“/”则直接添加index.htm,否则先添加“/” if path[-1] == '/': path += deffile else: path += '/' + deffile # 取出path中的目录部分(www.shellbye.com\\blog),然后与本地目录结合为一个目录('D:\\www.shellbye.com\\blog') ldir = dirname(abspath(path)) # 如果不是以“/”作为目录分割符的类Unix系统,比如Windows,则需要把“”替换为相应的目录分割符 # 因为类Unix系统的目录分割符与URI地址的分割符一样,所以可以不处理 if sep != '/': # os-indep. path separator ldir = replace(ldir, '/', sep) # 如果ldir目录不存在在创建 if not isdir(ldir): # create archive dir if nec. # 如果ldir存在但是不是目录则删除ldir。注,unlink即remove。 if exists(ldir): unlink(ldir) makedirs(ldir) return path def download(self): # download Web page try: # 下载self.url到self.file里 retval = urllib.urlretrieve(self.url, self.file) except IOError: retval = ('*** ERROR: invalid URL "%s"' % self.url, ) return retval def parseAndGetLinks(self): # 创建一个基本的HTML解释器,可能需要单独一篇文章来说这句 self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) # 解析html文件,获取所有的连接(带有href的) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter): HTMLParser.__init__(self, fmt) self.result = "" self.open_tags = [] # A list of the only tags allowed. Be careful adding to this. Adding # "script," for example, would not be smart. 'img' is out by default # because of the danger of IMG embedded commands, and/or web bugs. self.permitted_tags = [ "a", "b", "blockquote", "br", "i", "sup", "sub", "strike", "hr", "u", "li", "ol", "ul", "p", "cite", "img", "style", "font", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "div", ] # A list of tags that require no closing tag. self.requires_no_close = ["img", "br"] # A dictionary showing the only attributes allowed for particular tags. # If a tag is not listed here, it is allowed no attributes. Adding # "on" tags, like "onhover," would not be smart. Also be very careful # of "background" and "style." self.allowed_attributes = {"a": ["href", "title"], "img": ["src", "alt"], "blockquote": ["type"]} # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ["http", "https", "ftp"]
def parse(self, url): self.base = "" self.href = "" m = re.compile(".*/").match(url) if m != None: self.base = m.string[m.start(0):m.end(0)] result = urlfetch.fetch(url, headers={ 'Cache-Control': 'max-age=30', 'Pragma': 'no-cache' }) if result.status_code == 200: logging.debug(str(result.status_code) + " OK " + url) HTMLParser.feed(self, result.content) HTMLParser.close(self) else: logging.error(str(result.status_code) + " NG " + url)
class Retriever(): # download web pages def __init__(self, url): self.url = url def download(self): # download web page print 'try to open url:', self.url, '\nthe true url process', string.split( self.url, '?')[0] try: retval = urlopen(string.split(self.url, '?')[0], None, 200) except urllib2.HTTPError as e: print "HTTPError", e return except socket.timeout as e: print "socket.timeout", e return except socket.error as e: print "socket.error", e return except urllib2.URLError as e: print "URLError: ", e return return retval def parseAndGetLinks(self): # parse HTML, save links self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) r = self.download() if r: print '________' try: try: s = r.read(50000) except socket.error as e: print "***************************socket error***************************", e return [] self.parser.feed(s) print '------------------' r.close() print '***************************' except HTMLParseError: print 'get links error\n' return [] self.parser.close() return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter): HTMLParser.__init__(self, fmt) self.result = "" self.open_tags = [] # A list of forbidden tags. self.forbidden_tags = ['script', 'embed', 'iframe', 'frame'] # A list of tags that require no closing tag. self.requires_no_close = ['img', 'br'] # A dictionary showing the only attributes allowed for particular tags. # If a tag is not listed here, it is allowed no attributes. Adding # "on" tags, like "onhover," would not be smart. Also be very careful # of "background" and "style." # <h5 style="text-align: center;"><b><i><u><font size="5" face="impact">THIS IS A TEST</font></u></i></b></h5> # <blockquote style="margin: 0 0 0 40px; border: none; padding: 0px;"><p style="text-align: center;"> # <font size="5" face="arial" color="#cc3333">of the EBS</font></p><p style="text-align: center;"> # <font size="5" face="arial"><br></font></p><p style="text-align: center;"><font size="5" face="arial"> # <sup>reddit</sup><sub>2</sub></font></p> # <p style="text-align: center;"><font size="5" face="arial"><sub><br></sub></font></p> # <p style="text-align: center;"><font size="5" face="arial">fiiiiiii<sub>4</sub></font></p> # <p style="text-align: center;"><font size="5" face="arial"><sub><br></sub></font></p> # <p style="text-align: center;"><hr><br></p><p style="text-align: center;"> # <strike>strike</strike></p></blockquote> self.allowed_attributes =\ {'a':['href','title','target','style'], 'p': ['style'], 'img':['src','alt','border','style','align'], 'blockquote':['type','style','align'], 'font':['size','face','align'], 'h5':['style'],'h4':['style'],'h3':['style'],'h2':['style'],'h1':['style'], 'table': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'tbody': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'tr': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'td': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'div': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], 'span': ['border', 'width', 'height', 'style', 'align', 'bgcolor'], } # The only schemes allowed in URLs (for href and src attributes). # Adding "javascript" or "vbscript" to this list would not be smart. self.allowed_schemes = ['http', 'https', 'ftp']
class Retriever(object): def __init__(self, url): self.url = url self.file = self.filename(url) print self.file def filename(self, url, deffile='index.htm'): parsedurl = urlparse(url, 'http', 0) path = parsedurl[1] + parsedurl[2] ext = splitext(path) #分解文件名的扩展名 print path,ext if ext[1] == '': if path[-1] == '/': path += deffile else: path += '/' + deffile print path ldir = dirname(path) #应该和操作系统相关 windows下:sep = \ if sep != '/': ldir = replace(ldir, '/', sep) print ldir if not isdir(ldir): if exists(ldir): return makedirs(ldir) return path def download(self): try: retval = urlretrieve(self.url, self.file) except IOError: retval = ('*** Error URL "%s"' % self.url) return retval def parseAndGetLinks(self): """StringIO是从内存中读取数据 DumbWriter将事件流转换为存文本文档 AbstractFormatter 类进行格式化 """ self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() return self.parser.anchorlist
def _clean_text(self, text): try: text = text.replace(" ", " ") text = text.strip() parser = HTMLParser(None) parser.save_bgn() parser.feed(text) return parser.save_end() except: return text
def create_plaintext_message(message): """ Create clean plain text version of email message Parse the html and remove style and javacript tags and then create a plain-text-message by parsing the html and attaching links as endnotes """ cleaner = Cleaner() cleaner.javascript = True cleaner.style = True cleaner.kill_tags = ['style'] doc = message.decode('utf-8', 'ignore') to_clean = lxml.html.fromstring(doc) cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean)) plain_text_maxcols = 72 textout = cStringIO.StringIO() formtext = formatter.AbstractFormatter( formatter.DumbWriter(textout, plain_text_maxcols)) parser = HTMLParser(formtext) parser.feed(cleaned_msg) parser.close() # append the anchorlist at the bottom of a message # to keep the message readable. counter = 0 anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n" for item in parser.anchorlist: counter += 1 if item.startswith('https://'): new_item = item.replace('https://', 'http://') else: new_item = item anchorlist += "[%d] %s\n" % (counter, new_item) text = textout.getvalue() + anchorlist del textout, formtext, parser, anchorlist return text
def OpenURL(self, url): from htmllib import HTMLParser import formatter self.url = url m = re.match('http://([^/]+)(/\S*)\s*', url) if m: host = m.groups()[0] path = m.groups()[1] else: m = re.match('http://(\S+)\s*', url) if not m: # Invalid URL self.logprint("Invalid or unsupported URL: %s" % (url)) return host = m.groups()[0] path = '' f = self.RetrieveAsFile(host, path) if not f: self.logprint("Could not open %s" % (url)) return self.logprint("Receiving data...") data = f.read() tmp = open('hangman_dict.txt', 'w') fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp)) p = HTMLParser(fmt) self.logprint("Parsing data...") p.feed(data) p.close() tmp.close()
def parseAndGetLinks(self): # parse HTML, save links self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) r = self.download() if r: print '________' try: try: s = r.read(50000) except socket.error as e: print "***************************socket error***************************", e return [] self.parser.feed(s) print '------------------' r.close() print '***************************' except HTMLParseError: print 'get links error\n' return [] self.parser.close() return self.parser.anchorlist
class Retriever(object): #download Web pages def __init__(self, url): self.url = url self.file = self.filename(url) def filename(self, url): path=url path = re.sub("\W","_",path) path+=".html" return path def isForbidden(self): return 0; def isForbidden(self): return 0; def download(self): try: if True: retval = urlretrieve(self.url, self.file) javaGroupContent=JavaGroupContent.JavaGroupContent() javaGroupContent.meet_page(self.url, self.file) else: retval = '*** INFO: no need to download ' except IOError: retval = ('*** ERROR: invalid URL "%s"' % self.url,) return retval def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) try: self.parser.feed(open(self.file).read()) self.parser.close() except IOError: pass return self.parser.anchorlist
def get_urls(url): data = urllib.urlopen(url).read() parser = HTMLParser( formatter.AbstractFormatter(formatter.DumbWriter( cStringIO.StringIO()))) parser.feed(data) parser.close() url_list = parser.anchorlist return url_list
def parse_links(self): f = open(self.file, 'r') data = f.read() f.close() paeser = HTMLParser( formatter.AbstractFormatter( formatter.DumbWriter(cStringIO.StringIO()))) paeser.feed(data) paeser.close() return paeser.anchorlist