コード例 #1
0
class Retriever(object):#下载网页类

    def __init__(self,url):
        self.url = url
        self.file = self.filename(url)

    def filename(self,url,deffile ="index.htm"):
        parsedurl = urlparse(url,"http:",0) #解析路径
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path)
        if ext[1] == "": #如果没有文件,使用默认
            if path[-1] == "/":
                path += deffile
            else:
                path += "/" + deffile
        ldir = dirname(path) #本地目录
        if sep != "/":
            ldir = replace(ldir,"/",sep)
        if not isdir(ldir): #如果没有目录,创建一个
            if exists(ldir):unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):# 下载网页
        try:
            retval = urlretrieve(self.url,self.file)
        except IOError:
            retval = ('***Error: invalid URL: "%s"' % self.url,)
        return retval

    def parseAndGetLinks(self): #解析HTML,保存链接
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #2
0
class Retriever(htmllib.HTMLParser): #download Web Pages
	
	def __init__(self, url):
		self.url = url
		self.file = self.filename(url)

	def filename(self, url, deffile='index.htm'):
		parsedurl = urlparse(url,'http:',0) #parse path
		path = parsedurl[1] + parsedurl[2]
		text = splitext(path)
		if text[1]=='': #its not file use default
			if text[-1] == '/':
				path = path + deffile
			else:
				path = path + '/' + deffile
		print "PATH:%s" % path
		dir = dirname(path)
		if not isdir(dir): #create new archieve dir if necessary
			if exists(dir): unlink(dir)
			makedirs(dir)
		return path
	
	def download(self): #download web pages
		try:
			retval = urlretrieve(self.url,self.file)
		except IOError:
			retval =('***ERROR: invalid URL "%s"' % self.url)
		
		return retval
	def parseAndGetLinks(self): #Parse HTML
		self.parser = HTMLParser(AbstractFormatter(\
				DumbWriter(StringIO())))
		self.parser.feed(open(self.file).read())
		self.parser.close()
		return self.parser.anchorlist
コード例 #3
0
class Retriever(object):  # download Web pages
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile='index.htm'):
        parsedurl = urlparse(url, 'http:', 0)
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path)
        if ext[1] == '':  # no file, use default
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
        ldir = dirname(path)  # local directory
        if sep != '/':  # os-indep. path separator
            ldir = replace(ldir, '/', sep)
        if not isdir(ldir):  # create archive dir if nec.
            if exists(ldir):
                unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):  # download Web page
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** ERROR: invalid URL "%s"' % self.url, )
        return retval

    def parseAndGetLinks(self):  # parse HTML, save links
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #4
0
ファイル: crawl.py プロジェクト: gothack329/sirius.py
class Retriever(object):
	def __init__(self,url):
		self.url=url
		self.file=self.filename(url)
	def filename(self,url,deffile='index.html'):
		parsedurl=urlparse(url,'http:',0)
		path=parsedurl[1]+parsedurl[2]	#weibo.com+/gothack
		ext=splitext(path) #weibo.com/gothack , ''  #split by .
		if ext[1]=='':
			if path[-1]=='/':
				path+=deffile
			else:
				path+='/'+deffile
		ldir=dirname(path)	#weibo.com #before the last /
		if sep != '/':	#default value is /
			ldir=replce(ldir,'/',sep)	#replace  / with sep #(string,old,new)
		if not isdir(ldir):
			if exists(ldir):unlink(ldir)
			makedirs(ldir)
		return path
	def download(self):
		try:
			retval=urlretrieve(self.url,self.file)
		except IOError:
			retval=('***ERROR: invalid URL "%s"' %self.url,)
		return retval
	def parseAndGetLinks(self):
		self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
		self.parser.feed(open(self.file).read())
		self.parser.close()
		return self.parser.anchorlist
コード例 #5
0
def create_plaintext_message(message):
    """ Create clean plain text version of email message

        Parse the html and remove style and javacript tags and then
        create a plain-text-message by parsing the html
        and attaching links as endnotes
    """
    cleaner = Cleaner()
    cleaner.javascript = True
    cleaner.style = True
    cleaner.kill_tags = ['style']
    doc = message.decode('utf-8', 'ignore')
    to_clean = lxml.html.fromstring(doc)
    cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean))
    plain_text_maxcols = 72
    textout = cStringIO.StringIO()
    formtext = formatter.AbstractFormatter(
        formatter.DumbWriter(textout, plain_text_maxcols))
    parser = HTMLParser(formtext)
    parser.feed(cleaned_msg)
    parser.close()
    # append the anchorlist at the bottom of a message
    # to keep the message readable.
    counter = 0
    anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
    for item in parser.anchorlist:
        counter += 1
        if item.startswith('https://'):
            new_item = item.replace('https://', 'http://')
        else:
            new_item = item
        anchorlist += "[%d] %s\n" % (counter, new_item)
    text = textout.getvalue() + anchorlist
    del textout, formtext, parser, anchorlist
    return text
コード例 #6
0
    def create_plaintext_message(self, text):
        """ Create a plain-text-message by parsing the html
            and attaching links as endnotes

            Modified from EasyNewsletter/content/ENLIssue.py
        """
        # This reflows text which we don't want, but it creates
        # parser.anchorlist which we do want.
        textout = StringIO.StringIO()
        formtext = formatter.AbstractFormatter(formatter.DumbWriter(textout))
        parser = HTMLParser(formtext)
        parser.feed(text)
        parser.close()

        # append the anchorlist at the bottom of a message
        # to keep the message readable.
        counter = 0
        anchorlist = "\n\n" + '----' + "\n\n"
        for item in parser.anchorlist:
            counter += 1
            anchorlist += "[%d] %s\n" % (counter, item)

        # This reflows text:
        # text = textout.getvalue() + anchorlist
        # This just strips tags, no reflow
        text = html.fromstring(text).text_content()
        text += anchorlist
        del textout, formtext, parser, anchorlist
        return text
コード例 #7
0
class Retriever(object):  #下载网页类
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile="index.htm"):
        parsedurl = urlparse(url, "http:", 0)  #解析路径
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path)
        if ext[1] == "":  #如果没有文件,使用默认
            if path[-1] == "/":
                path += deffile
            else:
                path += "/" + deffile
        ldir = dirname(path)  #本地目录
        if sep != "/":
            ldir = replace(ldir, "/", sep)
        if not isdir(ldir):  #如果没有目录,创建一个
            if exists(ldir): unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):  # 下载网页
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = ('***Error: invalid URL: "%s"' % self.url, )
        return retval

    def parseAndGetLinks(self):  #解析HTML,保存链接
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #8
0
class Restriever(object):
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile = 'index.htm'):
        parsedurl = urlparse(url, 'http:', 0)
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path)
        if ext[1] == '':
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile

        ldir = dirname(path)
        if sep != '/':
            ldir = replace(ldir, '/', sep)
        if not isdir(ldir):
            if exists(ldir) : unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** ERROR: invalid URL "%s"' % self.url)
            return retval

    def parseAndGetLinks(self):
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #9
0
ファイル: 1.py プロジェクト: ligy2016/untitled
class Retriever(object):  # download Web pages

    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile='index.htm'):
        parsedurl = urlparse(url, 'http:', 0)  ## parse path
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path)
        if ext[1] == '':  # no file, use default
            if path[-1] == '/':
                path += deffile
        else:
            path += '/' + deffile
        ldir = dirname(path)  # local directory
        if sep != '/':  # os-indep. path separator
            ldir = replace(ldir, '/', sep)
        if not isdir(ldir):  # create archive dir if nec.
            if exists(ldir): unlink(ldir)
        makedirs(ldir)
        return path

    def parseAndGetLinks(self):
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #10
0
ファイル: mail.py プロジェクト: hwaring/python-zoom
def get_plain_from_html(html):
    textout = StringIO()
    formtext = AbstractFormatter(DumbWriter(textout))
    parser = HTMLParser(formtext)
    parser.feed(html)
    parser.close()
    return textout.getvalue()
コード例 #11
0
ファイル: dispatcher.py プロジェクト: a25kk/kjf-pmapp
 def create_plaintext_message(self, text):
     """ Create a plain-text-message by parsing the html
         and attaching links as endnotes
     """
     plain_text_maxcols = 72
     textout = cStringIO.StringIO()
     formtext = formatter.AbstractFormatter(formatter.DumbWriter(
                                            textout, plain_text_maxcols))
     parser = HTMLParser(formtext)
     parser.feed(text)
     parser.close()
     # append the anchorlist at the bottom of a message
     # to keep the message readable.
     counter = 0
     anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
     for item in parser.anchorlist:
         counter += 1
         if item.startswith('https://'):
             new_item = item.replace('https://', 'http://')
         else:
             new_item = item
         anchorlist += "[%d] %s\n" % (counter, new_item)
     text = textout.getvalue() + anchorlist
     del textout, formtext, parser, anchorlist
     return text
コード例 #12
0
ファイル: hangman.py プロジェクト: arnelh/Examples
 def OpenURL(self,url):
     from htmllib import HTMLParser
     import formatter
     self.url = url
     m = re.match('http://([^/]+)(/\S*)\s*', url)
     if m:
         host = m.groups()[0]
         path = m.groups()[1]
     else:
         m = re.match('http://(\S+)\s*', url)
         if not m:
             # Invalid URL
             self.logprint("Invalid or unsupported URL: %s" % (url))
             return
         host = m.groups()[0]
         path = ''
     f = self.RetrieveAsFile(host,path)
     if not f:
         self.logprint("Could not open %s" % (url))
         return
     self.logprint("Receiving data...")
     data = f.read()
     tmp = open('hangman_dict.txt','w')
     fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp))
     p = HTMLParser(fmt)
     self.logprint("Parsing data...")
     p.feed(data)
     p.close()
     tmp.close()
コード例 #13
0
ファイル: mailer.py プロジェクト: a25kk/julia
def create_plaintext_message(message):
        """ Create clean plain text version of email message

            Parse the html and remove style and javacript tags and then
            create a plain-text-message by parsing the html
            and attaching links as endnotes
        """
        cleaner = Cleaner()
        cleaner.javascript = True
        cleaner.style = True
        cleaner.kill_tags = ['style']
        doc = message.decode('utf-8', 'ignore')
        to_clean = lxml.html.fromstring(doc)
        cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean))
        plain_text_maxcols = 72
        textout = cStringIO.StringIO()
        formtext = formatter.AbstractFormatter(formatter.DumbWriter(
                                               textout, plain_text_maxcols))
        parser = HTMLParser(formtext)
        parser.feed(cleaned_msg)
        parser.close()
        # append the anchorlist at the bottom of a message
        # to keep the message readable.
        counter = 0
        anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
        for item in parser.anchorlist:
            counter += 1
            if item.startswith('https://'):
                new_item = item.replace('https://', 'http://')
            else:
                new_item = item
            anchorlist += "[%d] %s\n" % (counter, new_item)
        text = textout.getvalue() + anchorlist
        del textout, formtext, parser, anchorlist
        return text
コード例 #14
0
ファイル: ex20_2.py プロジェクト: Crayon277/python_me
class Retriever(object):    # download Web pages
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile = 'index.htm'):
        parsedurl = urlparse(url, 'http:', 0)   # parse path
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path)
        if ext[1] == '': # no file, use default
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
        ldir = dirname(path)    # local directory
        if sep != '/':  
            ldir = replace(ldir, '/', sep)
        if not isdir(ldir):
            if exists(ldir): 
                unlink(ldir)
                makedirs(ldir)
        return path

    def download(self):
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** ERROR: invalid URL "%s"' % self.url)
        return retval

    def parseAndGetLinks(self):
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #15
0
 def OpenURL(self, url):
     from htmllib import HTMLParser
     import formatter
     self.url = url
     m = re.match('http://([^/]+)(/\S*)\s*', url)
     if m:
         host = m.groups()[0]
         path = m.groups()[1]
     else:
         m = re.match('http://(\S+)\s*', url)
         if not m:
             # Invalid URL
             self.logprint("Invalid or unsupported URL: %s" % (url))
             return
         host = m.groups()[0]
         path = ''
     f = self.RetrieveAsFile(host, path)
     if not f:
         self.logprint("Could not open %s" % (url))
         return
     self.logprint("Receiving data...")
     data = f.read()
     tmp = open('hangman_dict.txt', 'w')
     fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp))
     p = HTMLParser(fmt)
     self.logprint("Parsing data...")
     p.feed(data)
     p.close()
     tmp.close()
コード例 #16
0
ファイル: SiteSpider.py プロジェクト: xiancode/Site_Crawler
class Retriever(object):
    def __init__(self,url):
        self.url=url
        self.file=self.filename(url)
    def filename(self,url,deffile='index.html'):
        """
        生成下载连接和文件名
        """
        full_url = ""
        if url.endswith(DOM_SUFFIX):
            full_url = url + '/'
        else:
            full_url = url
        parsedurl=urlparse(full_url,'http:',0)
        path=parsedurl[1]+parsedurl[2]
        ext=splitext(path)
        if ext[1]=='':
            if path[-1]=='/':
                path+=deffile
            else:
                path+='/'+deffile
        ldir=dirname(path)
        if sep!='/':
            ldir=replace(ldir,'/',sep)
            path=replace(path,'/',sep)

        if not isdir(ldir):
            if exists(ldir):
                    #unlink(ldir)
                pass
            else:    
                makedirs(ldir)
        print path
        return path
            
    def download(self):
        """
        下载文件
        """
        try:
            retval=urlretrieve(self.url,self.file)
        except IOError:
            retval=('***ERROR :invalid URL "%s"' %self.url,)
        return retval   
    
    def parseAndGetLinks(self):
        """
        获取网页中的链接
        """
        #print 'Get Html Links from file:%s' % self.file
        #self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO)))
        self.parser=HTMLParser(NullFormatter())
        #self.parser.feed(open(self.file).read())
        try:
            self.parser.feed(open(self.file).read())
            self.parser.close()
            return self.parser.anchorlist
        except:
            print self.file + " error !"
            return [] 
コード例 #17
0
ファイル: crawl.py プロジェクト: refinedKing/scripts
class Retrive(object):
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile='index.php'):
        parsedurl = urlparse(url,'http:', 0)
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path)
        if ext[1] == '':
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
        ldir = dirname(path)
        if sep != '/':
            ldir = replace(ldir, '/', sep)
        if not isdir(ldir):
            if exists(ldir):
                unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = 'error'
            return retval

    def parseAndGetLinks(self):
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #18
0
ファイル: textformat.py プロジェクト: andik/robin
def html2text(htmldata):
	# patch htmldata
	htmldata = htmldata.replace("<br/>", "<br>")
	
	fmt = HTMLtoTextFormatter()
	prs = HTMLParser(fmt)
	prs.feed(htmldata)
	return fmt.getText()
コード例 #19
0
ファイル: textformat.py プロジェクト: mishas/robin
def html2text(htmldata):
    # patch htmldata
    htmldata = htmldata.replace("<br/>", "<br>")

    fmt = HTMLtoTextFormatter()
    prs = HTMLParser(fmt)
    prs.feed(htmldata)
    return fmt.getText()
コード例 #20
0
def get_urls(url):
    data = urllib.urlopen(url).read()
    parser = HTMLParser(
        formatter.AbstractFormatter(formatter.DumbWriter(
            cStringIO.StringIO())))
    parser.feed(data)
    parser.close()
    url_list = parser.anchorlist
    return url_list
コード例 #21
0
 def parse_links(self):
     'Parse out the links found in downloaded HTML file'
     f = open(self.file, 'r')
     data = f.read()
     f.close()
     parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
     parser.feed(data)
     parser.close()
     return parser.anchorlist
コード例 #22
0
ファイル: default.py プロジェクト: drrlramsey/xbmc-addons
 def _clean_text( self, text ):
     try:
         text = text.replace( "&nbsp;", " " )
         text = text.strip()
         parser = HTMLParser( None )
         parser.save_bgn()
         parser.feed( text )
         return parser.save_end()
     except:
         return text
コード例 #23
0
 def parse_links(self):
     'Parse out the links found in downloaded HTML file'
     f = open(self.file, 'r')
     data = f.read()
     f.close()
     parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(
         cStringIO.StringIO())))
     parser.feed(data)
     parser.close()
     return parser.anchorlist
コード例 #24
0
ファイル: pocket_sendmail.py プロジェクト: louxiu/pocket-push
 def parse_links(self):
     """fetch all links from page
     """
     f = open(self.save_file, 'r')
     data = f.read()
     f.close()
     parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
     parser.feed(data)
     parser.close()
     return parser.anchorlist
コード例 #25
0
class Retriever(object):
	def __init__(self,url):
		self.url = url

	#parse HTML ,save links
	def parseAndGetLinks(self):
		self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
		self.parser.feed(urlopen(self.url).read())
		self.parser.close()
		return self.parser.anchorlist
コード例 #26
0
 def _clean_text(self, text):
     try:
         text = text.replace("&nbsp;", " ")
         text = text.strip()
         parser = HTMLParser(None)
         parser.save_bgn()
         parser.feed(text)
         return parser.save_end()
     except:
         return text
コード例 #27
0
 def parse_links(self):
     f = open(self.file, 'r')
     data = f.read()
     f.close()
     paeser = HTMLParser(
         formatter.AbstractFormatter(
             formatter.DumbWriter(cStringIO.StringIO())))
     paeser.feed(data)
     paeser.close()
     return paeser.anchorlist
コード例 #28
0
def html2text(html):
    f = StringIO()
    parser = HTMLParser(AbstractFormatter(DumbWriter(f)))
    try:
        parser.feed(html)
    except HTMLParseError:
        return ''
    else:
        parser.close()
        return f.getvalue()
コード例 #29
0
def get_text_from_html( html_input ):
  "Strip tags and non-ascii characters from HTML input."
  my_stringio = StringIO.StringIO() # make an instance of this file-like string thing
  p = HTMLParser(AbstractFormatter(DumbWriter(my_stringio)))
  try: p.feed(html_input); p.close() #calling close is not usually needed, but let's play it safe
  except HTMLParseError: print '***HTML malformed***' #the html is badly malformed (or you found a bug)
  #return my_stringio.getvalue().replace('\xa0','')
  s = re.sub( r'[^\x00-\x7f]', r' ', my_stringio.getvalue() )
  s = s.replace('\r\n',' ').replace('\n',' ')
  s = re.sub( ' +', ' ', s )
  return s
コード例 #30
0
ファイル: crawl.py プロジェクト: yfang1644/web_training
 def parse_link(seld):
     'Parse out the link'
     f = open('seld.file', 'r')
     data = f.read()
     f.close()
     parser = HTMLParser(
         formatter.AbstractFormatter(
             formatter.DumbWriter(cStringIO.StringIO())))
     parser.feed(data)
     parser.close()
     return parser.anchorlist
コード例 #31
0
 def parse_links(self):
     """fetch all links from page
     """
     f = open(self.save_file, 'r')
     data = f.read()
     f.close()
     parser = HTMLParser(
         formatter.AbstractFormatter(
             formatter.DumbWriter(cStringIO.StringIO())))
     parser.feed(data)
     parser.close()
     return parser.anchorlist
コード例 #32
0
ファイル: crawler_ex.py プロジェクト: Johnson-wu/python
 def parse_links(self):
     f = open(self.file, "r")
     data = f.read()
     f.close()
     parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
     # parser = MyHTMLParser()
     parser.feed(data)
     parser.close()
     # 没有在模块中找到 anchorlist属性
     # 返回页面中所有的锚点,href
     # 该属性在2.6后就被弃用了,如果想用可以通过自定义parser继承HTMLParser来实现,参见evernote.或者用第三方库beautifulsoup
     return parser.anchorlist
コード例 #33
0
 def parse_html(self, html):
     from StringIO import StringIO
     from formatter import (AbstractFormatter, DumbWriter)
     from htmllib import HTMLParser
     _html = re.sub(self.notrans_tag, r" \1 ", html)
     buf = StringIO()
     p = HTMLParser(AbstractFormatter(DumbWriter(buf)))
     p.feed(_html)
     _sub = re.sub(self.whitespaces, " ", buf.getvalue())
     # FIXME: how can zerowidth be removed more simply?
     _sub = re.sub(self.zerowidth, "", _sub)
     _sub = re.sub(self.colon, r"\1", _sub)
     return _sub
コード例 #34
0
 def parse(self, url):
     self.base = ""
     self.href = ""
     m = re.compile(".*/").match(url)
     if m != None:
         self.base = m.string[m.start(0):m.end(0)]
     result = urlfetch.fetch(url, headers = {'Cache-Control' : 'max-age=30', 'Pragma' : 'no-cache'} )
     if result.status_code == 200:
         logging.debug(str(result.status_code) + " OK " + url)
         HTMLParser.feed(self, result.content)
         HTMLParser.close(self)
     else:
         logging.error(str(result.status_code) + " NG " + url)
コード例 #35
0
class Retriever(object):

    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile='index.htm'):
        # 使用http协议解析url地址,将url解析为这样的五元组:(scheme, netloc, path, query, fragment)
        parsedurl = urlparse(url, 'http:', 0)
        # 将主机地址(netloc)和路径(path)合并起来作为存储文件的路径名
        path = parsedurl[1] + parsedurl[2]
        # splitext将path分割为路径与后缀名,如(/path/to/file, txt)
        ext = splitext(path)
        # 如果后缀名为空,则给当前path添加默认的名字index.htm
        if ext[1] == '':
            # 若path结尾有“/”则直接添加index.htm,否则先添加“/”
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
        # 取出path中的目录部分(www.shellbye.com\\blog),然后与本地目录结合为一个目录('D:\\www.shellbye.com\\blog')
        ldir = dirname(abspath(path))
        # 如果不是以“/”作为目录分割符的类Unix系统,比如Windows,则需要把“”替换为相应的目录分割符
        # 因为类Unix系统的目录分割符与URI地址的分割符一样,所以可以不处理
        if sep != '/':        # os-indep. path separator
            ldir = replace(ldir, '/', sep)
        # 如果ldir目录不存在在创建
        if not isdir(ldir):      # create archive dir if nec.
            # 如果ldir存在但是不是目录则删除ldir。注,unlink即remove。
            if exists(ldir):
                unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):        # download Web page
        try:
            # 下载self.url到self.file里
            retval = urllib.urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** ERROR: invalid URL "%s"' % self.url, )
        return retval

    def parseAndGetLinks(self):
        # 创建一个基本的HTML解释器,可能需要单独一篇文章来说这句
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        # 解析html文件,获取所有的连接(带有href的)
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #36
0
ファイル: Crawl.py プロジェクト: mantofast/pythonWeb
class Retriever(object):#download web page
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    #local filename ,directory
    def filename(self, url, deffile = "index.htm"):
        parseurl = urlparse(url, 'http:', 0)
        path = parseurl[1] + parseurl[2]

        #将路径转换为一个元组,如果为目录则第二个元素为空,如果文件则第二个元素为文件扩展名
        #path = "D:/pycharmProjects/PythonWebApp/weblearning/Crawl.py"
        #print splitext(path)
        #('D:/pycharmProjects/PythonWebApp/weblearning/Crawl', '.py')

        ext = splitext(path)
        if ext[1] == '':#no file use default
            #tuple[-index],倒数第index个元素
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
        #获取path的dir,
        # path=D:/pycharmProjects/PythonWebApp/webLearning
        # dir(path)=D:/pycharmProjects/PythonWebApp
        ldir = dirname(path)#local directory
        if sep != '/': #os-indep. path separator
            ldir = replace(ldir, '/', sep)
        if not isdir(ldir): #create archive dir if nec.
            #如果存在文件,则删除
            if exists(ldir):
                unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):
        #urlretrieve()返回一个2元组,(filename,mime_hdrs)
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** ERROR: invalid URL "%s"' %self.url,)
            print 'erro,invalid url'
        return retval

    def parseAndGetLinks(self):#parse HTML , save links
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #37
0
class Retriever(object):
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile='index.htm'):
        # 使用http协议解析url地址,将url解析为这样的五元组:(scheme, netloc, path, query, fragment)
        parsedurl = urlparse(url, 'http:', 0)
        # 将主机地址(netloc)和路径(path)合并起来作为存储文件的路径名
        path = parsedurl[1] + parsedurl[2]
        # splitext将path分割为路径与后缀名,如(/path/to/file, txt)
        ext = splitext(path)
        # 如果后缀名为空,则给当前path添加默认的名字index.htm
        if ext[1] == '':
            # 若path结尾有“/”则直接添加index.htm,否则先添加“/”
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
        # 取出path中的目录部分(www.shellbye.com\\blog),然后与本地目录结合为一个目录('D:\\www.shellbye.com\\blog')
        ldir = dirname(abspath(path))
        # 如果不是以“/”作为目录分割符的类Unix系统,比如Windows,则需要把“”替换为相应的目录分割符
        # 因为类Unix系统的目录分割符与URI地址的分割符一样,所以可以不处理
        if sep != '/':  # os-indep. path separator
            ldir = replace(ldir, '/', sep)
        # 如果ldir目录不存在在创建
        if not isdir(ldir):  # create archive dir if nec.
            # 如果ldir存在但是不是目录则删除ldir。注,unlink即remove。
            if exists(ldir):
                unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):  # download Web page
        try:
            # 下载self.url到self.file里
            retval = urllib.urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** ERROR: invalid URL "%s"' % self.url, )
        return retval

    def parseAndGetLinks(self):
        # 创建一个基本的HTML解释器,可能需要单独一篇文章来说这句
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        # 解析html文件,获取所有的连接(带有href的)
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #38
0
ファイル: mail.py プロジェクト: brettgoss/datazoomer
def get_plain_from_html(html):
    """extract plain text from html

    >>> test_html = "<div><h1>Hey<h1><p>This is some text</p></div>"
    >>> get_plain_from_html(test_html)
    '\\nHey\\n\\nThis is some text'

    """
    from htmllib import HTMLParser  # import here to avoid high startup cost

    textout = StringIO()
    formtext = AbstractFormatter(DumbWriter(textout))
    parser = HTMLParser(formtext)
    parser.feed(html)
    parser.close()
    return textout.getvalue()
コード例 #39
0
ファイル: mail.py プロジェクト: sean-hayes/python-zoom
def get_plain_from_html(html):
    """extract plain text from html

    >>> test_html = "<div><h1>Hey<h1><p>This is some text</p></div>"
    >>> get_plain_from_html(test_html)
    '\\nHey\\n\\nThis is some text'

    """
    from htmllib import HTMLParser  # import here to avoid high startup cost

    textout = StringIO()
    formtext = AbstractFormatter(DumbWriter(textout))
    parser = HTMLParser(formtext)
    parser.feed(html)
    parser.close()
    return textout.getvalue()
コード例 #40
0
ファイル: crawler.py プロジェクト: shreya-inamdar/Crawler-1.0
class Retriever:
    '''
    responsibilities:
    download, parse and queue
    '''
    def __init__(self,url):
        '''
        contructor of class.Instantiates the Retriver object and stores the url and filename as local attributes
        '''
        self.url = url
        self.file = self.filename(url)

    def filename(self, url, deffile = 'index.html'):
        '''
        input: url
        removes the http prefix
        index.html will be the default file name for storage of the url:this can be overridden by passing arguments to filename()
        '''
        parsedurl = urlparse(url,"http:",0) #parse path
        path = parsedurl[1] + parsedurl[2]
        text = splitext(path)
        if text[1] == '': #no file, use default
            path  =  path + deffile
        else:
            path = path + '/' + deffile
        dir = dirname(path)
        if not isdir(dir):  #create a new directory if necessary
            if exists(dir): unlink(dir)
            makedirs(dir)
        return path

    def download(self): #download web page
        try:
            retval = urlretrieve(self.url,self.file)
        except IOError:
            retval = ('***ERROR invalid url "%s"'%self.url,)
        return retval

    def parseAndGetLinks(self): #parse HTML and getlinks
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        #try:
        self.parser.feed(open(self.file).read())
        #except HTMLParseError:
        self.parser.close()

        self.parser.close()
        return self.parser.anchorlist
コード例 #41
0
 def parse(self, url):
     self.base = ""
     self.href = ""
     m = re.compile(".*/").match(url)
     if m != None:
         self.base = m.string[m.start(0):m.end(0)]
     result = urlfetch.fetch(url,
                             headers={
                                 'Cache-Control': 'max-age=30',
                                 'Pragma': 'no-cache'
                             })
     if result.status_code == 200:
         logging.debug(str(result.status_code) + " OK " + url)
         HTMLParser.feed(self, result.content)
         HTMLParser.close(self)
     else:
         logging.error(str(result.status_code) + " NG " + url)
コード例 #42
0
ファイル: crawl.py プロジェクト: wengowl/gae
class Retriever():  # download web pages
    def __init__(self, url):
        self.url = url

    def download(self):  # download web page
        print 'try to open url:', self.url, '\nthe true url process', string.split(
            self.url, '?')[0]
        try:
            retval = urlopen(string.split(self.url, '?')[0], None, 200)
        except urllib2.HTTPError as e:
            print "HTTPError", e
            return
        except socket.timeout as e:
            print "socket.timeout", e
            return
        except socket.error as e:
            print "socket.error", e
            return
        except urllib2.URLError as e:
            print "URLError: ", e
            return
        return retval

    def parseAndGetLinks(self):  # parse HTML, save links
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        r = self.download()
        if r:
            print '________'
            try:
                try:
                    s = r.read(50000)
                except socket.error as e:
                    print "***************************socket error***************************", e
                    return []
                self.parser.feed(s)
                print '------------------'

                r.close()
                print '***************************'
            except HTMLParseError:
                print 'get links error\n'
                return []

        self.parser.close()
        return self.parser.anchorlist
コード例 #43
0
 def compactor(dev_filename, rel_filename):
     # Use compactor to generate release version.
     echo('Compacting: %s -> %s' % (dev_filename, rel_filename))
     source_data = open(dev_filename, 'r').read()
     try:
         # Verify that the html file is correct
         htmlparser = HTMLParser(NullFormatter())
         htmlparser.feed(source_data)
         htmlparser.close()
         # Now try to minify
         output_file = open(rel_filename, 'wb')
         compactor = HTMLMinifier(output_file.write, True)
         compactor.feed(source_data)
         compactor.close()
         output_file.close()
     except HTMLParseError as e:
         error(str(e))
         exit(1)
コード例 #44
0
 def compactor(dev_filename, rel_filename):
     # Use compactor to generate release version.
     echo('Compacting: %s -> %s' % (dev_filename, rel_filename))
     source_data = open(dev_filename, 'r').read()
     try:
         # Verify that the html file is correct
         htmlparser = HTMLParser(NullFormatter())
         htmlparser.feed(source_data)
         htmlparser.close()
         # Now try to minify
         output_file = open(rel_filename, 'wb')
         compactor = HTMLMinifier(output_file.write, True)
         compactor.feed(source_data)
         compactor.close()
         output_file.close()
     except HTMLParseError as e:
         error(str(e))
         exit(1)
コード例 #45
0
ファイル: link_finder.py プロジェクト: zangree/Spider
class LinkFinder(object):
    def __init__(self, base_url, page_url):
        self.base_url = base_url
        self.page_url = page_url

    def parseAndGetLinks(self, html_string):
        try:
            self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
            self.parser.feed(html_string)
            self.parser.close()
            links = []
            for eachLink in self.parser.anchorlist:
                if eachLink[:4] != "http" and find(eachLink, "://") == -1:
                    eachLink = urljoin(self.base_url, eachLink)
                links.append(eachLink)
            return links
        except IOError:
            return []
コード例 #46
0
ファイル: crawl.py プロジェクト: wengowl/gae
class Retriever():  # download web pages
    def __init__(self, url):
        self.url = url

    def download(self):  # download web page
        print 'try to open url:', self.url, '\nthe true url process', string.split(self.url, '?')[0]
        try:
            retval = urlopen(string.split(self.url, '?')[0], None, 200)
        except urllib2.HTTPError as e:
            print "HTTPError", e
            return
        except socket.timeout as e:
            print "socket.timeout", e
            return
        except socket.error as e:
            print "socket.error", e
            return
        except urllib2.URLError as e:
            print "URLError: ", e
            return
        return retval

    def parseAndGetLinks(self):  # parse HTML, save links
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        r = self.download()
        if r:
            print '________'
            try:
                try:
                    s = r.read(50000)
                except socket.error as e:
                    print "***************************socket error***************************", e
                    return []
                self.parser.feed(s)
                print '------------------'

                r.close()
                print '***************************'
            except HTMLParseError:
                print 'get links error\n'
                return []

        self.parser.close()
        return self.parser.anchorlist
コード例 #47
0
def collectURLSFromPage(page):
    """
    This returns a list of URLS that come from a certain page.
    Useful for spiders. It takes just a string as an argument.
    """

    resultList = []
    if page == "":
        #nothing to parse, so nothing to return
        return resultList

    #print "Doing form parser"
    if page.count("<form") > 0:
        otherlist = daveFormParse(page)
        for key in otherlist:
            resultList.append(key)
            pass

    #DEBUG
    #return resultList

    #print "Doing RAW Parser"
    spamList = rawParse(page)
    for key in spamList:
        resultList.append(key)
        pass

    #This needs to be documented somehow, but I have no idea what it does
    try:
        parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        parser.feed(page)
        parser.close()

    except:
        #print "DEBUG: Caught an exception trying to parse that html file."
        #print "(Not sure why this happens - you'll have to crawl this page manually)"
        return resultList

    #print "Adding HTML Parser data"
    for key in parser.anchorlist:
        resultList.append(key)
        pass

    return resultList
コード例 #48
0
ファイル: crawl.py プロジェクト: dawndiy/spider
class Retriever(object):	# download Web pages
	"""docstring for Retriever"""
	def __init__(self, url):
		self.url = url
		self.file = self.filename(url)
		
	def filename(self, url, deffile='index.html'):
		parsedurl = urlparse(url, 'http:', 0)	# parse path
		print '====PARSEDURL====',parsedurl
		if parsedurl[2] == '':
			path = parsedurl[1] + '/'
		else:
			path = parsedurl[1] + parsedurl[2]
		print '------PATH-----', path
		ext = splitext(path)
		print '-----EXT----', ext
		if ext[1] == '':	# no file, use default
			if path[-1] == '/':
				path +=deffile
			else:
				path += '/' + deffile
		ldir = dirname(path)	# local directory
		print '+++++++++++++++++', ldir
		if sep != '/':	# os-indep. path separator
			ldir = replace(ldir, '/', sep)
		if not isdir(ldir):	# create archive dir if nec.
			if exists(ldir): unlink(ldir)
			makedirs(ldir)
		return path

	def download(self):	# download Web page
		try:
			retval = urlretrieve(self.url, self.file)
		except IOError:
			retval = ('*** ERROR: invalid URL "%s"' % self.url,)
		return retval

	def parseAndGetLinks(self):	# parse HTML, save links
		# self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO)))
		self.parser = HTMLParser(AbstractFormatter(DumbWriter()))
		self.parser.feed(open(self.file).read())
		self.parser.close()
		return self.parser.anchorlist
コード例 #49
0
ファイル: crawl.py プロジェクト: DaZhu/all
class Retriever(object):	# download Web pages

	def __init__(self, url):
		self.url = url
		self.file = self.filename(url)
		

	def filename(self, url, deffile='index.htm'):
		parsedurl = urlparse(url, 'http:', 0)  # parse path
		path = parsedurl[1] + parsedurl[2]
		ext = splitext(path)
		print path
		if ext[1] == '':
			if path[-1] == '/':
				path += deffile
			else:
				path += '/' + deffile
		ldir = dirname(path)	# local directory
		
		if sep != '/':		# os-indep. path separator
			ldir = replace(ldir, ',', sep)
			if not isdir(ldir):	  # create archive dir if nec.
				if exists(ldir): unlink(ldir)
				print ldir, "aaaaaaaaa"
				makedirs(ldir)
				
			return path

	def download(self):		# download Web page
		try:
			retval = urllib.urlretrieve(self.url, self.file)
		except IOError:
			retval = ('*** ERROR: invalid URL "%s"' % \
				self.url, )
		return retval

	def parseAndGetLinks(self):	# pars HTML, save links
		self.parser = HTMLParser(AbstractFormatter( \
			DumbWriter(StringIO())))
		self.parser.feed(open(self.file).read())
		self.parser.close()
		print self.parser
		return self.parser.anchorlist
コード例 #50
0
ファイル: link_finder.py プロジェクト: zangree/Spider
class LinkFinder(object):

    def __init__(self,base_url,page_url):
        self.base_url = base_url
        self.page_url = page_url

    def parseAndGetLinks(self,html_string):
        try:
            self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
            self.parser.feed(html_string)
            self.parser.close()
            links = []
            for eachLink in self.parser.anchorlist:
                if eachLink[:4] != "http" and find(eachLink, "://") == -1:
                    eachLink = urljoin(self.base_url, eachLink)
                links.append(eachLink)
            return links
        except IOError:
            return []
コード例 #51
0
ファイル: spider.py プロジェクト: gt11799/spider-python
class Retriever(object):
    'download web pages'
    
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)
        
    def filename(self, url, deffile='index.htm'):
        parsedurl = urlparse(url, 'http:', 0)
        path = parsedurl[1] + parsedurl[2]
        print path
        ext = splitext(path)#return (filename, extension)
        if ext[1] == '': #no file, use default
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
                
        ldir = dirname(path) #local directory
        if sep != '/': #os-indep.path separator
            ldir = replace(ldir, '/', sep)
        if not isdir(ldir): #create archive dir if nec
            if exists(ldir):
                unlink(ldir)
            makedirs(ldir)
            
        return path
        
    def download(self):
        'download web page'
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** ERROR: Invalid URL "%s"' %self.url)
        return retval
            
    def parseAndGetLinks(self):
        'parse HTML, save links'
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #52
0
ファイル: neteasy_crawler.py プロジェクト: didaladida/toys
class Retriever(object):
    
    def __init__(self,url):
        self.url=url
        self.file=self.filename(url)
    
    def filename(self,url,deffile='index.htm'):
        parsedurl=urlparse(url,'http:',0)
        path=parsedurl[1]+parsedurl[2]
        ext=splitext(path)
        # 爬取的必须是静态html
        if ext[1]=='':
            if path[-1]=='/':
                path+=deffile
            else:
                path+='/'+deffile
        #print path
        # 建立文件目录
        ldir=dirname(path)
        #print ldir
        #将url中的/转为windows的
        if sep!='/':
            ldir=replace(ldir,'/',sep)
        
        if not isdir(ldir):
            if exists(ldir): unlink(ldir)
            makedirs(ldir)
        return path

    def download(self):
        try:
            retval=urlretrieve(self.url,self.file)
        except IOError:
            retval=('***error in url "%s"'%self.url)
        return retval

    def parseAndGetLink(self):
        #构造一个解析器
        self.parser= HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #53
0
ファイル: Test.py プロジェクト: lgwjd0090/testgit
class Retriever(object):
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)
        print self.file
    def filename(self, url, deffile='index.htm'):
        parsedurl = urlparse(url, 'http', 0)
        path = parsedurl[1] + parsedurl[2]
        ext = splitext(path) #分解文件名的扩展名
        print path,ext
        if ext[1] == '':
            if path[-1] == '/':
                path += deffile
            else:
                path += '/' + deffile
        print path
        ldir = dirname(path)
        #应该和操作系统相关 windows下:sep = \
        if sep != '/':
            ldir = replace(ldir, '/', sep)
        print ldir
        if not isdir(ldir):
            if exists(ldir):
                return
            makedirs(ldir)
        return path

    def download(self):
        try:
            retval = urlretrieve(self.url, self.file)
        except IOError:
            retval = ('*** Error URL "%s"' % self.url)
        return retval

    def parseAndGetLinks(self):
        """StringIO是从内存中读取数据 DumbWriter将事件流转换为存文本文档  AbstractFormatter 类进行格式化
        """
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        self.parser.feed(open(self.file).read())
        self.parser.close()
        return self.parser.anchorlist
コード例 #54
0
    def _create_plaintext_message(self, text):
        """ Create a plain-text-message by parsing the html
            and attaching links as endnotes
        """
        plain_text_maxcols = 72
        textout = cStringIO.StringIO()
        formtext = formatter.AbstractFormatter(
            formatter.DumbWriter(textout, plain_text_maxcols))
        parser = HTMLParser(formtext)
        parser.feed(text)
        parser.close()

        # append the anchorlist at the bottom of a message
        # to keep the message readable.
        anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
        for counter, item in enumerate(parser.anchorlist):
            anchorlist += "[{0:d}] {1:s}\n".format(counter, item)

        text = textout.getvalue() + anchorlist
        del textout, formtext, parser, anchorlist
        return text
コード例 #55
0
ファイル: daveutil.py プロジェクト: ChadThomsonPSC/owasp-wte
def collectURLSFromPage(page):

    resultList = []

    #print "Doing form parser"
    if page.count("<form") > 0:
        otherlist = daveFormParse(page)
        for key in otherlist:
            resultList.append(key)
            pass

    #DEBUG
    #return resultList

    #print "Doing RAW Parser"
    spamList = rawParse(page)
    for key in spamList:
        resultList.append(key)
        pass

    #the whole "AbstractFormater()" line is a bunch of crap I copied
    #That needs to be documented somehow, but I have no idea what it does
    try:
        parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        parser.feed(page)
        parser.close()

    except:
        #print "DEBUG: Caught an exception trying to parse that html file."
        #print "(Not sure why this happens - you'll have to crawl this page manually)"
        return resultList

    #print "Adding HTML Parser data"
    for key in parser.anchorlist:
        resultList.append(key)
        pass

    return resultList
コード例 #56
0
ファイル: csdnSpider.py プロジェクト: yinianfengchen/PYTHON
class Retriever(object): #download Web pages
    def __init__(self, url):
        self.url = url
        self.file = self.filename(url)
        
    def filename(self, url): 
        path=url

        path = re.sub("\W","_",path)
        path+=".html"
        return path
    
    def isForbidden(self):
        return 0;
    
    def isForbidden(self):
        return 0;
    
    def download(self):
        try:
            if True:
                retval = urlretrieve(self.url, self.file)
                javaGroupContent=JavaGroupContent.JavaGroupContent() 
                javaGroupContent.meet_page(self.url, self.file)
            else:
                retval = '*** INFO: no need to download '
        except IOError:
            retval = ('*** ERROR: invalid URL "%s"' % self.url,)
        return retval
        
    def parseAndGetLinks(self):
        self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
        try:
            self.parser.feed(open(self.file).read())
            self.parser.close()
        except IOError:
            pass
        return self.parser.anchorlist
コード例 #57
0
#coding:utf-8

import urllib2
from htmllib import HTMLParser
from formatter import NullFormatter
import os
import re

url_name = "http://b.hatena.ne.jp/hotentry"
html_data = urllib2.urlopen(url_name)
parser = HTMLParser(NullFormatter())

try:
    parser.feed(html_data.read())
except TypeError:
    print "type error"

pat = re.compile("^http.*")
for link in parser.anchorlist:
    x = pat.search(link)
    if x is not None:
        print x.group(0)

コード例 #58
0
def unescape(data):
    p = HTMLParser(None)
    p.save_bgn()
    p.feed(data)
    return p.save_end()