class Retriever(object):#下载网页类
def __init__(self,url):
self.url = url
self.file = self.filename(url)
def filename(self,url,deffile ="index.htm"):
parsedurl = urlparse(url,"http:",0) #解析路径
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == "": #如果没有文件，使用默认
if path[-1] == "/":
path += deffile
else:
path += "/" + deffile
ldir = dirname(path) #本地目录
if sep != "/":
ldir = replace(ldir,"/",sep)
if not isdir(ldir): #如果没有目录，创建一个
if exists(ldir):unlink(ldir)
makedirs(ldir)
return path
def download(self):# 下载网页
try:
retval = urlretrieve(self.url,self.file)
except IOError:
retval = ('***Error: invalid URL: "%s"' % self.url,)
return retval
def parseAndGetLinks(self): #解析HTML，保存链接
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(htmllib.HTMLParser): #download Web Pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url,'http:',0) #parse path
path = parsedurl[1] + parsedurl[2]
text = splitext(path)
if text[1]=='': #its not file use default
if text[-1] == '/':
path = path + deffile
else:
path = path + '/' + deffile
print "PATH:%s" % path
dir = dirname(path)
if not isdir(dir): #create new archieve dir if necessary
if exists(dir): unlink(dir)
makedirs(dir)
return path
def download(self): #download web pages
try:
retval = urlretrieve(self.url,self.file)
except IOError:
retval =('***ERROR: invalid URL "%s"' % self.url)
return retval
def parseAndGetLinks(self): #Parse HTML
self.parser = HTMLParser(AbstractFormatter(\
DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object): # download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http:', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '': # no file, use default
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): # create archive dir if nec.
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self): # download Web page
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url, )
return retval
def parseAndGetLinks(self): # parse HTML, save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object):
def __init__(self,url):
self.url=url
self.file=self.filename(url)
def filename(self,url,deffile='index.html'):
parsedurl=urlparse(url,'http:',0)
path=parsedurl[1]+parsedurl[2] #weibo.com+/gothack
ext=splitext(path) #weibo.com/gothack , '' #split by .
if ext[1]=='':
if path[-1]=='/':
path+=deffile
else:
path+='/'+deffile
ldir=dirname(path) #weibo.com #before the last /
if sep != '/': #default value is /
ldir=replce(ldir,'/',sep) #replace / with sep #(string,old,new)
if not isdir(ldir):
if exists(ldir):unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval=urlretrieve(self.url,self.file)
except IOError:
retval=('***ERROR: invalid URL "%s"' %self.url,)
return retval
def parseAndGetLinks(self):
self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def create_plaintext_message(message):
""" Create clean plain text version of email message
Parse the html and remove style and javacript tags and then
create a plain-text-message by parsing the html
and attaching links as endnotes
"""
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.kill_tags = ['style']
doc = message.decode('utf-8', 'ignore')
to_clean = lxml.html.fromstring(doc)
cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean))
plain_text_maxcols = 72
textout = cStringIO.StringIO()
formtext = formatter.AbstractFormatter(
formatter.DumbWriter(textout, plain_text_maxcols))
parser = HTMLParser(formtext)
parser.feed(cleaned_msg)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
for item in parser.anchorlist:
counter += 1
if item.startswith('https://'):
new_item = item.replace('https://', 'http://')
else:
new_item = item
anchorlist += "[%d] %s\n" % (counter, new_item)
text = textout.getvalue() + anchorlist
del textout, formtext, parser, anchorlist
return text
def create_plaintext_message(self, text):
""" Create a plain-text-message by parsing the html
and attaching links as endnotes
Modified from EasyNewsletter/content/ENLIssue.py
"""
# This reflows text which we don't want, but it creates
# parser.anchorlist which we do want.
textout = StringIO.StringIO()
formtext = formatter.AbstractFormatter(formatter.DumbWriter(textout))
parser = HTMLParser(formtext)
parser.feed(text)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + '----' + "\n\n"
for item in parser.anchorlist:
counter += 1
anchorlist += "[%d] %s\n" % (counter, item)
# This reflows text:
# text = textout.getvalue() + anchorlist
# This just strips tags, no reflow
text = html.fromstring(text).text_content()
text += anchorlist
del textout, formtext, parser, anchorlist
return text
class Retriever(object): #下载网页类
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile="index.htm"):
parsedurl = urlparse(url, "http:", 0) #解析路径
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == "": #如果没有文件，使用默认
if path[-1] == "/":
path += deffile
else:
path += "/" + deffile
ldir = dirname(path) #本地目录
if sep != "/":
ldir = replace(ldir, "/", sep)
if not isdir(ldir): #如果没有目录，创建一个
if exists(ldir): unlink(ldir)
makedirs(ldir)
return path
def download(self): # 下载网页
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('***Error: invalid URL: "%s"' % self.url, )
return retval
def parseAndGetLinks(self): #解析HTML，保存链接
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Restriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile = 'index.htm'):
parsedurl = urlparse(url, 'http:', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '':
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path)
if sep != '/':
ldir = replace(ldir, '/', sep)
if not isdir(ldir):
if exists(ldir) : unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url)
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object): # download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http:', 0) ## parse path
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '': # no file, use default
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): # create archive dir if nec.
if exists(ldir): unlink(ldir)
makedirs(ldir)
return path
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def get_plain_from_html(html):
textout = StringIO()
formtext = AbstractFormatter(DumbWriter(textout))
parser = HTMLParser(formtext)
parser.feed(html)
parser.close()
return textout.getvalue()
def create_plaintext_message(self, text):
""" Create a plain-text-message by parsing the html
and attaching links as endnotes
"""
plain_text_maxcols = 72
textout = cStringIO.StringIO()
formtext = formatter.AbstractFormatter(formatter.DumbWriter(
textout, plain_text_maxcols))
parser = HTMLParser(formtext)
parser.feed(text)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
for item in parser.anchorlist:
counter += 1
if item.startswith('https://'):
new_item = item.replace('https://', 'http://')
else:
new_item = item
anchorlist += "[%d] %s\n" % (counter, new_item)
text = textout.getvalue() + anchorlist
del textout, formtext, parser, anchorlist
return text
def OpenURL(self,url):
from htmllib import HTMLParser
import formatter
self.url = url
m = re.match('http://([^/]+)(/\S*)\s*', url)
if m:
host = m.groups()[0]
path = m.groups()[1]
else:
m = re.match('http://(\S+)\s*', url)
if not m:
# Invalid URL
self.logprint("Invalid or unsupported URL: %s" % (url))
return
host = m.groups()[0]
path = ''
f = self.RetrieveAsFile(host,path)
if not f:
self.logprint("Could not open %s" % (url))
return
self.logprint("Receiving data...")
data = f.read()
tmp = open('hangman_dict.txt','w')
fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp))
p = HTMLParser(fmt)
self.logprint("Parsing data...")
p.feed(data)
p.close()
tmp.close()
def create_plaintext_message(message):
""" Create clean plain text version of email message
Parse the html and remove style and javacript tags and then
create a plain-text-message by parsing the html
and attaching links as endnotes
"""
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.kill_tags = ['style']
doc = message.decode('utf-8', 'ignore')
to_clean = lxml.html.fromstring(doc)
cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean))
plain_text_maxcols = 72
textout = cStringIO.StringIO()
formtext = formatter.AbstractFormatter(formatter.DumbWriter(
textout, plain_text_maxcols))
parser = HTMLParser(formtext)
parser.feed(cleaned_msg)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
for item in parser.anchorlist:
counter += 1
if item.startswith('https://'):
new_item = item.replace('https://', 'http://')
else:
new_item = item
anchorlist += "[%d] %s\n" % (counter, new_item)
text = textout.getvalue() + anchorlist
del textout, formtext, parser, anchorlist
return text
class Retriever(object): # download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile = 'index.htm'):
parsedurl = urlparse(url, 'http:', 0) # parse path
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '': # no file, use default
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
if sep != '/':
ldir = replace(ldir, '/', sep)
if not isdir(ldir):
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url)
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def OpenURL(self, url):
from htmllib import HTMLParser
import formatter
self.url = url
m = re.match('http://([^/]+)(/\S*)\s*', url)
if m:
host = m.groups()[0]
path = m.groups()[1]
else:
m = re.match('http://(\S+)\s*', url)
if not m:
# Invalid URL
self.logprint("Invalid or unsupported URL: %s" % (url))
return
host = m.groups()[0]
path = ''
f = self.RetrieveAsFile(host, path)
if not f:
self.logprint("Could not open %s" % (url))
return
self.logprint("Receiving data...")
data = f.read()
tmp = open('hangman_dict.txt', 'w')
fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp))
p = HTMLParser(fmt)
self.logprint("Parsing data...")
p.feed(data)
p.close()
tmp.close()
class Retriever(object):
def __init__(self,url):
self.url=url
self.file=self.filename(url)
def filename(self,url,deffile='index.html'):
"""
生成下载连接和文件名
"""
full_url = ""
if url.endswith(DOM_SUFFIX):
full_url = url + '/'
else:
full_url = url
parsedurl=urlparse(full_url,'http:',0)
path=parsedurl[1]+parsedurl[2]
ext=splitext(path)
if ext[1]=='':
if path[-1]=='/':
path+=deffile
else:
path+='/'+deffile
ldir=dirname(path)
if sep!='/':
ldir=replace(ldir,'/',sep)
path=replace(path,'/',sep)
if not isdir(ldir):
if exists(ldir):
#unlink(ldir)
pass
else:
makedirs(ldir)
print path
return path
def download(self):
"""
下载文件
"""
try:
retval=urlretrieve(self.url,self.file)
except IOError:
retval=('***ERROR :invalid URL "%s"' %self.url,)
return retval
def parseAndGetLinks(self):
"""
获取网页中的链接
"""
#print 'Get Html Links from file:%s' % self.file
#self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO)))
self.parser=HTMLParser(NullFormatter())
#self.parser.feed(open(self.file).read())
try:
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
except:
print self.file + " error !"
return []
class Retrive(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.php'):
parsedurl = urlparse(url,'http:', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '':
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path)
if sep != '/':
ldir = replace(ldir, '/', sep)
if not isdir(ldir):
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = 'error'
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def html2text(htmldata):
# patch htmldata
htmldata = htmldata.replace("<br/>", "<br>")
fmt = HTMLtoTextFormatter()
prs = HTMLParser(fmt)
prs.feed(htmldata)
return fmt.getText()
def html2text(htmldata):
# patch htmldata
htmldata = htmldata.replace("<br/>", "<br>")
fmt = HTMLtoTextFormatter()
prs = HTMLParser(fmt)
prs.feed(htmldata)
return fmt.getText()
def get_urls(url):
data = urllib.urlopen(url).read()
parser = HTMLParser(
formatter.AbstractFormatter(formatter.DumbWriter(
cStringIO.StringIO())))
parser.feed(data)
parser.close()
url_list = parser.anchorlist
return url_list
def parse_links(self):
'Parse out the links found in downloaded HTML file'
f = open(self.file, 'r')
data = f.read()
f.close()
parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
parser.feed(data)
parser.close()
return parser.anchorlist
def _clean_text( self, text ):
try:
text = text.replace( " ", " " )
text = text.strip()
parser = HTMLParser( None )
parser.save_bgn()
parser.feed( text )
return parser.save_end()
except:
return text
def parse_links(self):
'Parse out the links found in downloaded HTML file'
f = open(self.file, 'r')
data = f.read()
f.close()
parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(
cStringIO.StringIO())))
parser.feed(data)
parser.close()
return parser.anchorlist
def parse_links(self):
"""fetch all links from page
"""
f = open(self.save_file, 'r')
data = f.read()
f.close()
parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
parser.feed(data)
parser.close()
return parser.anchorlist
class Retriever(object): def __init__(self,url): self.url = url #parse HTML ,save links def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(urlopen(self.url).read()) self.parser.close() return self.parser.anchorlist
def _clean_text(self, text):
try:
text = text.replace(" ", " ")
text = text.strip()
parser = HTMLParser(None)
parser.save_bgn()
parser.feed(text)
return parser.save_end()
except:
return text
def parse_links(self):
f = open(self.file, 'r')
data = f.read()
f.close()
paeser = HTMLParser(
formatter.AbstractFormatter(
formatter.DumbWriter(cStringIO.StringIO())))
paeser.feed(data)
paeser.close()
return paeser.anchorlist
def html2text(html):
f = StringIO()
parser = HTMLParser(AbstractFormatter(DumbWriter(f)))
try:
parser.feed(html)
except HTMLParseError:
return ''
else:
parser.close()
return f.getvalue()
def get_text_from_html( html_input ):
"Strip tags and non-ascii characters from HTML input."
my_stringio = StringIO.StringIO() # make an instance of this file-like string thing
p = HTMLParser(AbstractFormatter(DumbWriter(my_stringio)))
try: p.feed(html_input); p.close() #calling close is not usually needed, but let's play it safe
except HTMLParseError: print '***HTML malformed***' #the html is badly malformed (or you found a bug)
#return my_stringio.getvalue().replace('\xa0','')
s = re.sub( r'[^\x00-\x7f]', r' ', my_stringio.getvalue() )
s = s.replace('\r\n',' ').replace('\n',' ')
s = re.sub( ' +', ' ', s )
return s
def parse_link(seld):
'Parse out the link'
f = open('seld.file', 'r')
data = f.read()
f.close()
parser = HTMLParser(
formatter.AbstractFormatter(
formatter.DumbWriter(cStringIO.StringIO())))
parser.feed(data)
parser.close()
return parser.anchorlist
def parse_links(self):
"""fetch all links from page
"""
f = open(self.save_file, 'r')
data = f.read()
f.close()
parser = HTMLParser(
formatter.AbstractFormatter(
formatter.DumbWriter(cStringIO.StringIO())))
parser.feed(data)
parser.close()
return parser.anchorlist
def parse_links(self):
f = open(self.file, "r")
data = f.read()
f.close()
parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
# parser = MyHTMLParser()
parser.feed(data)
parser.close()
# 没有在模块中找到 anchorlist属性
# 返回页面中所有的锚点，href
# 该属性在2.6后就被弃用了，如果想用可以通过自定义parser继承HTMLParser来实现，参见evernote.或者用第三方库beautifulsoup
return parser.anchorlist
def parse_html(self, html):
from StringIO import StringIO
from formatter import (AbstractFormatter, DumbWriter)
from htmllib import HTMLParser
_html = re.sub(self.notrans_tag, r" \1 ", html)
buf = StringIO()
p = HTMLParser(AbstractFormatter(DumbWriter(buf)))
p.feed(_html)
_sub = re.sub(self.whitespaces, " ", buf.getvalue())
# FIXME: how can zerowidth be removed more simply?
_sub = re.sub(self.zerowidth, "", _sub)
_sub = re.sub(self.colon, r"\1", _sub)
return _sub
def parse(self, url):
self.base = ""
self.href = ""
m = re.compile(".*/").match(url)
if m != None:
self.base = m.string[m.start(0):m.end(0)]
result = urlfetch.fetch(url, headers = {'Cache-Control' : 'max-age=30', 'Pragma' : 'no-cache'} )
if result.status_code == 200:
logging.debug(str(result.status_code) + " OK " + url)
HTMLParser.feed(self, result.content)
HTMLParser.close(self)
else:
logging.error(str(result.status_code) + " NG " + url)
class Retriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
# 使用http协议解析url地址，将url解析为这样的五元组：(scheme, netloc, path, query, fragment)
parsedurl = urlparse(url, 'http:', 0)
# 将主机地址（netloc）和路径（path）合并起来作为存储文件的路径名
path = parsedurl[1] + parsedurl[2]
# splitext将path分割为路径与后缀名，如（/path/to/file, txt）
ext = splitext(path)
# 如果后缀名为空，则给当前path添加默认的名字index.htm
if ext[1] == '':
# 若path结尾有“/”则直接添加index.htm，否则先添加“/”
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
# 取出path中的目录部分（www.shellbye.com\\blog），然后与本地目录结合为一个目录('D:\\www.shellbye.com\\blog')
ldir = dirname(abspath(path))
# 如果不是以“/”作为目录分割符的类Unix系统，比如Windows，则需要把“”替换为相应的目录分割符
# 因为类Unix系统的目录分割符与URI地址的分割符一样，所以可以不处理
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
# 如果ldir目录不存在在创建
if not isdir(ldir): # create archive dir if nec.
# 如果ldir存在但是不是目录则删除ldir。注，unlink即remove。
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self): # download Web page
try:
# 下载self.url到self.file里
retval = urllib.urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url, )
return retval
def parseAndGetLinks(self):
# 创建一个基本的HTML解释器，可能需要单独一篇文章来说这句
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
# 解析html文件，获取所有的连接（带有href的）
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object):#download web page
def __init__(self, url):
self.url = url
self.file = self.filename(url)
#local filename ,directory
def filename(self, url, deffile = "index.htm"):
parseurl = urlparse(url, 'http:', 0)
path = parseurl[1] + parseurl[2]
#将路径转换为一个元组，如果为目录则第二个元素为空，如果文件则第二个元素为文件扩展名
#path = "D:/pycharmProjects/PythonWebApp/weblearning/Crawl.py"
#print splitext(path)
#('D:/pycharmProjects/PythonWebApp/weblearning/Crawl', '.py')
ext = splitext(path)
if ext[1] == '':#no file use default
#tuple[-index],倒数第index个元素
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
#获取path的dir,
# path=D:/pycharmProjects/PythonWebApp/webLearning
# dir(path)=D:/pycharmProjects/PythonWebApp
ldir = dirname(path)#local directory
if sep != '/': #os-indep. path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): #create archive dir if nec.
#如果存在文件，则删除
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self):
#urlretrieve()返回一个2元组，（filename,mime_hdrs）
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' %self.url,)
print 'erro,invalid url'
return retval
def parseAndGetLinks(self):#parse HTML , save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
# 使用http协议解析url地址，将url解析为这样的五元组：(scheme, netloc, path, query, fragment)
parsedurl = urlparse(url, 'http:', 0)
# 将主机地址（netloc）和路径（path）合并起来作为存储文件的路径名
path = parsedurl[1] + parsedurl[2]
# splitext将path分割为路径与后缀名，如（/path/to/file, txt）
ext = splitext(path)
# 如果后缀名为空，则给当前path添加默认的名字index.htm
if ext[1] == '':
# 若path结尾有“/”则直接添加index.htm，否则先添加“/”
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
# 取出path中的目录部分（www.shellbye.com\\blog），然后与本地目录结合为一个目录('D:\\www.shellbye.com\\blog')
ldir = dirname(abspath(path))
# 如果不是以“/”作为目录分割符的类Unix系统，比如Windows，则需要把“”替换为相应的目录分割符
# 因为类Unix系统的目录分割符与URI地址的分割符一样，所以可以不处理
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
# 如果ldir目录不存在在创建
if not isdir(ldir): # create archive dir if nec.
# 如果ldir存在但是不是目录则删除ldir。注，unlink即remove。
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self): # download Web page
try:
# 下载self.url到self.file里
retval = urllib.urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url, )
return retval
def parseAndGetLinks(self):
# 创建一个基本的HTML解释器，可能需要单独一篇文章来说这句
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
# 解析html文件，获取所有的连接（带有href的）
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def get_plain_from_html(html):
"""extract plain text from html
>>> test_html = "<div><h1>Hey<h1><p>This is some text</p></div>"
>>> get_plain_from_html(test_html)
'\\nHey\\n\\nThis is some text'
"""
from htmllib import HTMLParser # import here to avoid high startup cost
textout = StringIO()
formtext = AbstractFormatter(DumbWriter(textout))
parser = HTMLParser(formtext)
parser.feed(html)
parser.close()
return textout.getvalue()
def get_plain_from_html(html):
"""extract plain text from html
>>> test_html = "<div><h1>Hey<h1><p>This is some text</p></div>"
>>> get_plain_from_html(test_html)
'\\nHey\\n\\nThis is some text'
"""
from htmllib import HTMLParser # import here to avoid high startup cost
textout = StringIO()
formtext = AbstractFormatter(DumbWriter(textout))
parser = HTMLParser(formtext)
parser.feed(html)
parser.close()
return textout.getvalue()
class Retriever:
'''
responsibilities:
download, parse and queue
'''
def __init__(self,url):
'''
contructor of class.Instantiates the Retriver object and stores the url and filename as local attributes
'''
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile = 'index.html'):
'''
input: url
removes the http prefix
index.html will be the default file name for storage of the url:this can be overridden by passing arguments to filename()
'''
parsedurl = urlparse(url,"http:",0) #parse path
path = parsedurl[1] + parsedurl[2]
text = splitext(path)
if text[1] == '': #no file, use default
path = path + deffile
else:
path = path + '/' + deffile
dir = dirname(path)
if not isdir(dir): #create a new directory if necessary
if exists(dir): unlink(dir)
makedirs(dir)
return path
def download(self): #download web page
try:
retval = urlretrieve(self.url,self.file)
except IOError:
retval = ('***ERROR invalid url "%s"'%self.url,)
return retval
def parseAndGetLinks(self): #parse HTML and getlinks
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
#try:
self.parser.feed(open(self.file).read())
#except HTMLParseError:
self.parser.close()
self.parser.close()
return self.parser.anchorlist
def parse(self, url):
self.base = ""
self.href = ""
m = re.compile(".*/").match(url)
if m != None:
self.base = m.string[m.start(0):m.end(0)]
result = urlfetch.fetch(url,
headers={
'Cache-Control': 'max-age=30',
'Pragma': 'no-cache'
})
if result.status_code == 200:
logging.debug(str(result.status_code) + " OK " + url)
HTMLParser.feed(self, result.content)
HTMLParser.close(self)
else:
logging.error(str(result.status_code) + " NG " + url)
class Retriever(): # download web pages
def __init__(self, url):
self.url = url
def download(self): # download web page
print 'try to open url:', self.url, '\nthe true url process', string.split(
self.url, '?')[0]
try:
retval = urlopen(string.split(self.url, '?')[0], None, 200)
except urllib2.HTTPError as e:
print "HTTPError", e
return
except socket.timeout as e:
print "socket.timeout", e
return
except socket.error as e:
print "socket.error", e
return
except urllib2.URLError as e:
print "URLError: ", e
return
return retval
def parseAndGetLinks(self): # parse HTML, save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
r = self.download()
if r:
print '________'
try:
try:
s = r.read(50000)
except socket.error as e:
print "***************************socket error***************************", e
return []
self.parser.feed(s)
print '------------------'
r.close()
print '***************************'
except HTMLParseError:
print 'get links error\n'
return []
self.parser.close()
return self.parser.anchorlist
def compactor(dev_filename, rel_filename):
# Use compactor to generate release version.
echo('Compacting: %s -> %s' % (dev_filename, rel_filename))
source_data = open(dev_filename, 'r').read()
try:
# Verify that the html file is correct
htmlparser = HTMLParser(NullFormatter())
htmlparser.feed(source_data)
htmlparser.close()
# Now try to minify
output_file = open(rel_filename, 'wb')
compactor = HTMLMinifier(output_file.write, True)
compactor.feed(source_data)
compactor.close()
output_file.close()
except HTMLParseError as e:
error(str(e))
exit(1)
def compactor(dev_filename, rel_filename):
# Use compactor to generate release version.
echo('Compacting: %s -> %s' % (dev_filename, rel_filename))
source_data = open(dev_filename, 'r').read()
try:
# Verify that the html file is correct
htmlparser = HTMLParser(NullFormatter())
htmlparser.feed(source_data)
htmlparser.close()
# Now try to minify
output_file = open(rel_filename, 'wb')
compactor = HTMLMinifier(output_file.write, True)
compactor.feed(source_data)
compactor.close()
output_file.close()
except HTMLParseError as e:
error(str(e))
exit(1)
class LinkFinder(object):
def __init__(self, base_url, page_url):
self.base_url = base_url
self.page_url = page_url
def parseAndGetLinks(self, html_string):
try:
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(html_string)
self.parser.close()
links = []
for eachLink in self.parser.anchorlist:
if eachLink[:4] != "http" and find(eachLink, "://") == -1:
eachLink = urljoin(self.base_url, eachLink)
links.append(eachLink)
return links
except IOError:
return []
class Retriever(): # download web pages
def __init__(self, url):
self.url = url
def download(self): # download web page
print 'try to open url:', self.url, '\nthe true url process', string.split(self.url, '?')[0]
try:
retval = urlopen(string.split(self.url, '?')[0], None, 200)
except urllib2.HTTPError as e:
print "HTTPError", e
return
except socket.timeout as e:
print "socket.timeout", e
return
except socket.error as e:
print "socket.error", e
return
except urllib2.URLError as e:
print "URLError: ", e
return
return retval
def parseAndGetLinks(self): # parse HTML, save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
r = self.download()
if r:
print '________'
try:
try:
s = r.read(50000)
except socket.error as e:
print "***************************socket error***************************", e
return []
self.parser.feed(s)
print '------------------'
r.close()
print '***************************'
except HTMLParseError:
print 'get links error\n'
return []
self.parser.close()
return self.parser.anchorlist
def collectURLSFromPage(page):
"""
This returns a list of URLS that come from a certain page.
Useful for spiders. It takes just a string as an argument.
"""
resultList = []
if page == "":
#nothing to parse, so nothing to return
return resultList
#print "Doing form parser"
if page.count("<form") > 0:
otherlist = daveFormParse(page)
for key in otherlist:
resultList.append(key)
pass
#DEBUG
#return resultList
#print "Doing RAW Parser"
spamList = rawParse(page)
for key in spamList:
resultList.append(key)
pass
#This needs to be documented somehow, but I have no idea what it does
try:
parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
parser.feed(page)
parser.close()
except:
#print "DEBUG: Caught an exception trying to parse that html file."
#print "(Not sure why this happens - you'll have to crawl this page manually)"
return resultList
#print "Adding HTML Parser data"
for key in parser.anchorlist:
resultList.append(key)
pass
return resultList
class Retriever(object): # download Web pages
"""docstring for Retriever"""
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.html'):
parsedurl = urlparse(url, 'http:', 0) # parse path
print '====PARSEDURL====',parsedurl
if parsedurl[2] == '':
path = parsedurl[1] + '/'
else:
path = parsedurl[1] + parsedurl[2]
print '------PATH-----', path
ext = splitext(path)
print '-----EXT----', ext
if ext[1] == '': # no file, use default
if path[-1] == '/':
path +=deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
print '+++++++++++++++++', ldir
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): # create archive dir if nec.
if exists(ldir): unlink(ldir)
makedirs(ldir)
return path
def download(self): # download Web page
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url,)
return retval
def parseAndGetLinks(self): # parse HTML, save links
# self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO)))
self.parser = HTMLParser(AbstractFormatter(DumbWriter()))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object): # download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http:', 0) # parse path
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
print path
if ext[1] == '':
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
if sep != '/': # os-indep. path separator
ldir = replace(ldir, ',', sep)
if not isdir(ldir): # create archive dir if nec.
if exists(ldir): unlink(ldir)
print ldir, "aaaaaaaaa"
makedirs(ldir)
return path
def download(self): # download Web page
try:
retval = urllib.urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % \
self.url, )
return retval
def parseAndGetLinks(self): # pars HTML, save links
self.parser = HTMLParser(AbstractFormatter( \
DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
print self.parser
return self.parser.anchorlist
class LinkFinder(object):
def __init__(self,base_url,page_url):
self.base_url = base_url
self.page_url = page_url
def parseAndGetLinks(self,html_string):
try:
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(html_string)
self.parser.close()
links = []
for eachLink in self.parser.anchorlist:
if eachLink[:4] != "http" and find(eachLink, "://") == -1:
eachLink = urljoin(self.base_url, eachLink)
links.append(eachLink)
return links
except IOError:
return []
class Retriever(object):
'download web pages'
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http:', 0)
path = parsedurl[1] + parsedurl[2]
print path
ext = splitext(path)#return (filename, extension)
if ext[1] == '': #no file, use default
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) #local directory
if sep != '/': #os-indep.path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): #create archive dir if nec
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self):
'download web page'
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: Invalid URL "%s"' %self.url)
return retval
def parseAndGetLinks(self):
'parse HTML, save links'
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object):
def __init__(self,url):
self.url=url
self.file=self.filename(url)
def filename(self,url,deffile='index.htm'):
parsedurl=urlparse(url,'http:',0)
path=parsedurl[1]+parsedurl[2]
ext=splitext(path)
# 爬取的必须是静态html
if ext[1]=='':
if path[-1]=='/':
path+=deffile
else:
path+='/'+deffile
#print path
# 建立文件目录
ldir=dirname(path)
#print ldir
#将url中的/转为windows的
if sep!='/':
ldir=replace(ldir,'/',sep)
if not isdir(ldir):
if exists(ldir): unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval=urlretrieve(self.url,self.file)
except IOError:
retval=('***error in url "%s"'%self.url)
return retval
def parseAndGetLink(self):
#构造一个解析器
self.parser= HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
print self.file
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path) #分解文件名的扩展名
print path,ext
if ext[1] == '':
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
print path
ldir = dirname(path)
#应该和操作系统相关 windows下：sep = \
if sep != '/':
ldir = replace(ldir, '/', sep)
print ldir
if not isdir(ldir):
if exists(ldir):
return
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** Error URL "%s"' % self.url)
return retval
def parseAndGetLinks(self):
"""StringIO是从内存中读取数据 DumbWriter将事件流转换为存文本文档 AbstractFormatter 类进行格式化
"""
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def _create_plaintext_message(self, text):
""" Create a plain-text-message by parsing the html
and attaching links as endnotes
"""
plain_text_maxcols = 72
textout = cStringIO.StringIO()
formtext = formatter.AbstractFormatter(
formatter.DumbWriter(textout, plain_text_maxcols))
parser = HTMLParser(formtext)
parser.feed(text)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
for counter, item in enumerate(parser.anchorlist):
anchorlist += "[{0:d}] {1:s}\n".format(counter, item)
text = textout.getvalue() + anchorlist
del textout, formtext, parser, anchorlist
return text
def collectURLSFromPage(page):
resultList = []
#print "Doing form parser"
if page.count("<form") > 0:
otherlist = daveFormParse(page)
for key in otherlist:
resultList.append(key)
pass
#DEBUG
#return resultList
#print "Doing RAW Parser"
spamList = rawParse(page)
for key in spamList:
resultList.append(key)
pass
#the whole "AbstractFormater()" line is a bunch of crap I copied
#That needs to be documented somehow, but I have no idea what it does
try:
parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
parser.feed(page)
parser.close()
except:
#print "DEBUG: Caught an exception trying to parse that html file."
#print "(Not sure why this happens - you'll have to crawl this page manually)"
return resultList
#print "Adding HTML Parser data"
for key in parser.anchorlist:
resultList.append(key)
pass
return resultList
class Retriever(object): #download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url):
path=url
path = re.sub("\W","_",path)
path+=".html"
return path
def isForbidden(self):
return 0;
def isForbidden(self):
return 0;
def download(self):
try:
if True:
retval = urlretrieve(self.url, self.file)
javaGroupContent=JavaGroupContent.JavaGroupContent()
javaGroupContent.meet_page(self.url, self.file)
else:
retval = '*** INFO: no need to download '
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url,)
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
try:
self.parser.feed(open(self.file).read())
self.parser.close()
except IOError:
pass
return self.parser.anchorlist
#coding:utf-8
import urllib2
from htmllib import HTMLParser
from formatter import NullFormatter
import os
import re
url_name = "http://b.hatena.ne.jp/hotentry"
html_data = urllib2.urlopen(url_name)
parser = HTMLParser(NullFormatter())
try:
parser.feed(html_data.read())
except TypeError:
print "type error"
pat = re.compile("^http.*")
for link in parser.anchorlist:
x = pat.search(link)
if x is not None:
print x.group(0)
def unescape(data):
p = HTMLParser(None)
p.save_bgn()
p.feed(data)
return p.save_end()
