class Retriever(object):
def __init__(self,url):
self.url=url
self.file=self.filename(url)
def filename(self,url,deffile='index.html'):
parsedurl=urlparse(url,'http:',0)
path=parsedurl[1]+parsedurl[2] #weibo.com+/gothack
ext=splitext(path) #weibo.com/gothack , '' #split by .
if ext[1]=='':
if path[-1]=='/':
path+=deffile
else:
path+='/'+deffile
ldir=dirname(path) #weibo.com #before the last /
if sep != '/': #default value is /
ldir=replce(ldir,'/',sep) #replace / with sep #(string,old,new)
if not isdir(ldir):
if exists(ldir):unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval=urlretrieve(self.url,self.file)
except IOError:
retval=('***ERROR: invalid URL "%s"' %self.url,)
return retval
def parseAndGetLinks(self):
self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def create_plaintext_message(self, text):
""" Create a plain-text-message by parsing the html
and attaching links as endnotes
Modified from EasyNewsletter/content/ENLIssue.py
"""
# This reflows text which we don't want, but it creates
# parser.anchorlist which we do want.
textout = StringIO.StringIO()
formtext = formatter.AbstractFormatter(formatter.DumbWriter(textout))
parser = HTMLParser(formtext)
parser.feed(text)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + '----' + "\n\n"
for item in parser.anchorlist:
counter += 1
anchorlist += "[%d] %s\n" % (counter, item)
# This reflows text:
# text = textout.getvalue() + anchorlist
# This just strips tags, no reflow
text = html.fromstring(text).text_content()
text += anchorlist
del textout, formtext, parser, anchorlist
return text
class Retriever(object): # download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http:', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '': # no file, use default
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): # create archive dir if nec.
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self): # download Web page
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url, )
return retval
def parseAndGetLinks(self): # parse HTML, save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object): #下载网页类
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile="index.htm"):
parsedurl = urlparse(url, "http:", 0) #解析路径
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == "": #如果没有文件，使用默认
if path[-1] == "/":
path += deffile
else:
path += "/" + deffile
ldir = dirname(path) #本地目录
if sep != "/":
ldir = replace(ldir, "/", sep)
if not isdir(ldir): #如果没有目录，创建一个
if exists(ldir): unlink(ldir)
makedirs(ldir)
return path
def download(self): # 下载网页
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('***Error: invalid URL: "%s"' % self.url, )
return retval
def parseAndGetLinks(self): #解析HTML，保存链接
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def __init__(self, funFormatter, objHere):
self.objH1 = None
self.objH2 = None
self.objH3 = None
self.dodHelp = GetDOD(objHere, "E3Help")
HTMLParser.__init__(self, funFormatter)
class Retrive(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.php'):
parsedurl = urlparse(url,'http:', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '':
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path)
if sep != '/':
ldir = replace(ldir, '/', sep)
if not isdir(ldir):
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = 'error'
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter):
HTMLParser.__init__(self, fmt)
self.result = ""
self.open_tags = []
# A list of the only tags allowed. Be careful adding to this. Adding
# "script," for example, would not be smart. 'img' is out by default
# because of the danger of IMG embedded commands, and/or web bugs.
self.permitted_tags = ['a', 'b', 'blockquote', 'br', 'i',
'li', 'ol', 'ul', 'p', 'cite']
# A list of tags that require no closing tag.
self.requires_no_close = ['img', 'br']
# A dictionary showing the only attributes allowed for particular tags.
# If a tag is not listed here, it is allowed no attributes. Adding
# "on" tags, like "onhover," would not be smart. Also be very careful
# of "background" and "style."
self.allowed_attributes = \
{'a':['href', 'title'],
'img':['src', 'alt'],
'blockquote':['type']}
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ['http', 'https', 'ftp']
def __init__(
self,
permitted_tags=['a', 'b', 'blockquote', 'br/', 'i',
'li', 'ol', 'ul', 'p', 'cite',
'code', 'pre', 'img/',],
allowed_attributes={
'a': ['href', 'title'],
'img': ['src', 'alt'],
'blockquote': ['type']},
fmt=AbstractFormatter,
strip_disallowed=False
):
HTMLParser.__init__(self, fmt)
self.result = ''
self.open_tags = []
self.permitted_tags = [i for i in permitted_tags if i[-1] != '/']
self.requires_no_close = [i[:-1] for i in permitted_tags
if i[-1] == '/']
self.permitted_tags += self.requires_no_close
self.allowed_attributes = allowed_attributes
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ['http', 'https', 'ftp']
#to strip or escape disallowed tags?
self.strip_disallowed = strip_disallowed
self.in_disallowed = False
def __init__(self, fmt = AbstractFormatter):
HTMLParser.__init__(self, fmt)
self.result = ""
self.open_tags = []
# A list of forbidden tags.
self.forbidden_tags = ['script', 'embed', 'iframe', 'frame' ]
# A list of tags that require no closing tag.
self.requires_no_close = ['img', 'br']
# A dictionary showing the only attributes allowed for particular tags.
# If a tag is not listed here, it is allowed no attributes. Adding
# "on" tags, like "onhover," would not be smart. Also be very careful
# of "background" and "style."
self.allowed_attributes =\
{'a':['href','title','target','style'],
'img':['src','alt','border','style'],
'blockquote':['type','style'],
'table': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'tbody': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'tr': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'td': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'div': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'span': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
}
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ['http','https','ftp']
def parseAndGetLinks(self):
"""StringIO是从内存中读取数据 DumbWriter将事件流转换为存文本文档 AbstractFormatter 类进行格式化
"""
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def parseAndGetLinks(self):
# 创建一个基本的HTML解释器，可能需要单独一篇文章来说这句
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
# 解析html文件，获取所有的连接（带有href的）
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def get_plain_from_html(html):
textout = StringIO()
formtext = AbstractFormatter(DumbWriter(textout))
parser = HTMLParser(formtext)
parser.feed(html)
parser.close()
return textout.getvalue()
def create_plaintext_message(self, text):
""" Create a plain-text-message by parsing the html
and attaching links as endnotes
"""
plain_text_maxcols = 72
textout = cStringIO.StringIO()
formtext = formatter.AbstractFormatter(formatter.DumbWriter(
textout, plain_text_maxcols))
parser = HTMLParser(formtext)
parser.feed(text)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
for item in parser.anchorlist:
counter += 1
if item.startswith('https://'):
new_item = item.replace('https://', 'http://')
else:
new_item = item
anchorlist += "[%d] %s\n" % (counter, new_item)
text = textout.getvalue() + anchorlist
del textout, formtext, parser, anchorlist
return text
class Retriever(object): # download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http:', 0) ## parse path
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '': # no file, use default
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): # create archive dir if nec.
if exists(ldir): unlink(ldir)
makedirs(ldir)
return path
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def create_plaintext_message(message):
""" Create clean plain text version of email message
Parse the html and remove style and javacript tags and then
create a plain-text-message by parsing the html
and attaching links as endnotes
"""
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.kill_tags = ['style']
doc = message.decode('utf-8', 'ignore')
to_clean = lxml.html.fromstring(doc)
cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean))
plain_text_maxcols = 72
textout = cStringIO.StringIO()
formtext = formatter.AbstractFormatter(formatter.DumbWriter(
textout, plain_text_maxcols))
parser = HTMLParser(formtext)
parser.feed(cleaned_msg)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
for item in parser.anchorlist:
counter += 1
if item.startswith('https://'):
new_item = item.replace('https://', 'http://')
else:
new_item = item
anchorlist += "[%d] %s\n" % (counter, new_item)
text = textout.getvalue() + anchorlist
del textout, formtext, parser, anchorlist
return text
class Retriever(object):#下载网页类
def __init__(self,url):
self.url = url
self.file = self.filename(url)
def filename(self,url,deffile ="index.htm"):
parsedurl = urlparse(url,"http:",0) #解析路径
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == "": #如果没有文件，使用默认
if path[-1] == "/":
path += deffile
else:
path += "/" + deffile
ldir = dirname(path) #本地目录
if sep != "/":
ldir = replace(ldir,"/",sep)
if not isdir(ldir): #如果没有目录，创建一个
if exists(ldir):unlink(ldir)
makedirs(ldir)
return path
def download(self):# 下载网页
try:
retval = urlretrieve(self.url,self.file)
except IOError:
retval = ('***Error: invalid URL: "%s"' % self.url,)
return retval
def parseAndGetLinks(self): #解析HTML，保存链接
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def OpenURL(self,url):
from htmllib import HTMLParser
import formatter
self.url = url
m = re.match('http://([^/]+)(/\S*)\s*', url)
if m:
host = m.groups()[0]
path = m.groups()[1]
else:
m = re.match('http://(\S+)\s*', url)
if not m:
# Invalid URL
self.logprint("Invalid or unsupported URL: %s" % (url))
return
host = m.groups()[0]
path = ''
f = self.RetrieveAsFile(host,path)
if not f:
self.logprint("Could not open %s" % (url))
return
self.logprint("Receiving data...")
data = f.read()
tmp = open('hangman_dict.txt','w')
fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp))
p = HTMLParser(fmt)
self.logprint("Parsing data...")
p.feed(data)
p.close()
tmp.close()
class Retriever(object): # download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile = 'index.htm'):
parsedurl = urlparse(url, 'http:', 0) # parse path
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '': # no file, use default
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path) # local directory
if sep != '/':
ldir = replace(ldir, '/', sep)
if not isdir(ldir):
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url)
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def parseAndGetLinks(self): # pars HTML, save links
self.parser = HTMLParser(AbstractFormatter( \
DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
print self.parser
return self.parser.anchorlist
class Restriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile = 'index.htm'):
parsedurl = urlparse(url, 'http:', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path)
if ext[1] == '':
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
ldir = dirname(path)
if sep != '/':
ldir = replace(ldir, '/', sep)
if not isdir(ldir):
if exists(ldir) : unlink(ldir)
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url)
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter):
HTMLParser.__init__(self, fmt)
self.result = ""
self.open_tags = []
# A list of forbidden tags.
self.forbidden_tags = ["script", "embed", "iframe", "frame"]
# A list of tags that require no closing tag.
self.requires_no_close = ["img", "br"]
# A dictionary showing the only attributes allowed for particular tags.
# If a tag is not listed here, it is allowed no attributes. Adding
# "on" tags, like "onhover," would not be smart. Also be very careful
# of "background" and "style."
self.allowed_attributes = {
"a": ["href", "title", "target", "style"],
"img": ["src", "alt", "border", "style"],
"blockquote": ["type", "style"],
"font": ["size", "face"],
"h5": ["style"],
"h4": ["style"],
"h3": ["style"],
"h2": ["style"],
"h1": ["style"],
"table": ["border", "width", "height", "style", "align", "bgcolor"],
"tbody": ["border", "width", "height", "style", "align", "bgcolor"],
"tr": ["border", "width", "height", "style", "align", "bgcolor"],
"td": ["border", "width", "height", "style", "align", "bgcolor"],
"div": ["border", "width", "height", "style", "align", "bgcolor"],
"span": ["border", "width", "height", "style", "align", "bgcolor"],
}
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ["http", "https", "ftp"]
def reset(self):
self._data_is_title = False
self._data_is_comment = False
self.output = None
self._current = None
self._container = []
HTMLParser.reset(self)
class Retriever(object):
def __init__(self,url):
self.url=url
self.file=self.filename(url)
def filename(self,url,deffile='index.html'):
"""
生成下载连接和文件名
"""
full_url = ""
if url.endswith(DOM_SUFFIX):
full_url = url + '/'
else:
full_url = url
parsedurl=urlparse(full_url,'http:',0)
path=parsedurl[1]+parsedurl[2]
ext=splitext(path)
if ext[1]=='':
if path[-1]=='/':
path+=deffile
else:
path+='/'+deffile
ldir=dirname(path)
if sep!='/':
ldir=replace(ldir,'/',sep)
path=replace(path,'/',sep)
if not isdir(ldir):
if exists(ldir):
#unlink(ldir)
pass
else:
makedirs(ldir)
print path
return path
def download(self):
"""
下载文件
"""
try:
retval=urlretrieve(self.url,self.file)
except IOError:
retval=('***ERROR :invalid URL "%s"' %self.url,)
return retval
def parseAndGetLinks(self):
"""
获取网页中的链接
"""
#print 'Get Html Links from file:%s' % self.file
#self.parser=HTMLParser(AbstractFormatter(DumbWriter(StringIO)))
self.parser=HTMLParser(NullFormatter())
#self.parser.feed(open(self.file).read())
try:
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
except:
print self.file + " error !"
return []
def __init__(
self,
permitted_tags=[
'a',
'b',
'blockquote',
'br/',
'i',
'li',
'ol',
'ul',
'p',
'cite',
'code',
'pre',
'img/',
],
allowed_attributes={'a': ['href', 'title'], 'img': ['src', 'alt'
], 'blockquote': ['type']},
fmt=AbstractFormatter,
):
HTMLParser.__init__(self, fmt)
self.result = ''
self.open_tags = []
self.permitted_tags = [i for i in permitted_tags if i[-1] != '/']
self.requires_no_close = [i[:-1] for i in permitted_tags
if i[-1] == '/']
self.permitted_tags += self.requires_no_close
self.allowed_attributes = allowed_attributes
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ['http', 'https', 'ftp']
class Retriever(htmllib.HTMLParser): #download Web Pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url,'http:',0) #parse path
path = parsedurl[1] + parsedurl[2]
text = splitext(path)
if text[1]=='': #its not file use default
if text[-1] == '/':
path = path + deffile
else:
path = path + '/' + deffile
print "PATH:%s" % path
dir = dirname(path)
if not isdir(dir): #create new archieve dir if necessary
if exists(dir): unlink(dir)
makedirs(dir)
return path
def download(self): #download web pages
try:
retval = urlretrieve(self.url,self.file)
except IOError:
retval =('***ERROR: invalid URL "%s"' % self.url)
return retval
def parseAndGetLinks(self): #Parse HTML
self.parser = HTMLParser(AbstractFormatter(\
DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter):
HTMLParser.__init__(self, fmt)
self.result = ""
self.open_tags = []
# A list of the only tags allowed. Be careful adding to this. Adding
# "script," for example, would not be smart. 'img' is out by default
# because of the danger of IMG embedded commands, and/or web bugs.
self.permitted_tags = [
'a', 'b', 'blockquote', 'br', 'i', 'li', 'ol', 'ul', 'p', 'cite'
]
# A list of tags that require no closing tag.
self.requires_no_close = ['img', 'br']
# A dictionary showing the only attributes allowed for particular tags.
# If a tag is not listed here, it is allowed no attributes. Adding
# "on" tags, like "onhover," would not be smart. Also be very careful
# of "background" and "style."
self.allowed_attributes = \
{'a':['href','title'],
'img':['src','alt'],
'blockquote':['type']}
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ['http', 'https', 'ftp']
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
try:
self.parser.feed(open(self.file).read())
self.parser.close()
except IOError:
pass
return self.parser.anchorlist
def html2text(htmldata):
# patch htmldata
htmldata = htmldata.replace("<br/>", "<br>")
fmt = HTMLtoTextFormatter()
prs = HTMLParser(fmt)
prs.feed(htmldata)
return fmt.getText()
def html2text(htmldata):
# patch htmldata
htmldata = htmldata.replace("<br/>", "<br>")
fmt = HTMLtoTextFormatter()
prs = HTMLParser(fmt)
prs.feed(htmldata)
return fmt.getText()
def __init__(self, formatter, path, output):
HTMLParser.__init__(self, formatter)
self.path = path # relative path
self.ft = output # output file
self.indent = 0 # number of tabs for pretty printing of files
self.proc = False # True when actively processing, else False
# (headers, footers, etc)
# XXX This shouldn't need to be a stack -- anchors shouldn't nest.
# XXX See SF bug <http://www.python.org/sf/546579>.
self.hrefstack = [] # stack of hrefs from anchor begins
class Retriever(object): def __init__(self,url): self.url = url #parse HTML ,save links def parseAndGetLinks(self): self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO()))) self.parser.feed(urlopen(self.url).read()) self.parser.close() return self.parser.anchorlist
def __init__(self, formatter, path, output):
HTMLParser.__init__(self, formatter)
self.path = path # relative path
self.ft = output # output file
self.indent = 0 # number of tabs for pretty printing of files
self.proc = False # True when actively processing, else False
# (headers, footers, etc)
# XXX This shouldn't need to be a stack -- anchors shouldn't nest.
# XXX See SF bug <http://www.python.org/sf/546579>.
self.hrefstack = [] # stack of hrefs from anchor begins
def parse_links(self):
"""fetch all links from page
"""
f = open(self.save_file, 'r')
data = f.read()
f.close()
parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
parser.feed(data)
parser.close()
return parser.anchorlist
def parse_links(self):
'Parse out the links found in downloaded HTML file'
f = open(self.file, 'r')
data = f.read()
f.close()
parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(
cStringIO.StringIO())))
parser.feed(data)
parser.close()
return parser.anchorlist
def __init__(self, formatter=AbstractFormatter(DumbWriter())):
HTMLParser.__init__(self,formatter)
self.intoTheBox=False #ingresso/uscita dall'ambiente del box
self.intoTheTitle=None #ingresso/uscita dalla riga titolo
self.intoTheNews=None #ingresso/uscita dal testo della notizia
self.effectiveTitle=False #tag <span> del titolo
self.effectiveDate=False #tag <span> della data
self.effectiveAuth=False #tag <div> dell'autore
self.writable=False #testo scrivibile
self.cookedText="" #testo xml risultante
def parse_links(self):
""" Parse out the links found is downloaded HTML file"""
f = open(self.file, "r")
data = f.read()
f.close()
parser = HTMLParser(
formatter.AbstractFormatter(
formatter.DumbWriter(cStringIO.StringIO())))
parser.feed(data)
return parser.anchorlist
def parseAndGetLinks(self):
'''解析html页面，获取页面中的链接，并保存链接'''
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
#使用HTMLParser的方法进行处理 , StringIO是从内存中读取数据，DumbWriter将事件流转换为存文本文档。
self.parser.feed(open(self.file).read())
#将self.file文件打开，并一次性读入上面定的文件中
self.parser.close()
print 'self.parser.anchorlist --> ', self.parser.anchorlist
return self.parser.anchorlist #anchorlist 记录href 地址
def parse_links(self):
f = open(self.file, "r")
data = f.read()
f.close()
parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
# parser = MyHTMLParser()
parser.feed(data)
parser.close()
# 没有在模块中找到 anchorlist属性
# 返回页面中所有的锚点，href
# 该属性在2.6后就被弃用了，如果想用可以通过自定义parser继承HTMLParser来实现，参见evernote.或者用第三方库beautifulsoup
return parser.anchorlist
def parseAndGetLinks(self, html_string):
try:
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(html_string)
self.parser.close()
links = []
for eachLink in self.parser.anchorlist:
if eachLink[:4] != "http" and find(eachLink, "://") == -1:
eachLink = urljoin(self.base_url, eachLink)
links.append(eachLink)
return links
except IOError:
return []
def parse_html(self, html):
from StringIO import StringIO
from formatter import (AbstractFormatter, DumbWriter)
from htmllib import HTMLParser
_html = re.sub(self.notrans_tag, r" \1 ", html)
buf = StringIO()
p = HTMLParser(AbstractFormatter(DumbWriter(buf)))
p.feed(_html)
_sub = re.sub(self.whitespaces, " ", buf.getvalue())
# FIXME: how can zerowidth be removed more simply?
_sub = re.sub(self.zerowidth, "", _sub)
_sub = re.sub(self.colon, r"\1", _sub)
return _sub
def __init__(self, formatter=AbstractFormatter(DumbWriter())):
#HTMLParser.__init__(self)
HTMLParser.__init__(self,formatter)
self.intoTheAnchor=False #ingresso/uscita dai tag anchor
self.intoTheDate=False #ingresso/uscita dal box data
self.intoTheTitle=False #ingresso/uscita dal box titolo
self.intoTheNews=False #ingresso/uscita dal box notizia
self.writable=False #testo scrivibile
self.maxNews=10 #totale notizie visualizzate
self.cookedText="" #testo xml risultante
self.gotTitle=False #flag indicante la reale presenza del titolo
self.pseudoTitle="" #titolo fake (se mancante)
def parse(self, url):
self.base = ""
self.href = ""
m = re.compile(".*/").match(url)
if m != None:
self.base = m.string[m.start(0):m.end(0)]
result = urlfetch.fetch(url, headers = {'Cache-Control' : 'max-age=30', 'Pragma' : 'no-cache'} )
if result.status_code == 200:
logging.debug(str(result.status_code) + " OK " + url)
HTMLParser.feed(self, result.content)
HTMLParser.close(self)
else:
logging.error(str(result.status_code) + " NG " + url)
class Retriever(object):#download web page
def __init__(self, url):
self.url = url
self.file = self.filename(url)
#local filename ,directory
def filename(self, url, deffile = "index.htm"):
parseurl = urlparse(url, 'http:', 0)
path = parseurl[1] + parseurl[2]
#将路径转换为一个元组，如果为目录则第二个元素为空，如果文件则第二个元素为文件扩展名
#path = "D:/pycharmProjects/PythonWebApp/weblearning/Crawl.py"
#print splitext(path)
#('D:/pycharmProjects/PythonWebApp/weblearning/Crawl', '.py')
ext = splitext(path)
if ext[1] == '':#no file use default
#tuple[-index],倒数第index个元素
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
#获取path的dir,
# path=D:/pycharmProjects/PythonWebApp/webLearning
# dir(path)=D:/pycharmProjects/PythonWebApp
ldir = dirname(path)#local directory
if sep != '/': #os-indep. path separator
ldir = replace(ldir, '/', sep)
if not isdir(ldir): #create archive dir if nec.
#如果存在文件，则删除
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self):
#urlretrieve()返回一个2元组，（filename,mime_hdrs）
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' %self.url,)
print 'erro,invalid url'
return retval
def parseAndGetLinks(self):#parse HTML , save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
# 使用http协议解析url地址，将url解析为这样的五元组：(scheme, netloc, path, query, fragment)
parsedurl = urlparse(url, 'http:', 0)
# 将主机地址（netloc）和路径（path）合并起来作为存储文件的路径名
path = parsedurl[1] + parsedurl[2]
# splitext将path分割为路径与后缀名，如（/path/to/file, txt）
ext = splitext(path)
# 如果后缀名为空，则给当前path添加默认的名字index.htm
if ext[1] == '':
# 若path结尾有“/”则直接添加index.htm，否则先添加“/”
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
# 取出path中的目录部分（www.shellbye.com\\blog），然后与本地目录结合为一个目录('D:\\www.shellbye.com\\blog')
ldir = dirname(abspath(path))
# 如果不是以“/”作为目录分割符的类Unix系统，比如Windows，则需要把“”替换为相应的目录分割符
# 因为类Unix系统的目录分割符与URI地址的分割符一样，所以可以不处理
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
# 如果ldir目录不存在在创建
if not isdir(ldir): # create archive dir if nec.
# 如果ldir存在但是不是目录则删除ldir。注，unlink即remove。
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self): # download Web page
try:
# 下载self.url到self.file里
retval = urllib.urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url, )
return retval
def parseAndGetLinks(self):
# 创建一个基本的HTML解释器，可能需要单独一篇文章来说这句
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
# 解析html文件，获取所有的连接（带有href的）
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
class Retriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url, deffile='index.htm'):
# 使用http协议解析url地址，将url解析为这样的五元组：(scheme, netloc, path, query, fragment)
parsedurl = urlparse(url, 'http:', 0)
# 将主机地址（netloc）和路径（path）合并起来作为存储文件的路径名
path = parsedurl[1] + parsedurl[2]
# splitext将path分割为路径与后缀名，如（/path/to/file, txt）
ext = splitext(path)
# 如果后缀名为空，则给当前path添加默认的名字index.htm
if ext[1] == '':
# 若path结尾有“/”则直接添加index.htm，否则先添加“/”
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
# 取出path中的目录部分（www.shellbye.com\\blog），然后与本地目录结合为一个目录('D:\\www.shellbye.com\\blog')
ldir = dirname(abspath(path))
# 如果不是以“/”作为目录分割符的类Unix系统，比如Windows，则需要把“”替换为相应的目录分割符
# 因为类Unix系统的目录分割符与URI地址的分割符一样，所以可以不处理
if sep != '/': # os-indep. path separator
ldir = replace(ldir, '/', sep)
# 如果ldir目录不存在在创建
if not isdir(ldir): # create archive dir if nec.
# 如果ldir存在但是不是目录则删除ldir。注，unlink即remove。
if exists(ldir):
unlink(ldir)
makedirs(ldir)
return path
def download(self): # download Web page
try:
# 下载self.url到self.file里
retval = urllib.urlretrieve(self.url, self.file)
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url, )
return retval
def parseAndGetLinks(self):
# 创建一个基本的HTML解释器，可能需要单独一篇文章来说这句
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
# 解析html文件，获取所有的连接（带有href的）
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter):
HTMLParser.__init__(self, fmt)
self.result = ""
self.open_tags = []
# A list of the only tags allowed. Be careful adding to this. Adding
# "script," for example, would not be smart. 'img' is out by default
# because of the danger of IMG embedded commands, and/or web bugs.
self.permitted_tags = [
"a",
"b",
"blockquote",
"br",
"i",
"sup",
"sub",
"strike",
"hr",
"u",
"li",
"ol",
"ul",
"p",
"cite",
"img",
"style",
"font",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"pre",
"div",
]
# A list of tags that require no closing tag.
self.requires_no_close = ["img", "br"]
# A dictionary showing the only attributes allowed for particular tags.
# If a tag is not listed here, it is allowed no attributes. Adding
# "on" tags, like "onhover," would not be smart. Also be very careful
# of "background" and "style."
self.allowed_attributes = {"a": ["href", "title"], "img": ["src", "alt"], "blockquote": ["type"]}
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ["http", "https", "ftp"]
def parseAndGetLinks(self): # pars HTML, save links self.parser = HTMLParser(AbstractFormatter( \ DumbWriter(StringIO()))) self.parser.feed(open(self.file).read()) self.parser.close() print self.parser return self.parser.anchorlist
def parse(self, url):
self.base = ""
self.href = ""
m = re.compile(".*/").match(url)
if m != None:
self.base = m.string[m.start(0):m.end(0)]
result = urlfetch.fetch(url,
headers={
'Cache-Control': 'max-age=30',
'Pragma': 'no-cache'
})
if result.status_code == 200:
logging.debug(str(result.status_code) + " OK " + url)
HTMLParser.feed(self, result.content)
HTMLParser.close(self)
else:
logging.error(str(result.status_code) + " NG " + url)
class Retriever(): # download web pages
def __init__(self, url):
self.url = url
def download(self): # download web page
print 'try to open url:', self.url, '\nthe true url process', string.split(
self.url, '?')[0]
try:
retval = urlopen(string.split(self.url, '?')[0], None, 200)
except urllib2.HTTPError as e:
print "HTTPError", e
return
except socket.timeout as e:
print "socket.timeout", e
return
except socket.error as e:
print "socket.error", e
return
except urllib2.URLError as e:
print "URLError: ", e
return
return retval
def parseAndGetLinks(self): # parse HTML, save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
r = self.download()
if r:
print '________'
try:
try:
s = r.read(50000)
except socket.error as e:
print "***************************socket error***************************", e
return []
self.parser.feed(s)
print '------------------'
r.close()
print '***************************'
except HTMLParseError:
print 'get links error\n'
return []
self.parser.close()
return self.parser.anchorlist
def __init__(self, fmt=AbstractFormatter):
HTMLParser.__init__(self, fmt)
self.result = ""
self.open_tags = []
# A list of forbidden tags.
self.forbidden_tags = ['script', 'embed', 'iframe', 'frame']
# A list of tags that require no closing tag.
self.requires_no_close = ['img', 'br']
# A dictionary showing the only attributes allowed for particular tags.
# If a tag is not listed here, it is allowed no attributes. Adding
# "on" tags, like "onhover," would not be smart. Also be very careful
# of "background" and "style."
# <h5 style="text-align: center;"><b><i><u><font size="5" face="impact">THIS IS A TEST</font></u></i></b></h5>
# <blockquote style="margin: 0 0 0 40px; border: none; padding: 0px;"><p style="text-align: center;">
# <font size="5" face="arial" color="#cc3333">of the EBS</font></p><p style="text-align: center;">
# <font size="5" face="arial"><br></font></p><p style="text-align: center;"><font size="5" face="arial">
# <sup>reddit</sup><sub>2</sub></font></p>
# <p style="text-align: center;"><font size="5" face="arial"><sub><br></sub></font></p>
# <p style="text-align: center;"><font size="5" face="arial">fiiiiiii<sub>4</sub></font></p>
# <p style="text-align: center;"><font size="5" face="arial"><sub><br></sub></font></p>
# <p style="text-align: center;"><hr><br></p><p style="text-align: center;">
# <strike>strike</strike></p></blockquote>
self.allowed_attributes =\
{'a':['href','title','target','style'],
'p': ['style'],
'img':['src','alt','border','style','align'],
'blockquote':['type','style','align'],
'font':['size','face','align'],
'h5':['style'],'h4':['style'],'h3':['style'],'h2':['style'],'h1':['style'],
'table': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'tbody': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'tr': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'td': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'div': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
'span': ['border', 'width', 'height', 'style', 'align', 'bgcolor'],
}
# The only schemes allowed in URLs (for href and src attributes).
# Adding "javascript" or "vbscript" to this list would not be smart.
self.allowed_schemes = ['http', 'https', 'ftp']
class Retriever(object):
def __init__(self, url):
self.url = url
self.file = self.filename(url)
print self.file
def filename(self, url, deffile='index.htm'):
parsedurl = urlparse(url, 'http', 0)
path = parsedurl[1] + parsedurl[2]
ext = splitext(path) #分解文件名的扩展名
print path,ext
if ext[1] == '':
if path[-1] == '/':
path += deffile
else:
path += '/' + deffile
print path
ldir = dirname(path)
#应该和操作系统相关 windows下：sep = \
if sep != '/':
ldir = replace(ldir, '/', sep)
print ldir
if not isdir(ldir):
if exists(ldir):
return
makedirs(ldir)
return path
def download(self):
try:
retval = urlretrieve(self.url, self.file)
except IOError:
retval = ('*** Error URL "%s"' % self.url)
return retval
def parseAndGetLinks(self):
"""StringIO是从内存中读取数据 DumbWriter将事件流转换为存文本文档 AbstractFormatter 类进行格式化
"""
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
self.parser.feed(open(self.file).read())
self.parser.close()
return self.parser.anchorlist
def _clean_text(self, text):
try:
text = text.replace(" ", " ")
text = text.strip()
parser = HTMLParser(None)
parser.save_bgn()
parser.feed(text)
return parser.save_end()
except:
return text
def create_plaintext_message(message):
""" Create clean plain text version of email message
Parse the html and remove style and javacript tags and then
create a plain-text-message by parsing the html
and attaching links as endnotes
"""
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.kill_tags = ['style']
doc = message.decode('utf-8', 'ignore')
to_clean = lxml.html.fromstring(doc)
cleaned_msg = lxml.html.tostring(cleaner.clean_html(to_clean))
plain_text_maxcols = 72
textout = cStringIO.StringIO()
formtext = formatter.AbstractFormatter(
formatter.DumbWriter(textout, plain_text_maxcols))
parser = HTMLParser(formtext)
parser.feed(cleaned_msg)
parser.close()
# append the anchorlist at the bottom of a message
# to keep the message readable.
counter = 0
anchorlist = "\n\n" + ("-" * plain_text_maxcols) + "\n\n"
for item in parser.anchorlist:
counter += 1
if item.startswith('https://'):
new_item = item.replace('https://', 'http://')
else:
new_item = item
anchorlist += "[%d] %s\n" % (counter, new_item)
text = textout.getvalue() + anchorlist
del textout, formtext, parser, anchorlist
return text
def OpenURL(self, url):
from htmllib import HTMLParser
import formatter
self.url = url
m = re.match('http://([^/]+)(/\S*)\s*', url)
if m:
host = m.groups()[0]
path = m.groups()[1]
else:
m = re.match('http://(\S+)\s*', url)
if not m:
# Invalid URL
self.logprint("Invalid or unsupported URL: %s" % (url))
return
host = m.groups()[0]
path = ''
f = self.RetrieveAsFile(host, path)
if not f:
self.logprint("Could not open %s" % (url))
return
self.logprint("Receiving data...")
data = f.read()
tmp = open('hangman_dict.txt', 'w')
fmt = formatter.AbstractFormatter(formatter.DumbWriter(tmp))
p = HTMLParser(fmt)
self.logprint("Parsing data...")
p.feed(data)
p.close()
tmp.close()
def parseAndGetLinks(self): # parse HTML, save links
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
r = self.download()
if r:
print '________'
try:
try:
s = r.read(50000)
except socket.error as e:
print "***************************socket error***************************", e
return []
self.parser.feed(s)
print '------------------'
r.close()
print '***************************'
except HTMLParseError:
print 'get links error\n'
return []
self.parser.close()
return self.parser.anchorlist
class Retriever(object): #download Web pages
def __init__(self, url):
self.url = url
self.file = self.filename(url)
def filename(self, url):
path=url
path = re.sub("\W","_",path)
path+=".html"
return path
def isForbidden(self):
return 0;
def isForbidden(self):
return 0;
def download(self):
try:
if True:
retval = urlretrieve(self.url, self.file)
javaGroupContent=JavaGroupContent.JavaGroupContent()
javaGroupContent.meet_page(self.url, self.file)
else:
retval = '*** INFO: no need to download '
except IOError:
retval = ('*** ERROR: invalid URL "%s"' % self.url,)
return retval
def parseAndGetLinks(self):
self.parser = HTMLParser(AbstractFormatter(DumbWriter(StringIO())))
try:
self.parser.feed(open(self.file).read())
self.parser.close()
except IOError:
pass
return self.parser.anchorlist
def get_urls(url):
data = urllib.urlopen(url).read()
parser = HTMLParser(
formatter.AbstractFormatter(formatter.DumbWriter(
cStringIO.StringIO())))
parser.feed(data)
parser.close()
url_list = parser.anchorlist
return url_list
def parse_links(self):
f = open(self.file, 'r')
data = f.read()
f.close()
paeser = HTMLParser(
formatter.AbstractFormatter(
formatter.DumbWriter(cStringIO.StringIO())))
paeser.feed(data)
paeser.close()
return paeser.anchorlist
