def parse_doc(url):
"""Parsing the documentation."""
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.find("div", "docstring").find("pre")
l = []
if not stuff:
ret = " \n" + "No documentation available!\n"
l.append(ret.split("\n"))
return l
ret = "Documentation: \n" + stuff.text
l.append(ret.split("\n"))
return l
def parse_source(url):
"""Parsing source"""
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.find("div", "source_content")
l = []
if not stuff:
ret = " \n" + "No source code available!\n"
l.append(ret.split("\n"))
return l
stuff = stuff.find("pre", "brush: clojure")
ret = "Source: \n" + stuff.text
l.append(ret.split("\n"))
return l
def parse_source(url):
"""Parsing source"""
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.find("div", "source_content")
l = []
if not stuff:
ret = " \n" + "No source code available!\n"
l.append(ret.split("\n"))
return l
stuff = stuff.find("pre", "brush: clojure")
ret = "Source: \n" + stuff.text
l.append(ret.split("\n"))
return l
def diagnose(data):
"""Diagnostic suite for isolating common problems."""
print "Diagnostic running on Beautiful Soup %s" % __version__
print "Python version %s" % sys.version
basic_parsers = ["html.parser", "html5lib", "lxml"]
for name in basic_parsers:
for builder in builder_registry.builders:
if name in builder.features:
break
else:
basic_parsers.remove(name)
print(
"I noticed that %s is not installed. Installing it may help." %
name)
if 'lxml' in basic_parsers:
basic_parsers.append(["lxml", "xml"])
from lxml import etree
print "Found lxml version %s" % ".".join(map(str, etree.LXML_VERSION))
if 'html5lib' in basic_parsers:
import html5lib
print "Found html5lib version %s" % html5lib.__version__
if hasattr(data, 'read'):
data = data.read()
elif os.path.exists(data):
print '"%s" looks like a filename. Reading data from the file.' % data
data = open(data).read()
elif data.startswith("http:") or data.startswith("https:"):
print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
return
print
for parser in basic_parsers:
print "Trying to parse your markup with %s" % parser
success = False
try:
soup = BeautifulSoup(data, parser)
success = True
except Exception, e:
print "%s could not parse the markup." % parser
traceback.print_exc()
if success:
print "Here's what %s did with the markup:" % parser
print soup.prettify()
print "-" * 80
def diagnose(data):
"""Diagnostic suite for isolating common problems."""
print "Diagnostic running on Beautiful Soup %s" % __version__
print "Python version %s" % sys.version
basic_parsers = ["html.parser", "html5lib", "lxml"]
for name in basic_parsers:
for builder in builder_registry.builders:
if name in builder.features:
break
else:
basic_parsers.remove(name)
print (
"I noticed that %s is not installed. Installing it may help." %
name)
if 'lxml' in basic_parsers:
basic_parsers.append(["lxml", "xml"])
from lxml import etree
print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
if 'html5lib' in basic_parsers:
import html5lib
print "Found html5lib version %s" % html5lib.__version__
if hasattr(data, 'read'):
data = data.read()
elif os.path.exists(data):
print '"%s" looks like a filename. Reading data from the file.' % data
data = open(data).read()
elif data.startswith("http:") or data.startswith("https:"):
print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
return
print
for parser in basic_parsers:
print "Trying to parse your markup with %s" % parser
success = False
try:
soup = BeautifulSoup(data, parser)
success = True
except Exception, e:
print "%s could not parse the markup." % parser
traceback.print_exc()
if success:
print "Here's what %s did with the markup:" % parser
print soup.prettify()
print "-" * 80
def parse_doc(url):
"""Parsing the documentation."""
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.find("div", "doc").find("div", "content")
for e in stuff.findAll("br"):
e.replace_with("\n")
l = []
if not stuff:
ret = " \n" + "No documentation available!\n"
l.append(ret.split("\n"))
return l
ret = "Documentation: \n" + stuff.text
l.append(ret.split("\n"))
return l
def parse_doc(url):
"""Parsing the documentation."""
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.find("div", "doc").find("div", "content")
for e in stuff.findAll("br"):
e.replace_with("\n")
l = []
if not stuff:
ret = " \n" + "No documentation available!\n"
l.append(ret.split("\n"))
return l
ret = "Documentation: \n" + stuff.text
l.append(ret.split("\n"))
return l
def parse_example(url):
"""Parsing examples"""
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.find_all("div", "hidden plain_content")
l = []
if not stuff:
ret = " \n" + "No examples available!\n"
l.append(ret.split("\n"))
return l
num = 1
for i in stuff:
ret = "Example #" + str(num) + ": \n" + i.text.rstrip("\n")
l.append(ret.split("\n"))
num += 1
return l
def parse_example(url):
"""Parsing examples"""
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.find_all("div", "hidden plain_content")
l = []
if not stuff:
ret = " \n" + "No examples available!\n"
l.append(ret.split("\n"))
return l
num = 1
for i in stuff:
ret = "Example #" + str(num) + ": \n" + i.text.rstrip("\n")
l.append(ret.split("\n"))
num += 1
return l
def seealso_search(url):
"""Searches the 'See Also...' part. """
v = content_request(url)
soup = BeautifulSoup(v)
stuff = soup.body.find_all("li", "see_also_item")
items = []
sites = []
for i in stuff:
item, web = new_parse(i)
items.append(item)
sites.append(web)
return (items, sites)
def bs4_parse(var):
"""Parses out the indvidual search item"""
var = request(var)
soup = BeautifulSoup(var)
stuff = soup.body.find_all("div", "search_result")
items = []
sites = []
for i in stuff:
item, web = parse_list(i)
items.append(item)
sites.append(web)
return (items, sites)
def benchmark_parsers(num_elements=100000):
"""Very basic head-to-head performance benchmark."""
print "Comparative parser benchmark on Beautiful Soup %s" % __version__
data = rdoc(num_elements)
print "Generated a large invalid HTML document (%d bytes)." % len(data)
for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
success = False
try:
a = time.time()
soup = BeautifulSoup(data, parser)
b = time.time()
success = True
except Exception, e:
print "%s could not parse the markup." % parser
traceback.print_exc()
if success:
print "BS4+%s parsed the markup in %.2fs." % (parser, b - a)
