def parse_profile_html(document):
"""
Parse an LXML document to retrieve the profile data
:param document: the LXML document to parse
:return: a dictionary representing the profile
"""
username_elements = document.xpath("//*[@class='author']")
registered_elements = document.xpath("//*[@class='registered']")
avatar_elements = document.xpath("//*[@class='title']//img")
info_elements = document.xpath("//*[@class='info']")
userid_elements = document.xpath("//*[@name='userid']")
profile = {}
if userid_elements:
profile["id"] = userid_elements[0].attrib["value"]
if username_elements:
profile["username"] = username_elements[0].text_content()
if registered_elements:
profile["registered"] = registered_elements[0].text_content()
if avatar_elements:
profile["avatar"] = avatar_elements[0].attrib["src"] if avatar_elements[0].attrib["src"] else ""
profile["is_newbie"] = profile["avatar"].endswith("/images/newbie.gif")
if info_elements:
info_text = info_elements[0].text_content()
post_count = re.search(r"Post Count(\d+)", info_text)
if post_count:
profile["post_count"] = post_count.group(1)
post_rate = re.search(r"Post Rate([\d\.]+)", info_text)
if post_rate:
profile["post_rate"] = post_rate.group(1)
last_post = re.search(r"Last Post(.+)", info_text)
if last_post:
profile["last_post"] = last_post.group(1)
gender = re.search(r"claims to be a ([-a-z0-9 ]+)", info_text)
if gender:
profile["gender"] = gender.group(1).lower()
if "id" in profile:
profile["profile_link"] = http.prepare_url(PROFILE_URL, { "action": "getinfo", "userid": profile["id"]})
elif "username" in profile:
profile["profile_link"] = http.prepare_url(PROFILE_URL, {"action": "getinfo", "username": profile["username"]})
return profile
def parse_profile_html(document):
"""
Parse an LXML document to retrieve the profile data
:param document: the LXML document to parse
:return: a dictionary representing the profile
"""
username_elements = document.xpath("//*[@class='author']")
registered_elements = document.xpath("//*[@class='registered']")
avatar_elements = document.xpath("//*[@class='title']//img")
info_elements = document.xpath("//*[@class='info']")
userid_elements = document.xpath("//*[@name='userid']")
profile = {}
if userid_elements:
profile["id"] = userid_elements[0].attrib["value"]
if username_elements:
profile["username"] = username_elements[0].text_content()
if registered_elements:
profile["registered"] = registered_elements[0].text_content()
if avatar_elements:
profile["avatar"] = avatar_elements[0].attrib["src"] if avatar_elements[0].attrib["src"] else ""
profile["is_newbie"] = profile["avatar"].endswith("/images/newbie.gif")
if info_elements:
info_text = info_elements[0].text_content()
post_count = re.search(r"Post Count(\d+)", info_text)
if post_count:
profile["post_count"] = post_count.group(1)
post_rate = re.search(r"Post Rate([\d\.]+)", info_text)
if post_rate:
profile["post_rate"] = post_rate.group(1)
last_post = re.search(r"Last Post(.+)", info_text)
if last_post:
profile["last_post"] = last_post.group(1)
gender = re.search(r"claims to be a ([-a-z0-9 ]+)", info_text)
if gender:
profile["gender"] = gender.group(1).lower()
if "id" in profile:
profile["profile_link"] = http.prepare_url(PROFILE_URL, { "action": "getinfo", "userid": profile["id"]})
elif "username" in profile:
profile["profile_link"] = http.prepare_url(PROFILE_URL, {"action": "getinfo", "username": profile["username"]})
return profile
def parse_thread_html(document):
"""
Parse an LXML document to retrieve the thread data
:param document: the LXML document to parse
:return: a dictionary representing the thread
"""
breadcrumbs_elements = document.xpath("//div[@class='breadcrumbs']//a")
author_elements = document.xpath("//dt[contains(@class, author)]")
last_page_elements = document.xpath("//a[@title='Last page']")
if not breadcrumbs_elements:
return
if not author_elements:
return
if len(breadcrumbs_elements) < 2:
return
thread_id = int(breadcrumbs_elements[-1].attrib['href'].rsplit('=', 2)[1])
breadcrumbs = [ e.text_content() for e in breadcrumbs_elements ]
thread_title = breadcrumbs[-1]
forum_title = breadcrumbs[-2]
if author_elements:
author = author_elements[0].text_content().strip()
else:
author = 'Unknown Author'
# Handle GBS / FYAD / E/N / etc
if ':' in forum_title:
forum_title = forum_title.split(':')[0].strip()
if forum_title in FORUM_ABBREVS:
forum_title = FORUM_ABBREVS[forum_title]
if last_page_elements:
post_count = int(last_page_elements[0].text_content().split(" ")[0])
else:
post_count = 1
posts = {x.attrib['id']: (
x.xpath('.//dt[contains(@class, "author")]')[0].text_content(),
x.xpath('.//*[@class="postdate"]')[0].text_content().strip('\n #?'),
x.xpath('.//*[@class="postbody"]')[0].text_content().strip())
for x in document.xpath('//table[contains(@class, "post")]')}
return {
"id": thread_id,
"breadcrumbs": breadcrumbs,
"forum_title": forum_title,
"thread_title": thread_title,
"author": author,
"post_count": post_count,
"posts": posts,
"thread_link": http.prepare_url(THREAD_URL, {'threadid': thread_id}),
}
def parse_thread_html(document):
"""
Parse an LXML document to retrieve the thread data
:param document: the LXML document to parse
:return: a dictionary representing the thread
"""
breadcrumbs_elements = document.xpath("//div[@class='breadcrumbs']//a")
author_elements = document.xpath("//dt[contains(@class, author)]")
last_page_elements = document.xpath("//a[@title='Last page']")
if not breadcrumbs_elements:
return
if not author_elements:
return
if len(breadcrumbs_elements) < 2:
return
thread_id = int(breadcrumbs_elements[-1].attrib['href'].rsplit('=', 2)[1])
breadcrumbs = [ e.text_content() for e in breadcrumbs_elements ]
thread_title = breadcrumbs[-1]
forum_title = breadcrumbs[-2]
if author_elements:
author = author_elements[0].text_content().strip()
else:
author = 'Unknown Author'
# Handle GBS / FYAD / E/N / etc
if ':' in forum_title:
forum_title = forum_title.split(':')[0].strip()
if forum_title in FORUM_ABBREVS:
forum_title = FORUM_ABBREVS[forum_title]
if last_page_elements:
post_count = int(last_page_elements[0].text_content().split(" ")[0])
else:
post_count = 1
return {
"id": thread_id,
"breadcrumbs": breadcrumbs,
"forum_title": forum_title,
"thread_title": thread_title,
"author": author,
"post_count": post_count,
"thread_link": http.prepare_url(THREAD_URL, {'threadid': thread_id})
}
def qrcode(inp):
"""qrcode [link] returns a link for a QR code."""
args = {
"cht": "qr", # chart type (QR)
"chs": "200x200", # dimensions
"chl": inp # data
}
link = http.prepare_url("http://chart.googleapis.com/chart", args)
return web.try_isgd(link)