def get_smileys():
global n
logging.info('Récupération des émoticones')
n = 0
d = PyQuery(url=config.rooturl+'/admin/index.forum?part=themes&sub=avatars&mode=smilies&extended_admin=1&' + tid, opener=fa_opener)
result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text())
try:
pages = int(result.group(1))
usersperpages = int(result.group(2))
except:
pages = 1
usersperpages = 0
if config.debug:
progress = progressbar.NoProgressBar()
else:
progress = progressbar.ProgressBar(widgets=[BarVar(), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=pages-1)
progress.start()
for page in range(0,pages):
if page >= 1:
d = PyQuery(url=config.rooturl + '/admin/index.forum?part=themes&sub=avatars&mode=smilies&extended_admin=1&start=' + str(page*usersperpages) + '&' + tid, opener=fa_opener)
for i in d('table tr'):
e = PyQuery(i)
if e("td").eq(0).text() != None and e("td").eq(0).attr("colspan") == None:
save.smileys[e("td").eq(0).text()] = e("td").eq(1).text()
n += 1
progress.update(page)
progress.end()
def get_forums():
logging.info('Récupération des forums')
if config.debug:
progress = progressbar.NoProgressBar()
else:
progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()])
d = PyQuery(url=config.rooturl + '/a-f1/', opener=fa_opener)
save.forums = []
levels = {}
n = 1
for i in progress([i for i in d.find("select option") if i.get("value", "-1") != "-1"]):
id = i.get("value", "-1")
logging.debug('Récupération: forum %s', id)
title = re.search('(((\||\xa0)(\xa0\xa0\xa0))*)\|--([^<]+)', i.text).group(5)
level = len(re.findall('(\||\xa0)\xa0\xa0\xa0', i.text))
if level <= 0:
parent = 0
else:
parent = levels[level-1]
levels[level] = n
d = PyQuery(url=config.rooturl+'/admin/index.forum?part=general&sub=general&mode=edit&fid=' + id + '&extended_admin=1&' + tid, opener=fa_opener)
try:
description = d("textarea").text()
except:
description = ""
save.forums.append({'id': int(id[1:]), 'newid': n, 'type': id[0], 'parent': parent, 'title': title, 'description': description, 'parsed': False})
n += 1
def get_topics():
logging.info('Récupération des sujets')
if config.debug:
progress = progressbar.NoProgressBar()
else:
progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=save.nbtopics)
progress.start()
n = len(save.topics)
ids = [i["id"] for i in save.topics]
for forum in [i for i in save.forums if (i["type"] == "f" and i["parsed"] == False)]:
logging.debug('Récupération : sujets du forum %d', forum["id"])
subtopics = []
subids = []
d = PyQuery(url=config.rooturl + '/' + forum['type'] + str(forum['id']) + '-a', opener=fa_opener)
result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text())
try:
pages = int(result.group(1))
topicsperpages = int(result.group(2))
except:
pages = 1
topicsperpages = 0
for page in range(0,pages):
if page >= 1:
d = PyQuery(url=config.rooturl + '/' + forum['type'] + str(forum['id']) + 'p' + str(page*topicsperpages) + '-a', opener=fa_opener)
for i in d.find('div.topictitle'):
e = PyQuery(i)
id = int(re.search("/t(\d+)-.*", e("a").attr("href")).group(1))
if id not in ids and id not in subids:
logging.debug('Récupération : sujet %d', id)
f = e.parents().eq(-2)
locked = u"verrouillé" in f("td img").eq(0).attr("alt")
views = int(f("td").eq(5).text())
subtopics.append({'id': id, 'type': e("strong").text(), 'parent': forum['newid'], 'title': e("a").text(), 'locked': locked, 'views': views, 'parsed': False})
subids.append(id)
n += 1
progress.update(n)
else:
logging.warning('Le sujet %d a déjà été récupéré.', id)
save.topics.extend(subtopics)
ids.extend(subids)
[i for i in save.forums if i == forum][0]["parsed"] = True
progress.end()
def get_users():
global month
logging.info('Récupération des membres')
if config.debug:
progress = progressbar.NoProgressBar()
else:
progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=save.nbusers)
progress.start()
save.users = []
n = 2
d = PyQuery(url=config.rooturl+'/admin/index.forum?part=users_groups&sub=users&extended_admin=1&' + tid, opener=fa_opener)
result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text())
try:
pages = int(result.group(1))
usersperpages = int(result.group(2))
except:
pages = 1
usersperpages = 0
for page in range(0,pages):
if page >= 1:
d = PyQuery(url=config.rooturl + '/admin/index.forum?part=users_groups&sub=users&extended_admin=1&start=' + str(page*usersperpages) + '&' + tid, opener=fa_opener)
for i in d('tbody tr'):
e = PyQuery(i)
id = int(re.search("&u=(\d+)&", e("td a").eq(0).attr("href")).group(1))
logging.debug('Récupération : membre %d', id)
date = e("td").eq(3).text().split(" ")
date = time.mktime(time.struct_time((int(date[2]),month[date[1]],int(date[0]),0,0,0,0,0,0)))
lastvisit = e("td").eq(4).text()
if lastvisit != "":
lastvisit = lastvisit.split(" ")
lastvisit = time.mktime(time.struct_time((int(lastvisit[2]),month[lastvisit[1]],int(lastvisit[0]),0,0,0,0,0,0)))
else:
lastvisit = 0
save.users.append({'id': id, 'newid': n, 'name': e("td a").eq(0).text(), 'mail': e("td a").eq(1).text(), 'posts': int(e("td").eq(2).text()), 'date': int(date), 'lastvisit': int(lastvisit)})
n += 1
progress.update(n-2)
progress.end()
def get_posts():
global month
logging.info('Récupération des messages')
if config.debug:
progress = progressbar.NoProgressBar()
else:
progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=save.nbposts)
progress.start()
n = len(save.posts)
ids = [i["id"] for i in save.posts]
for topic in [i for i in save.topics if i["parsed"] == False]:
logging.debug('Récupération : messages du topic %d', topic["id"])
subposts = []
subids = []
d = PyQuery(url=config.rooturl + '/t' + str(topic['id']) + '-a', opener=fa_opener)
result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text())
try:
pages = int(result.group(1))
topicsperpages = int(result.group(2))
except:
pages = 1
topicsperpages = 0
for page in range(0,pages):
if page >= 1:
d = PyQuery(url=config.rooturl + '/a-t' + str(topic['id']) + '-' + str(page*topicsperpages) + '.htm', opener=fa_opener)
for i in d.find('tr.post'):
e = PyQuery(i)
id = int(e("td span.name a").attr("name"))
if id not in ids and id not in subids:
logging.debug('Récupération : message %d (topic %d)', id, topic["id"])
author = e("td span.name").text()
post = htmltobbcode.htmltobbcode(e("td div.postbody div").eq(0).html(), save.smileys)
result = e("table td span.postdetails").text().split(" ")
if result[-3] == "Aujourd'hui":
title = " ".join(e("table td span.postdetails").text().split(" ")[1:-3])
date = e("table td span.postdetails").text().split(" ")[-3:]
timestamp = time.mktime(datetime.datetime.combine(datetime.date.today(), datetime.time(int(date[2].split(":")[0]),int(date[2].split(":")[1]))).timetuple())
elif result[-3] == "Hier":
title = " ".join(e("table td span.postdetails").text().split(" ")[1:-3])
date = e("table td span.postdetails").text().split(" ")[-3:]
timestamp = time.mktime(datetime.datetime.combine(datetime.date.today()-datetime.timedelta(1), datetime.time(int(date[2].split(":")[0]),int(date[2].split(":")[1]))).timetuple())
else:
title = " ".join(e("table td span.postdetails").text().split(" ")[1:-6])
date = e("table td span.postdetails").text().split(" ")[-6:]
timestamp = time.mktime(datetime.datetime(int(date[3]),month[date[2]],int(date[1]),int(date[5].split(":")[0]),int(date[5].split(":")[1])).timetuple())
subposts.append({'id': id, 'post': post, 'title': title, 'topic': topic["id"], 'timestamp': int(timestamp), 'author': author})
subids.append(id)
n += 1
progress.update(n)
else:
logging.warning('Le message %d a déjà été récupéré.', id)
save.posts.extend(subposts)
ids.extend(subids)
[i for i in save.topics if i == topic][0]["parsed"] = True
progress.end()
