def get_smileys(): global n logging.info('Récupération des émoticones') n = 0 d = PyQuery(url=config.rooturl+'/admin/index.forum?part=themes&sub=avatars&mode=smilies&extended_admin=1&' + tid, opener=fa_opener) result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text()) try: pages = int(result.group(1)) usersperpages = int(result.group(2)) except: pages = 1 usersperpages = 0 if config.debug: progress = progressbar.NoProgressBar() else: progress = progressbar.ProgressBar(widgets=[BarVar(), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=pages-1) progress.start() for page in range(0,pages): if page >= 1: d = PyQuery(url=config.rooturl + '/admin/index.forum?part=themes&sub=avatars&mode=smilies&extended_admin=1&start=' + str(page*usersperpages) + '&' + tid, opener=fa_opener) for i in d('table tr'): e = PyQuery(i) if e("td").eq(0).text() != None and e("td").eq(0).attr("colspan") == None: save.smileys[e("td").eq(0).text()] = e("td").eq(1).text() n += 1 progress.update(page) progress.end()
def get_forums(): logging.info('Récupération des forums') if config.debug: progress = progressbar.NoProgressBar() else: progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()]) d = PyQuery(url=config.rooturl + '/a-f1/', opener=fa_opener) save.forums = [] levels = {} n = 1 for i in progress([i for i in d.find("select option") if i.get("value", "-1") != "-1"]): id = i.get("value", "-1") logging.debug('Récupération: forum %s', id) title = re.search('(((\||\xa0)(\xa0\xa0\xa0))*)\|--([^<]+)', i.text).group(5) level = len(re.findall('(\||\xa0)\xa0\xa0\xa0', i.text)) if level <= 0: parent = 0 else: parent = levels[level-1] levels[level] = n d = PyQuery(url=config.rooturl+'/admin/index.forum?part=general&sub=general&mode=edit&fid=' + id + '&extended_admin=1&' + tid, opener=fa_opener) try: description = d("textarea").text() except: description = "" save.forums.append({'id': int(id[1:]), 'newid': n, 'type': id[0], 'parent': parent, 'title': title, 'description': description, 'parsed': False}) n += 1
def get_topics(): logging.info('Récupération des sujets') if config.debug: progress = progressbar.NoProgressBar() else: progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=save.nbtopics) progress.start() n = len(save.topics) ids = [i["id"] for i in save.topics] for forum in [i for i in save.forums if (i["type"] == "f" and i["parsed"] == False)]: logging.debug('Récupération : sujets du forum %d', forum["id"]) subtopics = [] subids = [] d = PyQuery(url=config.rooturl + '/' + forum['type'] + str(forum['id']) + '-a', opener=fa_opener) result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text()) try: pages = int(result.group(1)) topicsperpages = int(result.group(2)) except: pages = 1 topicsperpages = 0 for page in range(0,pages): if page >= 1: d = PyQuery(url=config.rooturl + '/' + forum['type'] + str(forum['id']) + 'p' + str(page*topicsperpages) + '-a', opener=fa_opener) for i in d.find('div.topictitle'): e = PyQuery(i) id = int(re.search("/t(\d+)-.*", e("a").attr("href")).group(1)) if id not in ids and id not in subids: logging.debug('Récupération : sujet %d', id) f = e.parents().eq(-2) locked = u"verrouillé" in f("td img").eq(0).attr("alt") views = int(f("td").eq(5).text()) subtopics.append({'id': id, 'type': e("strong").text(), 'parent': forum['newid'], 'title': e("a").text(), 'locked': locked, 'views': views, 'parsed': False}) subids.append(id) n += 1 progress.update(n) else: logging.warning('Le sujet %d a déjà été récupéré.', id) save.topics.extend(subtopics) ids.extend(subids) [i for i in save.forums if i == forum][0]["parsed"] = True progress.end()
def get_users(): global month logging.info('Récupération des membres') if config.debug: progress = progressbar.NoProgressBar() else: progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=save.nbusers) progress.start() save.users = [] n = 2 d = PyQuery(url=config.rooturl+'/admin/index.forum?part=users_groups&sub=users&extended_admin=1&' + tid, opener=fa_opener) result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text()) try: pages = int(result.group(1)) usersperpages = int(result.group(2)) except: pages = 1 usersperpages = 0 for page in range(0,pages): if page >= 1: d = PyQuery(url=config.rooturl + '/admin/index.forum?part=users_groups&sub=users&extended_admin=1&start=' + str(page*usersperpages) + '&' + tid, opener=fa_opener) for i in d('tbody tr'): e = PyQuery(i) id = int(re.search("&u=(\d+)&", e("td a").eq(0).attr("href")).group(1)) logging.debug('Récupération : membre %d', id) date = e("td").eq(3).text().split(" ") date = time.mktime(time.struct_time((int(date[2]),month[date[1]],int(date[0]),0,0,0,0,0,0))) lastvisit = e("td").eq(4).text() if lastvisit != "": lastvisit = lastvisit.split(" ") lastvisit = time.mktime(time.struct_time((int(lastvisit[2]),month[lastvisit[1]],int(lastvisit[0]),0,0,0,0,0,0))) else: lastvisit = 0 save.users.append({'id': id, 'newid': n, 'name': e("td a").eq(0).text(), 'mail': e("td a").eq(1).text(), 'posts': int(e("td").eq(2).text()), 'date': int(date), 'lastvisit': int(lastvisit)}) n += 1 progress.update(n-2) progress.end()
def get_posts(): global month logging.info('Récupération des messages') if config.debug: progress = progressbar.NoProgressBar() else: progress = progressbar.ProgressBar(widgets=[progressbar.SimpleProgress('/'), ' ', progressbar.Bar("#","[","]"), progressbar.Percentage()], maxval=save.nbposts) progress.start() n = len(save.posts) ids = [i["id"] for i in save.posts] for topic in [i for i in save.topics if i["parsed"] == False]: logging.debug('Récupération : messages du topic %d', topic["id"]) subposts = [] subids = [] d = PyQuery(url=config.rooturl + '/t' + str(topic['id']) + '-a', opener=fa_opener) result = re.search('function do_pagination_start\(\)[^\}]*start = \(start > \d+\) \? (\d+) : start;[^\}]*start = \(start - 1\) \* (\d+);[^\}]*\}', d.text()) try: pages = int(result.group(1)) topicsperpages = int(result.group(2)) except: pages = 1 topicsperpages = 0 for page in range(0,pages): if page >= 1: d = PyQuery(url=config.rooturl + '/a-t' + str(topic['id']) + '-' + str(page*topicsperpages) + '.htm', opener=fa_opener) for i in d.find('tr.post'): e = PyQuery(i) id = int(e("td span.name a").attr("name")) if id not in ids and id not in subids: logging.debug('Récupération : message %d (topic %d)', id, topic["id"]) author = e("td span.name").text() post = htmltobbcode.htmltobbcode(e("td div.postbody div").eq(0).html(), save.smileys) result = e("table td span.postdetails").text().split(" ") if result[-3] == "Aujourd'hui": title = " ".join(e("table td span.postdetails").text().split(" ")[1:-3]) date = e("table td span.postdetails").text().split(" ")[-3:] timestamp = time.mktime(datetime.datetime.combine(datetime.date.today(), datetime.time(int(date[2].split(":")[0]),int(date[2].split(":")[1]))).timetuple()) elif result[-3] == "Hier": title = " ".join(e("table td span.postdetails").text().split(" ")[1:-3]) date = e("table td span.postdetails").text().split(" ")[-3:] timestamp = time.mktime(datetime.datetime.combine(datetime.date.today()-datetime.timedelta(1), datetime.time(int(date[2].split(":")[0]),int(date[2].split(":")[1]))).timetuple()) else: title = " ".join(e("table td span.postdetails").text().split(" ")[1:-6]) date = e("table td span.postdetails").text().split(" ")[-6:] timestamp = time.mktime(datetime.datetime(int(date[3]),month[date[2]],int(date[1]),int(date[5].split(":")[0]),int(date[5].split(":")[1])).timetuple()) subposts.append({'id': id, 'post': post, 'title': title, 'topic': topic["id"], 'timestamp': int(timestamp), 'author': author}) subids.append(id) n += 1 progress.update(n) else: logging.warning('Le message %d a déjà été récupéré.', id) save.posts.extend(subposts) ids.extend(subids) [i for i in save.topics if i == topic][0]["parsed"] = True progress.end()