def __init__(self, config, ui=None):
Node.__init__(self)
self.config = config
self.session = Session(self.config)
self.ui = ui
# Statistics
self.total_posts = 0
self.total_topics = 0
self.total_users = 0
self.current_posts = 0
self.current_topics = 0
self.current_users = 0
self.startdate = 0
self.record_online_date = 0
self.record_online_users = 0
self.site_name = ""
self.site_desc = ""
self.dump_time = 0
self.smilies = {}
self.users = {}
self.forums = {}
self.announcements = []
self.linkrewriter = LinkRewriter(self)
def __init__(self, config, ui=None):
Node.__init__(self)
self.config = config
self.session = Session(self.config)
self.ui = ui
# Statistics
self.total_posts = 0
self.total_topics = 0
self.total_users = 0
self.current_posts = 0
self.current_topics = 0
self.current_users = 0
self.startdate = 0
self.record_online_date = 0
self.record_online_users = 0
self.site_name = ""
self.site_desc = ""
self.dump_time = 0
self.smilies = {}
self.users = {}
self.forums = {}
self.announcements = []
self.linkrewriter = LinkRewriter(self)
def load(config=None, ui=None):
"""
Returns the BB node contained in the file save.pickle.
"""
logger = logging.getLogger("lalf.bb.load")
if ui is None:
ui = DummyUI()
if config is None:
config = read_config("config.cfg")
try:
with open("save.pickle", "rb") as fileobj:
bb = pickle.load(fileobj)
bb.config = config
bb.session = Session(config)
bb.ui = ui
except FileNotFoundError:
bb = BB(config, ui)
except EOFError:
logger.warning(
"Erreur lors du chargement de la sauvegarde. Réinitialisation.")
bb = BB(config, ui)
return bb
class BB(Node):
"""
The BB node is the root of the tree representing the forum.
Attrs:
total_posts (int) : The total number of posts to be exported
total_topics (int) : The total number of topics to be exported
total_users (int) : The total number of users to be exported
current_posts (int) : The number of posts that have been exported
current_topics (int) : The number of topics that have been exported
current_users (int) : The number of users that have been exported
dump_time (int): The time at the beginning of the dump
"""
# Attributes to save
STATE_KEEP = ["total_posts", "total_topics", "total_users",
"current_posts", "current_topics", "current_users",
"startdate", "record_online_date", "record_online_users",
"site_name", "site_desc", "smilies", "users",
"forums", "announcements"]
def __init__(self, config, ui=None):
Node.__init__(self)
self.config = config
self.session = Session(self.config)
self.ui = ui
# Statistics
self.total_posts = 0
self.total_topics = 0
self.total_users = 0
self.current_posts = 0
self.current_topics = 0
self.current_users = 0
self.startdate = 0
self.record_online_date = 0
self.record_online_users = 0
self.site_name = ""
self.site_desc = ""
self.dump_time = 0
self.smilies = {}
self.users = {}
self.forums = {}
self.announcements = []
self.linkrewriter = LinkRewriter(self)
def _export_(self):
self.logger.info('Récupération des statistiques')
response = self.session.get("/statistics")
document = PyQuery(response.text)
# Go through the table of statistics and save the relevant
# ones
stats = {}
for element in document.find("table.forumline tr"):
e = PyQuery(element)
stats[e("td span").eq(0).text()] = e("td span").eq(1).text()
stats[e("td span").eq(2).text()] = e("td span").eq(3).text()
self.total_posts = int(stats["Messages"])
self.total_topics = int(stats["Nombre de sujets ouvert dans le forum"])
self.total_users = int(stats["Nombre d'utilisateurs"])
self.startdate = parse_date(stats["Ouverture du forum"])
self.record_online_date = parse_date(stats["Date du record de connexions"])
self.record_online_users = int(
stats["Nombre record d'utilisateurs connectés en même temps"])
self.site_name = document("div.maintitle").eq(0).text().strip(" \n")
self.site_desc = document("div.maintitle").siblings("span.gen").eq(0).text().strip(" \n")
self.logger.debug('Messages : %d', self.total_posts)
self.logger.debug('Sujets : %d', self.total_topics)
self.logger.debug('Membres : %d', self.total_users)
# Add the children nodes, which respectively handle the
# exportation of the smilies, the users and the message
self.add_child(Smilies())
if self.config["use_ocr"]:
# Use Optical Character Recognition to get the users'
# emails
self.add_child(OcrUsers())
else:
self.add_child(Users())
self.add_child(Groups())
self.add_child(Forums())
def _dump_(self, sqlfile):
self.logger.info("Création du fichier phpbb.sql")
self.dump_time = int(time.time())
# Clean tables
sqlfile.truncate("users")
sqlfile.truncate("user_group")
sqlfile.truncate("bots")
sqlfile.truncate("forums")
sqlfile.truncate("acl_groups")
sqlfile.truncate("topics")
sqlfile.truncate("topics_posted")
sqlfile.truncate("posts")
sqlfile.truncate("privmsgs")
sqlfile.truncate("privmsgs_to")
sqlfile.truncate("bbcodes")
# Update configuration and statistics
sqlfile.set_config("board_startdate", self.startdate)
sqlfile.set_config("default_lang", self.config["default_lang"])
sqlfile.set_config("record_online_date", self.record_online_date)
sqlfile.set_config("record_online_users", self.record_online_users)
sqlfile.set_config("sitename", self.site_name)
sqlfile.set_config("site_desc", self.site_desc)
num_posts = sum(1 for _ in self.get_posts())
num_topics = sum(1 for _ in self.get_topics())
num_users = sum(1 for _ in self.users)
sqlfile.set_config("num_posts", num_posts)
sqlfile.set_config("num_topics", num_topics)
sqlfile.set_config("num_users", num_users)
newest_user_oldid = max(self.users)
newest_user = self.users[newest_user_oldid]
sqlfile.set_config("newest_user_id", newest_user.newid)
sqlfile.set_config("newest_username", newest_user.name)
sqlfile.set_config("newest_user_colour", newest_user.colour)
# Add bbcodes tags
for bbcode in phpbb.BBCODES:
sqlfile.insert("bbcodes", bbcode)
def __setstate__(self, state):
Node.__setstate__(self, state)
self.linkrewriter = LinkRewriter(self)
# TODO : recompute current counts
def save(self):
"""
Dump the tree representing the forum in a pickle file
"""
self.logger.info("Sauvegarde de l'état courant.")
with open("save.pickle", "wb") as fileobj:
pickle.dump(self, fileobj, 2)
def get_topics(self):
"""
Iterator on the topics of the forum
"""
for forum in self.forums.values():
for topic in forum.get_topics():
yield topic
def get_posts(self):
"""
Iterator on the posts of the forum
"""
for topic in self.get_topics():
for post in topic.get_posts():
yield post
class BB(Node):
"""
The BB node is the root of the tree representing the forum.
Attrs:
total_posts (int) : The total number of posts to be exported
total_topics (int) : The total number of topics to be exported
total_users (int) : The total number of users to be exported
current_posts (int) : The number of posts that have been exported
current_topics (int) : The number of topics that have been exported
current_users (int) : The number of users that have been exported
dump_time (int): The time at the beginning of the dump
"""
# Attributes to save
STATE_KEEP = [
"total_posts", "total_topics", "total_users", "current_posts",
"current_topics", "current_users", "startdate", "record_online_date",
"record_online_users", "site_name", "site_desc", "smilies", "users",
"forums", "announcements"
]
def __init__(self, config, ui=None):
Node.__init__(self)
self.config = config
self.session = Session(self.config)
self.ui = ui
# Statistics
self.total_posts = 0
self.total_topics = 0
self.total_users = 0
self.current_posts = 0
self.current_topics = 0
self.current_users = 0
self.startdate = 0
self.record_online_date = 0
self.record_online_users = 0
self.site_name = ""
self.site_desc = ""
self.dump_time = 0
self.smilies = {}
self.users = {}
self.forums = {}
self.announcements = []
self.linkrewriter = LinkRewriter(self)
def _export_(self):
self.logger.info('Récupération des statistiques')
response = self.session.get("/statistics")
document = PyQuery(response.text)
# Go through the table of statistics and save the relevant
# ones
stats = {}
for element in document.find("table.forumline tr"):
e = PyQuery(element)
stats[e("td span").eq(0).text()] = e("td span").eq(1).text()
stats[e("td span").eq(2).text()] = e("td span").eq(3).text()
self.total_posts = int(stats["Messages"])
self.total_topics = int(stats["Nombre de sujets ouvert dans le forum"])
self.total_users = int(stats["Nombre d'utilisateurs"])
self.startdate = parse_date(stats["Ouverture du forum"])
self.record_online_date = parse_date(
stats["Date du record de connexions"])
self.record_online_users = int(
stats["Nombre record d'utilisateurs connectés en même temps"])
self.site_name = document("div.maintitle").eq(0).text().strip(" \n")
self.site_desc = document("div.maintitle").siblings("span.gen").eq(
0).text().strip(" \n")
self.logger.debug('Messages : %d', self.total_posts)
self.logger.debug('Sujets : %d', self.total_topics)
self.logger.debug('Membres : %d', self.total_users)
# Add the children nodes, which respectively handle the
# exportation of the smilies, the users and the message
self.add_child(Smilies())
if self.config["use_ocr"]:
# Use Optical Character Recognition to get the users'
# emails
self.add_child(OcrUsers())
else:
self.add_child(Users())
self.add_child(Groups())
self.add_child(Forums())
def _dump_(self, sqlfile):
self.logger.info("Création du fichier phpbb.sql")
self.dump_time = int(time.time())
# Clean tables
sqlfile.truncate("users")
sqlfile.truncate("user_group")
sqlfile.truncate("bots")
sqlfile.truncate("forums")
sqlfile.truncate("acl_groups")
sqlfile.truncate("topics")
sqlfile.truncate("topics_posted")
sqlfile.truncate("posts")
sqlfile.truncate("privmsgs")
sqlfile.truncate("privmsgs_to")
sqlfile.truncate("bbcodes")
# Update configuration and statistics
sqlfile.set_config("board_startdate", self.startdate)
sqlfile.set_config("default_lang", self.config["default_lang"])
sqlfile.set_config("record_online_date", self.record_online_date)
sqlfile.set_config("record_online_users", self.record_online_users)
sqlfile.set_config("sitename", self.site_name)
sqlfile.set_config("site_desc", self.site_desc)
num_posts = sum(1 for _ in self.get_posts())
num_topics = sum(1 for _ in self.get_topics())
num_users = sum(1 for _ in self.users)
sqlfile.set_config("num_posts", num_posts)
sqlfile.set_config("num_topics", num_topics)
sqlfile.set_config("num_users", num_users)
newest_user_oldid = max(self.users)
newest_user = self.users[newest_user_oldid]
sqlfile.set_config("newest_user_id", newest_user.newid)
sqlfile.set_config("newest_username", newest_user.name)
sqlfile.set_config("newest_user_colour", newest_user.colour)
#Add user columns
#shouldn't they be located to another table ?
sqlfile.update_config("users", "user_sex", "varchar(1) default NULL")
sqlfile.update_config("users", "user_date_of_birth",
"int(11) default NULL")
sqlfile.update_config("users", "user_twitter", "varchar(255)")
sqlfile.update_config("users", "user_facebook", "varchar(255)")
sqlfile.update_config("users", "user_skype", "varchar(255)")
#Own columns (specific to my own forum)
sqlfile.update_config("users", "user_nom_bat", "varchar(255)")
sqlfile.update_config("users", "user_modele_bat", "varchar(255)")
sqlfile.update_config("users", "user_port_bat", "varchar(255)")
sqlfile.update_config("users", "user_mmsi_bat", "varchar(255)")
# Add bbcodes tags
for bbcode in phpbb.BBCODES:
sqlfile.insert("bbcodes", bbcode)
def __setstate__(self, state):
Node.__setstate__(self, state)
self.linkrewriter = LinkRewriter(self)
# TODO : recompute current counts
def save(self):
"""
Dump the tree representing the forum in a pickle file
"""
self.logger.info("Sauvegarde de l'état courant.")
with open("save.pickle", "wb") as fileobj:
pickle.dump(self, fileobj, 2)
def get_topics(self):
"""
Iterator on the topics of the forum
"""
for forum in self.forums.values():
for topic in forum.get_topics():
yield topic
def get_posts(self):
"""
Iterator on the posts of the forum
"""
for topic in self.get_topics():
for post in topic.get_posts():
yield post
