def load_from_db(self, _recursive=True):
""" Fetch data from database, if _recursive is True fetch each steps too """
db = Database()
result = db.select_one(
'''
SELECT questNumber, questName
FROM quest
WHERE questId = ?
''', (self.m_id, ))
if result:
self.m_number = result[0]
self.m_name = result[1]
result = db.select_all(
'''
SELECT questId, stepNumber
FROM step
WHERE step.questId = ?
ORDER BY stepNumber
''', (self.m_id, ))
if result:
for row in result:
self.add_step(Step(
row[0], # questId
row[1], # stepNumber
))
if _recursive:
for step in self.m_steps:
step.load_from_db()
def get_stats():
database = Database()
result = database.execute("""SELECT
(SELECT COUNT(id) FROM manga) AS manga_amount,
(SELECT COUNT(id) FROM chapter) AS chapter_amount,
(SELECT COUNT(id) FROM page) AS page_amount"""
)
return json.dumps(result[0])
def register(_username, _password):
""" Register new user in database with couple(username, password), return True if add, False if doesnt """
db = Database()
if db.select_one('''SELECT username FROM user WHERE username LIKE ?''', (_username, )) is not None:
return "Username already taken"
else:
pwh = str(bcrypt.hashpw(_password.encode('utf-8'), bcrypt.gensalt()))
db.update("INSERT INTO user(username,password) VALUES(?,?)", (_username, pwh[2:(len(pwh) - 1)]))
return User.login(_username, _password)
def save(self):
"""
Save the manga in the database
"""
database = Database()
query = """INSERT INTO manga VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s)"""
database.execute(query, [
self.muID, self.url, None, self.title, self.description,
self.status, 0, None
])
self.id = database.last_inserted_id()
def load_to_db(self):
""" Persist instance to database """
db = Database()
db.update(
'''
UPDATE step
SET textOfTheStep = ?
WHERE questId = ?
AND stepNumber = ?
''',
(self.m_text if self.m_text else "", self.quest_id, self.m_number))
def save(self):
"""
Saves the manga pages
"""
database = Database()
check_query = """SELECT id FROM page WHERE img_url=%s"""
insert_query = """INSERT INTO page VALUES (NULL, %s, %s, %s)"""
for page in self.pages:
result = database.execute(check_query, [page])
if result is ():
database.execute(insert_query,
[page, self.chapter_id, self.manga_id])
def load_to_db(self, _recursive=True):
""" Persist instance to database, if _recursive is True persist each steps too """
db = Database()
db.update(
'''
UPDATE quest
SET questNumber = ?
WHERE questId = ?
''', (self.m_name if self.m_name else "", self.m_id))
if _recursive:
for step in self.m_steps:
step.load_to_db()
def save(self):
"""
Save the chapter in the database
"""
new_chapters = 0
database = Database()
check_query = """SELECT id, all_pages FROM chapter WHERE manga_id=%s AND number=%s"""
insert_query = """INSERT INTO chapter VALUES (NULL, %s, %s, 0, %s)"""
update_query = """UPDATE chapter SET all_pages=1 WHERE id=%s"""
for url in self.urls:
chapter_id = None
chapter_number = url.split("/")[-1]
result = database.execute(check_query,
[self.manga_id, chapter_number])
if result is ():
database.execute(insert_query,
[chapter_number, url, self.manga_id])
chapter_id = database.last_inserted_id()
new_chapters += 1
else:
chapter_id = result[0][0]
self.all_pages = True if result[0][1] == 1 else False
if not self.all_pages:
chapter_pages = Pages(self.manga_id, chapter_id, url)
chapter_pages.save()
database.execute(update_query, [chapter_id])
self.log.info("Found %s new chapter(s)" % new_chapters)
def load_from_db(self):
""" Fetch data from database """
db = Database()
result = db.select_one(
'''
SELECT textOfTheStep, stepOptions
FROM step
WHERE questId = ? AND stepNumber = ?
''', (self.quest_id, self.m_number))
if result:
self.text = result[0]
self.options = result[1]
return result
def login(_username, _password):
""" Login with a check of the (username, password) couple in Database """
db = Database()
result = db.select_one('''SELECT idUser, password FROM user WHERE username LIKE ?''', (_username, ))
if result is not None:
if result[1] != "":
if bcrypt.checkpw(_password.encode('utf-8'), result[1].encode('utf-8')):
return User(result[0], _username)
else:
return "Error : Invalid Password"
else:
return "Error : Invalid Username"
else:
return "Error : no user found"
def save(self):
"""
Save the manga artists in the database
"""
new_artists = 0
database = Database()
check_query = """SELECT id FROM artists WHERE name=%s AND manga_id=%s"""
insert_query = """INSERT INTO artists VALUES (NULL, %s, %s)"""
for artist in self.artists:
result = database.execute(check_query, [artist, self.manga_id])
if result is ():
database.execute(insert_query, [artist, self.manga_id])
new_artists += 1
self.log.info("Found %s new artist(s)" % new_artists)
def save(self):
"""
Save the manga authors at the database
"""
new_authors = 0
database = Database()
check_query = """SELECT id FROM authors WHERE name=%s AND manga_id=%s"""
insert_query = """INSERT INTO authors VALUES (NULL, %s, %s)"""
for author in self.authors:
result = database.execute(check_query, [author, self.manga_id])
if result is ():
database.execute(insert_query, [author, self.manga_id])
new_authors += 1
self.log.info("Found %s new author(s)" % new_authors)
def load_from_db(self, _recursive=True):
""" Fetch data from database, if _recursive is True fetch each quest too """
db = Database()
result = db.select_all('''
SELECT questId
FROM quest
ORDER BY questNumber
''')
for row in result:
self.add_quest(Quest(row[0] # questId
))
if _recursive:
for quest in self.m_quests:
quest.load_from_db()
def delete(_username, _password, user):
""" Delete with a check of the password in Database """
db = Database()
result = db.select_one('''SELECT idUser, password FROM user WHERE username LIKE ?''', (_username, ))
if result is not None:
if result[1] != "":
if bcrypt.checkpw(_password.encode('utf-8'), result[1].encode('utf-8')):
for hero in user.heroes:
hero.delete()
db.delete("DELETE FROM user WHERE username = ?", (_username,))
else:
return "Error : Invalid Password"
else:
return "Error : Invalid Username"
else:
return "Error : no user found"
def save(self):
"""
Insert the manga alternative titles
in the database
"""
new_titles = 0
database = Database()
check_query = """SELECT id FROM titles WHERE name=%s AND manga_id=%s"""
insert_query = """INSERT INTO titles VALUES (NULL, %s, %s)"""
for title in self.titles:
result = database.execute(check_query, [title, self.manga_id])
title = title[1:] if title[0] == " " else title
if title != "-" and result is ():
database.execute(insert_query, [title, self.manga_id])
new_titles += 1
self.log.info("Found %s new alternative title(s)" % new_titles)
def __init__(self, url):
self.log = Logging("weeb_crawler")
conf = Config()
conf = conf.get("muID")
self.title_diff_ratio = float(conf['diff_ratio'])
self.url = url
self.id = None
query = """SELECT id FROM manga WHERE page_url=%s"""
database = Database()
result = database.execute(query, [url])
self.file = open("last-manga-content.html", "w")
self.page = self.get_page()
self.title = self.get_title()
self.muID = self.get_mu_id()
if result is ():
self.description = self.get_description()
self.alternative_titles = None
self.gender_tags = None
self.authors = None
self.artists = None
self.status = None
self.get_header_info()
self.save()
self.save_titles()
self.save_authors()
self.save_artists()
self.save_gender()
self.get_covers()
self.get_chapters()
self.log.info("Added new Manga: %s" % self.title)
else:
self.id = result[0][0]
self.get_covers()
self.get_chapters()
self.log.info("Updated Manga: %s" % self.title)
def load_from_db(self):
""" Fetch data from database, if _recursive is True fetch each steps too """
db = Database()
result = db.select_one(
'''
SELECT lvl, weapon, armor, passive, sex, idUser, numQuest, numStep
FROM hero
WHERE nameOfTheHero = ?
''',
(self.name,)
)
if result is not None:
self.lvl = result[0]
self.weapon = result[1]
self.armor = result[2]
self.passive = result[3]
self.sex = result[4]
self.user_id = result[5]
self.current_quest = result[6]
self.current_step = result[7]
def init_heroes(self):
db = Database()
result = db.select_all( # _name, _lvl, _weapon, _armor, _passive
'''
SELECT nameOfTheHero, lvl, weapon, armor, passive
FROM hero
WHERE idUser = ?
''',
(self.id,)
)
# print(f"result: {result}")
if result is not None:
for row in result:
self.add_hero(Hero(
_name=row[0], # nameOfTheHero
_lvl=row[1], # lvl
_weapon=row["weapon"], # weapon
_armor=row[3], # armor
_passive=row[4], # passive
))
else:
self.m_heroes = None
def save(self):
"""
Checks for existing genders on the database,
then links the gender to the manga
"""
gender_id = None
database = Database()
check_query = """SELECT id FROM gender_tags WHERE tag_name=%s"""
insert_query = """INSERT INTO gender_tags VALUES (NULL, %s)"""
foreign_query = """INSERT INTO manga_gender_tags VALUES (%s, %s)"""
for gender in self.genders:
result = database.execute(check_query, [gender])
if result is ():
database.execute(insert_query, [gender])
gender_id = database.last_inserted_id()
else:
gender_id = result[0][0]
database.execute(foreign_query, [gender_id, self.manga_id])
def load_to_db(self):
""" Persist instance to database, if _recursive is True persist each steps too """
db = Database()
if db.select_one("SELECT idHero FROM hero WHERE nameOfTheHero = ?", (self.name,)):
db.update(
'''
UPDATE hero
SET lvl = ?,weapon = ?, armor = ?, passive = ?, sex = ?, idUser = ?,numQuest = ?, numStep = ?
WHERE nameOfTheHero = ?
''',
(self.lvl,
self.weapon["name"],
self.armor,
self.passive,
1 if self.m_sex else 0,
self.user_id,
self.current_quest,
self.current_step,
self.name)
)
else:
db.add(
'''
INSERT INTO hero(nameOfTheHero, lvl, weapon, armor, passive, sex, idUser, numQuest, numStep)
VALUES(?,?,?,?,?,?,?,?,?)
''',
(self.name,
self.lvl,
self.weapon["name"],
self.armor,
self.passive,
1 if self.m_sex else 0,
self.user_id,
self.current_quest,
self.current_step)
)
def delete(self):
""" Delete table tuple corresponding to current instance """
db = Database()
db.delete("DELETE FROM hero WHERE nameOfTheHero = ?", (self.name,))
def check_hero_available(_name):
""" Verify if a hero with _name already exist, return True if None, False il already exist """
db = Database()
if not db.select_one('''SELECT nameOfTheHero FROM hero WHERE nameOfTheHero = ?''', (_name,)):
return True
return False