def save(self):
"""
Save the chapter in the database
"""
new_chapters = 0
database = Database()
check_query = """SELECT id, all_pages FROM chapter WHERE manga_id=%s AND number=%s"""
insert_query = """INSERT INTO chapter VALUES (NULL, %s, %s, 0, %s)"""
update_query = """UPDATE chapter SET all_pages=1 WHERE id=%s"""
for url in self.urls:
chapter_id = None
chapter_number = url.split("/")[-1]
result = database.execute(check_query,
[self.manga_id, chapter_number])
if result is ():
database.execute(insert_query,
[chapter_number, url, self.manga_id])
chapter_id = database.last_inserted_id()
new_chapters += 1
else:
chapter_id = result[0][0]
self.all_pages = True if result[0][1] == 1 else False
if not self.all_pages:
chapter_pages = Pages(self.manga_id, chapter_id, url)
chapter_pages.save()
database.execute(update_query, [chapter_id])
self.log.info("Found %s new chapter(s)" % new_chapters)
def save(self):
"""
Save the manga in the database
"""
database = Database()
query = """INSERT INTO manga VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s)"""
database.execute(query, [
self.muID, self.url, None, self.title, self.description,
self.status, 0, None
])
self.id = database.last_inserted_id()
def save(self):
"""
Saves the manga pages
"""
database = Database()
check_query = """SELECT id FROM page WHERE img_url=%s"""
insert_query = """INSERT INTO page VALUES (NULL, %s, %s, %s)"""
for page in self.pages:
result = database.execute(check_query, [page])
if result is ():
database.execute(insert_query,
[page, self.chapter_id, self.manga_id])
def save(self):
"""
Save the manga authors at the database
"""
new_authors = 0
database = Database()
check_query = """SELECT id FROM authors WHERE name=%s AND manga_id=%s"""
insert_query = """INSERT INTO authors VALUES (NULL, %s, %s)"""
for author in self.authors:
result = database.execute(check_query, [author, self.manga_id])
if result is ():
database.execute(insert_query, [author, self.manga_id])
new_authors += 1
self.log.info("Found %s new author(s)" % new_authors)
def save(self):
"""
Save the manga artists in the database
"""
new_artists = 0
database = Database()
check_query = """SELECT id FROM artists WHERE name=%s AND manga_id=%s"""
insert_query = """INSERT INTO artists VALUES (NULL, %s, %s)"""
for artist in self.artists:
result = database.execute(check_query, [artist, self.manga_id])
if result is ():
database.execute(insert_query, [artist, self.manga_id])
new_artists += 1
self.log.info("Found %s new artist(s)" % new_artists)
def get_stats():
database = Database()
result = database.execute("""SELECT
(SELECT COUNT(id) FROM manga) AS manga_amount,
(SELECT COUNT(id) FROM chapter) AS chapter_amount,
(SELECT COUNT(id) FROM page) AS page_amount"""
)
return json.dumps(result[0])
def save(self):
"""
Insert the manga alternative titles
in the database
"""
new_titles = 0
database = Database()
check_query = """SELECT id FROM titles WHERE name=%s AND manga_id=%s"""
insert_query = """INSERT INTO titles VALUES (NULL, %s, %s)"""
for title in self.titles:
result = database.execute(check_query, [title, self.manga_id])
title = title[1:] if title[0] == " " else title
if title != "-" and result is ():
database.execute(insert_query, [title, self.manga_id])
new_titles += 1
self.log.info("Found %s new alternative title(s)" % new_titles)
def save(self):
"""
Checks for existing genders on the database,
then links the gender to the manga
"""
gender_id = None
database = Database()
check_query = """SELECT id FROM gender_tags WHERE tag_name=%s"""
insert_query = """INSERT INTO gender_tags VALUES (NULL, %s)"""
foreign_query = """INSERT INTO manga_gender_tags VALUES (%s, %s)"""
for gender in self.genders:
result = database.execute(check_query, [gender])
if result is ():
database.execute(insert_query, [gender])
gender_id = database.last_inserted_id()
else:
gender_id = result[0][0]
database.execute(foreign_query, [gender_id, self.manga_id])
def __init__(self, url):
self.log = Logging("weeb_crawler")
conf = Config()
conf = conf.get("muID")
self.title_diff_ratio = float(conf['diff_ratio'])
self.url = url
self.id = None
query = """SELECT id FROM manga WHERE page_url=%s"""
database = Database()
result = database.execute(query, [url])
self.file = open("last-manga-content.html", "w")
self.page = self.get_page()
self.title = self.get_title()
self.muID = self.get_mu_id()
if result is ():
self.description = self.get_description()
self.alternative_titles = None
self.gender_tags = None
self.authors = None
self.artists = None
self.status = None
self.get_header_info()
self.save()
self.save_titles()
self.save_authors()
self.save_artists()
self.save_gender()
self.get_covers()
self.get_chapters()
self.log.info("Added new Manga: %s" % self.title)
else:
self.id = result[0][0]
self.get_covers()
self.get_chapters()
self.log.info("Updated Manga: %s" % self.title)