def scan_game_database(self, helper, database_name, game_database):
"""
:type helper: ScanHelper
:type database_name: str
:type game_database: fsgs.GameDatabase.GameDatabase
"""
database_cursor = helper.database.cursor()
# this holds a list of game entry UUIDs which must exist / be checked
# after variants have been processed
# ensure_updated_games = set()
game_database_cursor = game_database.cursor()
# this list will contain game entries which are not variants
game_rows = []
game_database_cursor.execute(
"SELECT id, uuid FROM game WHERE data != ''")
for row in game_database_cursor:
if self.stop_check():
return
variant_id, variant_uuid_bin = row
variant_uuid = binary_to_uuid(variant_uuid_bin)
update_stamp = variant_id
existing_variant = helper.existing_variants.get(
variant_uuid, (None, None, None))
existing_game = helper.existing_games.get(variant_uuid,
(None, None, None))
if update_stamp == existing_game[0]:
# Game entry, not updated.
game_rows.append(row)
continue
if update_stamp == existing_variant[0]:
# entry was already updated and has not changed
if existing_variant[1] and not helper.deleted_files:
# have this entry already and no files have been deleted
# since the last time
# print("skipping variant (no deleted files)")
helper.variant_seen(variant_uuid)
continue
if not existing_variant[1] and not helper.added_files:
# do not have this entry, but no files have been added
# since the last time
# print("skipping variant (no added files)")
helper.variant_seen(variant_uuid)
continue
else:
# when the game entry has changed, we always check it
# regardless of file database status, since file_list may
# have been changed, or download status... (or other info
# needs to be corrected)
# print("[SCANNER] Changed: {}".format(variant_uuid))
pass
self.scan_count += 1
self.set_status(
gettext("Scanning game variants ({count} scanned)").format(
count=self.scan_count),
variant_uuid,
)
try:
doc = self.fsgs.game.get_game_values_for_id(
game_database, variant_id)
except IncompleteGameException as e:
print("[SCANNER]", repr(e))
continue
file_list_json = doc.get("file_list", "")
if not file_list_json:
# not a game variant... (parent game only probably)
game_rows.append(row)
continue
entry_type = int(doc.get("_type", "0"))
if (entry_type & GAME_ENTRY_TYPE_VARIANT) == 0:
# game entry is not tagged with variant -- add to game list
# instead
game_rows.append(row)
continue
helper.variant_seen(variant_uuid)
all_files_found = True
try:
file_list = json.loads(file_list_json)
except Exception:
# invalid JSON string
# FIXME: log error
all_files_found = False
file_list = []
for file_item in file_list:
if file_item["name"].endswith("/"):
# skip directory entries:
continue
result = self.fsgs.file.check_sha1(file_item["sha1"])
if not result:
all_files_found = False
break
if all_files_found:
have_variant = 4
elif doc.get("download_file", ""):
have_variant = 2
elif doc.get("download_page", ""):
have_variant = 1
else:
have_variant = 0
parent_uuid = doc.get("parent_uuid", "")
variant_name = doc.get("variant_name", "")
# published_variant = doc.get("publish", "")
published_variant = doc.get("__publish_hack__", "")
if not published_variant:
published_variant = None
elif published_variant == "1":
published_variant = 1
else:
published_variant = 0
game_variant_id = existing_variant[2]
if not game_variant_id:
# variant is not in database
database_cursor.execute(
"INSERT INTO game_variant (uuid) VALUES (?)",
(variant_uuid, ),
)
game_variant_id = database_cursor.lastrowid
database_cursor.execute(
"UPDATE game_variant SET name = ?, game_uuid = ?, have = ?, "
"update_stamp = ?, database = ?, published = ? WHERE id = ?",
(
variant_name,
parent_uuid,
have_variant,
update_stamp,
database_name,
published_variant,
game_variant_id,
),
)
# ensure_updated_games.add(parent_uuid)
for row in game_rows:
if self.stop_check():
return
game_id, game_uuid_bin = row
game_uuid = binary_to_uuid(game_uuid_bin)
update_stamp = game_id
existing_game = helper.existing_games.get(game_uuid,
(None, None, None))
if update_stamp == existing_game[0]:
# after the loop has run its course, games to be removed
# are left in existing_games
helper.game_seen(game_uuid)
continue
self.scan_count += 1
self.set_status(
gettext("Scanning games ({count} scanned)").format(
count=self.scan_count),
game_uuid,
)
doc = game_database.get_game_values(game_id)
entry_type = int(doc.get("_type", "0"))
if (entry_type & GAME_ENTRY_TYPE_GAME) == 0:
continue
# after the loop has run its course, games to be removed
# are left in existing_games
try:
del helper.existing_games[game_uuid]
except KeyError:
pass
game_name = doc.get("game_name", "")
game_subtitle = doc.get("game_subtitle", "")
platform = doc.get("platform", "")
tags = doc.get("tags", "")
year = doc.get("year", "")
publisher = doc.get("publisher", "")
front_sha1 = doc.get("front_sha1", "")
title_sha1 = doc.get("title_sha1", "")
screen1_sha1 = doc.get("screen1_sha1", "")
screen2_sha1 = doc.get("screen2_sha1", "")
screen3_sha1 = doc.get("screen3_sha1", "")
screen4_sha1 = doc.get("screen4_sha1", "")
screen5_sha1 = doc.get("screen5_sha1", "")
thumb_sha1 = doc.get("thumb_sha1", "")
backdrop_sha1 = doc.get("backdrop_sha1", "")
published = doc.get("publish", "")
sort_key = doc.get("sort_key", "")
if not sort_key:
# FIXME: handle the/a (etc)
sort_key = game_name.lower()
game_id = existing_game[2]
if not game_id:
# game is not in database
database_cursor.execute("INSERT INTO game (uuid) VALUES (?)",
(game_uuid, ))
game_id = database_cursor.lastrowid
search_terms = set()
for key in [
"game_name",
"full_name",
"game_name_alt",
"search_terms",
"game_subtitle",
]:
value = doc.get(key, "")
if value:
search_terms.update(
GameNameUtil.extract_index_terms(value))
letter = ""
if len(sort_key) > 0:
letter = sort_key[0].lower()
if letter in "abcdefghijklmnopqrstuvwxyz":
pass
elif letter in "0123456789":
letter = "#"
else:
letter = "?"
search_terms.add("l:" + letter)
for tag in tags.split(","):
tag = tag.strip().lower()
search_terms.add("t:" + tag)
if year:
search_terms.add("y:" + str(year))
if platform:
search_terms.add("s:" + platform.lower())
if thumb_sha1:
search_terms.add("t:thumb")
if backdrop_sha1:
search_terms.add("t:backdrop")
backdrop_image = "sha1:{}/{}/{}/{}/{}".format(
backdrop_sha1,
doc.get("backdrop_zoom", "1.0"),
doc.get("backdrop_zoom", "1.0"),
doc.get("backdrop_halign", "0.5"),
doc.get("backdrop_valign", "0.5"),
)
else:
backdrop_image = ""
# if published:
# # search_terms.add("t:published")
# pass
# else:
# search_terms.add("t:unpublished")
min_players = 0
max_players = 0
sim_players = 0
players = doc.get("players", "")
if players == "1":
min_players = max_players = 1
search_terms.add("p:1")
elif players:
try:
parts1 = players.split("-")
parts2 = parts1[1].split("(")
min_players = int(parts1[0].strip())
max_players = int(parts2[0].strip())
sim_players = int(parts2[1].strip(" )"))
except Exception as e:
print("error parsing players")
print(repr(e))
if min_players > 0:
if max_players > 0:
# we ignore players = 1 here, that is reserved for games
# with max 1 players
for i in range(min_players + 1, max_players + 1):
search_terms.add("p:{0}".format(i))
if sim_players > 0:
# we ignore players = 1 here, that is reserved for games
# with max 1 players
for i in range(min_players + 1, sim_players + 1):
search_terms.add("p:{0}s".format(i))
database_cursor.execute(
"UPDATE game SET name = ?, update_stamp = ?, sort_key = ?, "
"platform = ?, "
"publisher = ?, year = ?, front_image = ?, title_image = ?, "
"screen1_image = ?, screen2_image = ?, screen3_image = ?, "
"screen4_image = ?, screen5_image = ?, "
"thumb_image = ?, backdrop_image = ?, "
"adult = ?, published = ?, subtitle = ? WHERE id = ?",
(
game_name,
update_stamp,
sort_key,
platform,
publisher or "",
year or 0,
"sha1:" + front_sha1 if front_sha1 else "",
"sha1:" + title_sha1 if title_sha1 else "",
"sha1:" + screen1_sha1 if screen1_sha1 else "",
"sha1:" + screen2_sha1 if screen2_sha1 else "",
"sha1:" + screen3_sha1 if screen3_sha1 else "",
"sha1:" + screen4_sha1 if screen4_sha1 else "",
"sha1:" + screen5_sha1 if screen5_sha1 else "",
"sha1:" + thumb_sha1 if thumb_sha1 else "",
backdrop_image,
1 if "t:adult" in search_terms else 0,
1 if published == "1" else 0,
game_subtitle,
game_id,
),
)
helper.database.update_game_search_terms(game_id, search_terms)
def create_search_terms(cls, name):
search_terms = set()
game_name = name.split("(", 1)[0]
search_terms.update(GameNameUtil.extract_index_terms(game_name))
return search_terms
def scan_game_database(self, helper, database_name, game_database):
"""
:type helper: ScanHelper
:type database_name: str
:type game_database: fsgs.GameDatabase.GameDatabase
"""
database_cursor = helper.database.cursor()
# this holds a list of game entry UUIDs which must exist / be checked
# after variants have been processed
# ensure_updated_games = set()
game_database_cursor = game_database.cursor()
# this list will contain game entries which are not variants
game_rows = []
game_database_cursor.execute(
"SELECT id, uuid FROM game WHERE data != ''"
)
for row in game_database_cursor:
if self.stop_check():
return
variant_id, variant_uuid_bin = row
variant_uuid = binary_to_uuid(variant_uuid_bin)
update_stamp = variant_id
existing_variant = helper.existing_variants.get(
variant_uuid, (None, None, None)
)
existing_game = helper.existing_games.get(
variant_uuid, (None, None, None)
)
if update_stamp == existing_game[0]:
# Game entry, not updated.
game_rows.append(row)
continue
if update_stamp == existing_variant[0]:
# entry was already updated and has not changed
if existing_variant[1] and not helper.deleted_files:
# have this entry already and no files have been deleted
# since the last time
# print("skipping variant (no deleted files)")
helper.variant_seen(variant_uuid)
continue
if not existing_variant[1] and not helper.added_files:
# do not have this entry, but no files have been added
# since the last time
# print("skipping variant (no added files)")
helper.variant_seen(variant_uuid)
continue
else:
# when the game entry has changed, we always check it
# regardless of file database status, since file_list may
# have been changed, or download status... (or other info
# needs to be corrected)
# print("[SCANNER] Changed: {}".format(variant_uuid))
pass
self.scan_count += 1
self.set_status(
gettext("Scanning game variants ({count} scanned)").format(
count=self.scan_count
),
variant_uuid,
)
try:
doc = self.fsgs.game.get_game_values_for_id(
game_database, variant_id
)
except IncompleteGameException as e:
print("[SCANNER]", repr(e))
continue
file_list_json = doc.get("file_list", "")
if not file_list_json:
# not a game variant... (parent game only probably)
game_rows.append(row)
continue
entry_type = int(doc.get("_type", "0"))
if (entry_type & GAME_ENTRY_TYPE_VARIANT) == 0:
# game entry is not tagged with variant -- add to game list
# instead
game_rows.append(row)
continue
helper.variant_seen(variant_uuid)
all_files_found = True
try:
file_list = json.loads(file_list_json)
except Exception:
# invalid JSON string
# FIXME: log error
all_files_found = False
file_list = []
for file_item in file_list:
if file_item["name"].endswith("/"):
# skip directory entries:
continue
result = self.fsgs.file.check_sha1(file_item["sha1"])
if not result:
all_files_found = False
break
if all_files_found:
have_variant = 4
elif doc.get("download_file", ""):
have_variant = 2
elif doc.get("download_page", ""):
have_variant = 1
else:
have_variant = 0
parent_uuid = doc.get("parent_uuid", "")
variant_name = doc.get("variant_name", "")
# published_variant = doc.get("publish", "")
published_variant = doc.get("__publish_hack__", "")
if not published_variant:
published_variant = None
elif published_variant == "1":
published_variant = 1
else:
published_variant = 0
game_variant_id = existing_variant[2]
if not game_variant_id:
# variant is not in database
database_cursor.execute(
"INSERT INTO game_variant (uuid) VALUES (?)",
(variant_uuid,),
)
game_variant_id = database_cursor.lastrowid
database_cursor.execute(
"UPDATE game_variant SET name = ?, game_uuid = ?, have = ?, "
"update_stamp = ?, database = ?, published = ? WHERE id = ?",
(
variant_name,
parent_uuid,
have_variant,
update_stamp,
database_name,
published_variant,
game_variant_id,
),
)
# ensure_updated_games.add(parent_uuid)
for row in game_rows:
if self.stop_check():
return
game_id, game_uuid_bin = row
game_uuid = binary_to_uuid(game_uuid_bin)
update_stamp = game_id
existing_game = helper.existing_games.get(
game_uuid, (None, None, None)
)
if update_stamp == existing_game[0]:
# after the loop has run its course, games to be removed
# are left in existing_games
helper.game_seen(game_uuid)
continue
self.scan_count += 1
self.set_status(
gettext("Scanning games ({count} scanned)").format(
count=self.scan_count
),
game_uuid,
)
doc = game_database.get_game_values(game_id)
entry_type = int(doc.get("_type", "0"))
if (entry_type & GAME_ENTRY_TYPE_GAME) == 0:
continue
# after the loop has run its course, games to be removed
# are left in existing_games
try:
del helper.existing_games[game_uuid]
except KeyError:
pass
game_name = doc.get("game_name", "")
game_subtitle = doc.get("game_subtitle", "")
platform = doc.get("platform", "")
tags = doc.get("tags", "")
year = doc.get("year", "")
publisher = doc.get("publisher", "")
front_sha1 = doc.get("front_sha1", "")
title_sha1 = doc.get("title_sha1", "")
screen1_sha1 = doc.get("screen1_sha1", "")
screen2_sha1 = doc.get("screen2_sha1", "")
screen3_sha1 = doc.get("screen3_sha1", "")
screen4_sha1 = doc.get("screen4_sha1", "")
screen5_sha1 = doc.get("screen5_sha1", "")
thumb_sha1 = doc.get("thumb_sha1", "")
backdrop_sha1 = doc.get("backdrop_sha1", "")
published = doc.get("publish", "")
sort_key = doc.get("sort_key", "")
if not sort_key:
# FIXME: handle the/a (etc)
sort_key = game_name.lower()
game_id = existing_game[2]
if not game_id:
# game is not in database
database_cursor.execute(
"INSERT INTO game (uuid) VALUES (?)", (game_uuid,)
)
game_id = database_cursor.lastrowid
search_terms = set()
for key in [
"game_name",
"full_name",
"game_name_alt",
"search_terms",
"game_subtitle",
]:
value = doc.get(key, "")
if value:
search_terms.update(
GameNameUtil.extract_index_terms(value)
)
letter = ""
if len(sort_key) > 0:
letter = sort_key[0].lower()
if letter in "abcdefghijklmnopqrstuvwxyz":
pass
elif letter in "0123456789":
letter = "#"
else:
letter = "?"
search_terms.add("l:" + letter)
for tag in tags.split(","):
tag = tag.strip().lower()
search_terms.add("t:" + tag)
if year:
search_terms.add("y:" + str(year))
if platform:
search_terms.add("s:" + platform.lower())
if thumb_sha1:
search_terms.add("t:thumb")
if backdrop_sha1:
search_terms.add("t:backdrop")
backdrop_image = "sha1:{}/{}/{}/{}/{}".format(
backdrop_sha1,
doc.get("backdrop_zoom", "1.0"),
doc.get("backdrop_zoom", "1.0"),
doc.get("backdrop_halign", "0.5"),
doc.get("backdrop_valign", "0.5"),
)
else:
backdrop_image = ""
# if published:
# # search_terms.add("t:published")
# pass
# else:
# search_terms.add("t:unpublished")
min_players = 0
max_players = 0
sim_players = 0
players = doc.get("players", "")
if players == "1":
min_players = max_players = 1
search_terms.add("p:1")
elif players:
try:
parts1 = players.split("-")
parts2 = parts1[1].split("(")
min_players = int(parts1[0].strip())
max_players = int(parts2[0].strip())
sim_players = int(parts2[1].strip(" )"))
except Exception as e:
print("error parsing players")
print(repr(e))
if min_players > 0:
if max_players > 0:
# we ignore players = 1 here, that is reserved for games
# with max 1 players
for i in range(min_players + 1, max_players + 1):
search_terms.add("p:{0}".format(i))
if sim_players > 0:
# we ignore players = 1 here, that is reserved for games
# with max 1 players
for i in range(min_players + 1, sim_players + 1):
search_terms.add("p:{0}s".format(i))
database_cursor.execute(
"UPDATE game SET name = ?, update_stamp = ?, sort_key = ?, "
"platform = ?, "
"publisher = ?, year = ?, front_image = ?, title_image = ?, "
"screen1_image = ?, screen2_image = ?, screen3_image = ?, "
"screen4_image = ?, screen5_image = ?, "
"thumb_image = ?, backdrop_image = ?, "
"adult = ?, published = ?, subtitle = ? WHERE id = ?",
(
game_name,
update_stamp,
sort_key,
platform,
publisher or "",
year or 0,
"sha1:" + front_sha1 if front_sha1 else "",
"sha1:" + title_sha1 if title_sha1 else "",
"sha1:" + screen1_sha1 if screen1_sha1 else "",
"sha1:" + screen2_sha1 if screen2_sha1 else "",
"sha1:" + screen3_sha1 if screen3_sha1 else "",
"sha1:" + screen4_sha1 if screen4_sha1 else "",
"sha1:" + screen5_sha1 if screen5_sha1 else "",
"sha1:" + thumb_sha1 if thumb_sha1 else "",
backdrop_image,
1 if "t:adult" in search_terms else 0,
1 if published == "1" else 0,
game_subtitle,
game_id,
),
)
helper.database.update_game_search_terms(game_id, search_terms)
def create_search_terms(cls, name):
search_terms = set()
game_name = name.split("(", 1)[0]
search_terms.update(GameNameUtil.extract_index_terms(game_name))
return search_terms
