def normalize_system_list(systems):
if systems is None:
return []
else:
# Sort in the order W, M, L so the changelog script works
# Should have used a set for this property :(
return sorted([normalize_system(s) for s in systems], reverse=True)
def load_support_command_v1(data):
source = FallbackDict(data, SUPPORT_COMMAND_V1_DEFAULTS)
return model.SupportCommandV1(
language = source["languages"][0],
executable = source["executable"],
product_id = int_or_none(source["gameID"]),
system = normalize_system(source["systems"][0])
)
def load_depot_v1(data):
source = FallbackDict(data, DEPOT_V1_DEFAULTS)
return model.DepotV1(
languages = source["languages"],
size = int(source["size"]),
game_ids = [int(x) for x in source["gameIDs"]],
system = normalize_system(source["systems"][0]),
manifest = source["manifest"]
)
def parse_softwaredls(software_cont):
return [
model.SoftwareDownload(
id=dl["id"],
name=dl["name"],
total_size=dl["total_size"],
os=normalize_system(dl["os"]),
language=model.Language(dl["language"], dl["language_full"]),
version=dl["version"],
files=[parse_file(dlfile) for dlfile in dl["files"]])
for dl in software_cont
]
def convert_softwaredl(l_dl, prod_id):
return new_model.SoftwareDownload(
id = l_dl.slug,
name = l_dl.name,
total_size = l_dl.total_size,
os = normalize_system(l_dl.os),
language = new_model.Language(
code=l_dl.language,
name=l_dl.language
),
version = l_dl.version,
files = convert_files(l_dl.files, prod_id)
)
def load_repository_v2(data):
source = FallbackDict(data, REPOSITORY_V2_DEFAULTS)
return model.RepositoryV2(
base_product_id = int_or_none(source["baseProductId"]),
client_id = source["clientId"],
client_secret = source["clientSecret"],
cloudsaves = [load_cloudsave_v2(x) for x in source["cloudSaves"]],
dependencies = source["dependencies"],
depots = [load_depot_v2(x) for x in source["depots"]],
install_directory = source["installDirectory"],
offline_depot = load_depot_v2(source["offlineDepot"]),
platform = normalize_system(source["platform"]),
products = [load_repositoryproduct_v2(x) for x in source["products"]],
script_interpreter = source["scriptInterpreter"],
tags = source["tags"]
)
def extract_properties_v0(prod, v0_cont):
prod.id = v0_cont["id"]
prod.access = 1
prod.title = v0_cont["title"]
prod.type = v0_cont["game_type"]
prod.slug = v0_cont["slug"]
prod.cs_systems = []
for cs_name in ["windows", "osx", "linux"]:
if v0_cont["content_system_compatibility"][cs_name]:
prod.cs_systems.append(normalize_system(cs_name))
prod.cs_systems.sort(reverse=True)
prod.store_date = parse_datetime(v0_cont["release_date"])
prod.is_in_development = v0_cont["in_development"]["active"]
#prod.is_pre_order = v0_cont["is_pre_order"] # deprecated
prod.image_logo = extract_imageid(v0_cont["images"]["logo"])
prod.image_background = extract_imageid(v0_cont["images"]["background"])
prod.image_icon = extract_imageid(v0_cont["images"]["sidebarIcon"])
prod.link_forum = v0_cont["links"]["forum"]
prod.link_store = v0_cont["links"]["product_card"]
prod.link_support = v0_cont["links"]["support"]
prod.screenshots = [x["image_id"] for x in v0_cont.get("screenshots", [])]
prod.videos = [
model.Video(video_url=v["video_url"],
thumbnail_url=v["thumbnail_url"],
provider=v["provider"]) for v in v0_cont.get("videos", [])
]
if v0_cont["dlcs"]:
prod.dlcs = [x["id"] for x in v0_cont["dlcs"]["products"]]
prod.changelog = v0_cont["changelog"] or None
def parse_file(file_cont):
return model.File(id=str(file_cont["id"]),
size=file_cont["size"],
downlink=file_cont["downlink"])
def parse_bonusdls(bonus_cont):
return [
model.BonusDownload(
id=str(dl["id"]),
name=dl["name"],
total_size=dl["total_size"],
bonus_type=dl["type"],
count=dl["count"],
files=[parse_file(dlfile) for dlfile in dl["files"]])
for dl in bonus_cont
]
prod.dl_bonus = parse_bonusdls(v0_cont["downloads"]["bonus_content"])
def parse_softwaredls(software_cont):
return [
model.SoftwareDownload(
id=dl["id"],
name=dl["name"],
total_size=dl["total_size"],
os=normalize_system(dl["os"]),
language=model.Language(dl["language"], dl["language_full"]),
version=dl["version"],
files=[parse_file(dlfile) for dlfile in dl["files"]])
for dl in software_cont
]
prod.dl_installer = parse_softwaredls(v0_cont["downloads"]["installers"])
prod.dl_langpack = parse_softwaredls(
v0_cont["downloads"]["language_packs"])
prod.dl_patch = parse_softwaredls(v0_cont["downloads"]["patches"])
def extract_properties_v2(prod, v2_cont):
v2_embed = v2_cont["_embedded"]
v2_links = v2_cont["_links"]
prod.features = [
model.Feature(id=x["id"], name=x["name"]) for x in v2_embed["features"]
]
localizations_map = collections.defaultdict(lambda: model.Localization())
for loc in v2_embed["localizations"]:
loc_embed = loc["_embedded"]
localization = localizations_map[loc_embed["language"]["code"]]
localization.code = loc_embed["language"]["code"]
localization.name = loc_embed["language"]["name"]
if loc_embed["localizationScope"]["type"] == "text":
localization.text = True
elif loc_embed["localizationScope"]["type"] == "audio":
localization.audio = True
prod.localizations = list(localizations_map.values())
prod.tags = [
model.Tag(id=x["id"], level=x["level"], name=x["name"], slug=x["slug"])
for x in v2_embed["tags"]
]
prod.comp_systems = [
normalize_system(support_entry["operatingSystem"]["name"])
for support_entry in v2_embed["supportedOperatingSystems"]
]
prod.comp_systems.sort(reverse=True)
prod.is_using_dosbox = v2_cont["isUsingDosBox"]
prod.developers = [x["name"] for x in v2_embed["developers"]]
prod.publisher = v2_embed["publisher"]["name"]
prod.copyright = v2_cont["copyrights"] or None
prod.global_date = parse_datetime(
v2_embed["product"].get("globalReleaseDate"))
if "gogRating" in v2_embed and v2_embed["gogRating"] is not None:
prod.age_rating = v2_embed["gogRating"].get("ageRating")
if "galaxyBackgroundImage" in v2_links:
prod.image_galaxy_background = extract_imageid(
v2_links["galaxyBackgroundImage"]["href"])
prod.image_boxart = extract_imageid(v2_links["boxArtImage"]["href"])
prod.image_icon_square = extract_imageid(v2_links["iconSquare"]["href"])
prod.editions = [
model.Edition(id=ed["id"],
name=ed["name"],
has_product_card=ed["hasProductCard"])
for ed in v2_embed["editions"]
]
prod.includes_games = [
extract_prodid(link["href"])
for link in v2_links.get("includesGames", [])
]
prod.is_included_in = [
extract_prodid(link["href"])
for link in v2_links.get("isIncludedInGames", [])
]
prod.required_by = [
extract_prodid(link["href"])
for link in v2_links.get("isRequiredByGames", [])
]
prod.requires = [
extract_prodid(link["href"])
for link in v2_links.get("requiresGames", [])
]
if v2_embed["series"]:
prod.series = model.Series(id=v2_embed["series"]["id"],
name=v2_embed["series"]["name"])
prod.description = v2_cont["description"]
def convert_product(l_prod):
n_prod = new_model.Product()
n_prod.id = l_prod.id
def is_add_product_record(changerec):
return changerec.action == "add" and changerec.type_prim == "product"
added_record = list(filter(is_add_product_record, l_prod.changes))
if added_record:
n_prod.added_on = added_record[0].timestamp.replace(tzinfo=datetime.timezone.utc)
#n_prod.last_updated
n_prod.title = l_prod.title
n_prod.type = l_prod.product_type
n_prod.slug = l_prod.slug
n_prod.access = l_prod.access
#n_prod.features = l_prod.
#n_prod.localizations = l_prod.
#n_prod.tags = l_prod.
n_prod.cs_systems = normalize_system_list(l_prod.cs_systems)
n_prod.comp_systems = normalize_system_list(l_prod.comp_systems)
#n_prod.is_using_dosbox = l_prod.
#n_prod.developers = l_prod.
#n_prod.publisher = l_prod.
#n_prod.copyright = l_prod.
n_prod.global_date = date_to_datetime(l_prod.release_date)
n_prod.store_date = date_to_datetime(l_prod.store_date)
n_prod.is_in_development = l_prod.development_active
n_prod.is_pre_order = l_prod.is_pre_order
#n_prod.sale_rank = l_prod.
n_prod.image_logo = l_prod.image_logo
n_prod.image_background = l_prod.image_background
n_prod.image_icon = l_prod.image_icon
#n_prod.image_galaxy_background = l_prod.
#n_prod.image_boxart = l_prod.
#n_prod.image_icon_square = l_prod.
#n_prod.link_forum = l_prod.
#n_prod.link_store = l_prod.
#n_prod.link_support = l_prod.
n_prod.screenshots = [l_scr.image_id for l_scr in l_prod.screenshots]
n_prod.videos = [
new_model.Video(
provider="youtube",
video_url=f"https://www.youtube.com/embed/{ l_video.video_id }?wmode=opaque&rel=0",
thumbnail_url=f"https://img.youtube.com/vi/{ l_video.video_id }/hqdefault.jpg"
) for l_video in l_prod.videos
]
#n_prod.editions = l_prod.
#n_prod.includes_games = l_prod.
#n_prod.is_included_in = l_prod.
#n_prod.required_by = l_prod.
if l_prod.base_prod_id:
n_prod.requires = [l_prod.base_prod_id]
#n_prod.series = l_prod.
n_prod.dlcs = [l_dlc.id for l_dlc in l_prod.dlcs]
n_prod.description = l_prod.description_full
n_prod.changelog = l_prod.changelog or None
for l_dl in l_prod.downloads:
if l_dl.deleted:
continue
if l_dl.type == "bonus_content":
n_prod.dl_bonus.append(convert_bonusdl(l_dl, l_prod.id))
elif l_dl.type == "installers":
n_prod.dl_installer.append(convert_softwaredl(l_dl, l_prod.id))
elif l_dl.type == "language_packs":
n_prod.dl_langpack.append(convert_softwaredl(l_dl, l_prod.id))
elif l_dl.type == "patches":
n_prod.dl_patch.append(convert_softwaredl(l_dl, l_prod.id))
for l_build in l_prod.builds:
if l_build.generation == 1:
manifest_url = "https://cdn.gog.com/content-system/v1/manifests/{}/windows/{}/repository.json".format(
l_prod.id, l_build.legacy_build_id)
elif l_build.generation == 2:
manifest_url = "https://cdn.gog.com/content-system/v2/meta/{}/{}/{}".format(
l_build.meta_id[0:2], l_build.meta_id[2:4], l_build.meta_id)
n_prod.builds.append(new_model.Build(
id = l_build.build_id,
product_id = l_build.prod_id,
os = normalize_system(l_build.os),
branch = None,
version = l_build.version,
tags = l_build.tags,
public = l_build.public,
date_published = l_build.date_published.astimezone(datetime.timezone.utc),
generation = l_build.generation,
legacy_build_id = l_build.legacy_build_id,
meta_id = l_build.meta_id,
link = manifest_url,
listed = True
))
return n_prod
def convert_changelog(l_prod):
l_changes = l_prod.changes
if not l_changes:
return []
# Group changes by date
l_changes_groups = []
last_date = l_changes[0].timestamp
l_cur_group = []
for l_changerec in l_changes:
if l_changerec.timestamp == last_date:
l_cur_group.append(l_changerec)
else:
l_changes_groups.append(l_cur_group)
l_cur_group = [l_changerec]
last_date = l_changerec.timestamp
l_changes_groups.append(l_cur_group)
dummy_prod = DummyProduct()
dummy_prod.id = l_prod.id
dummy_prod.title = l_prod.title
dummy_prod.comp_systems = normalize_system_list(l_prod.comp_systems)
dummy_prod.access = l_prod.access
dummy_prod.downloads = []
for l_dl in l_prod.downloads:
if l_dl.type == "bonus_content":
dummy_prod.downloads.append(
MetaDownload(
type = l_dl.type,
deleted = l_dl.deleted,
download = convert_bonusdl(l_dl, l_prod.id)
)
)
else:
dummy_prod.downloads.append(
MetaDownload(
type = l_dl.type,
deleted = l_dl.deleted,
download = convert_softwaredl(l_dl, l_prod.id)
)
)
# Moving in time backwards, so the previous state is actually in the future
future_productstate = dummy_prod
changelog = []
for l_ch_group in l_changes_groups:
cur_productstate = copy.deepcopy(future_productstate)
cur_date = l_ch_group[0].timestamp.replace(tzinfo=datetime.timezone.utc)
prod_changelogger = changelogger.Changelogger(future_productstate, cur_productstate, cur_date)
for l_change in l_ch_group:
action_type = l_change.action_type
if action_type == "add product":
prod_changelogger.prod_added()
# I decided to not convert this entry because it's 99% false positives
elif action_type == "change product.access":
#cur_productstate.access = int(l_change.old)
pass
elif action_type == "change product.cs":
pass
elif action_type == "change product.os":
cur_productstate.comp_systems = [normalize_system(s) for s in l_change.old.split(",")]
elif action_type == "change product.title":
cur_productstate.title = l_change.old
elif action_type == "change product.forum_slug":
pass
elif action_type == "add download":
cur_productstate.download_by_id(l_change.resource).deleted = True
elif action_type == "del download":
cur_productstate.download_by_id(l_change.resource).deleted = False
elif action_type == "change download.version":
cur_productstate.download_by_id(l_change.resource).download.version = l_change.old
elif action_type == "change download.name":
cur_productstate.download_by_id(l_change.resource).download.name = l_change.old
elif action_type == "change download.total_size":
cur_productstate.download_by_id(l_change.resource).download.total_size = int(l_change.old)
else:
raise RuntimeError("Unknown change type " + l_change.action_type)
prod_changelogger.property("title")
prod_changelogger.property("comp_systems")
prod_changelogger.property("access")
prod_changelogger.downloads("bonus")
prod_changelogger.downloads("installer")
prod_changelogger.downloads("langpack")
prod_changelogger.downloads("patch")
changelog += prod_changelogger.entries
future_productstate = cur_productstate
changelog.reverse() # reverse so entries are sorted oldest to most recent
return changelog