def normalize_system_list(systems): if systems is None: return [] else: # Sort in the order W, M, L so the changelog script works # Should have used a set for this property :( return sorted([normalize_system(s) for s in systems], reverse=True)
def load_support_command_v1(data): source = FallbackDict(data, SUPPORT_COMMAND_V1_DEFAULTS) return model.SupportCommandV1( language = source["languages"][0], executable = source["executable"], product_id = int_or_none(source["gameID"]), system = normalize_system(source["systems"][0]) )
def load_depot_v1(data): source = FallbackDict(data, DEPOT_V1_DEFAULTS) return model.DepotV1( languages = source["languages"], size = int(source["size"]), game_ids = [int(x) for x in source["gameIDs"]], system = normalize_system(source["systems"][0]), manifest = source["manifest"] )
def parse_softwaredls(software_cont): return [ model.SoftwareDownload( id=dl["id"], name=dl["name"], total_size=dl["total_size"], os=normalize_system(dl["os"]), language=model.Language(dl["language"], dl["language_full"]), version=dl["version"], files=[parse_file(dlfile) for dlfile in dl["files"]]) for dl in software_cont ]
def convert_softwaredl(l_dl, prod_id): return new_model.SoftwareDownload( id = l_dl.slug, name = l_dl.name, total_size = l_dl.total_size, os = normalize_system(l_dl.os), language = new_model.Language( code=l_dl.language, name=l_dl.language ), version = l_dl.version, files = convert_files(l_dl.files, prod_id) )
def load_repository_v2(data): source = FallbackDict(data, REPOSITORY_V2_DEFAULTS) return model.RepositoryV2( base_product_id = int_or_none(source["baseProductId"]), client_id = source["clientId"], client_secret = source["clientSecret"], cloudsaves = [load_cloudsave_v2(x) for x in source["cloudSaves"]], dependencies = source["dependencies"], depots = [load_depot_v2(x) for x in source["depots"]], install_directory = source["installDirectory"], offline_depot = load_depot_v2(source["offlineDepot"]), platform = normalize_system(source["platform"]), products = [load_repositoryproduct_v2(x) for x in source["products"]], script_interpreter = source["scriptInterpreter"], tags = source["tags"] )
def extract_properties_v0(prod, v0_cont): prod.id = v0_cont["id"] prod.access = 1 prod.title = v0_cont["title"] prod.type = v0_cont["game_type"] prod.slug = v0_cont["slug"] prod.cs_systems = [] for cs_name in ["windows", "osx", "linux"]: if v0_cont["content_system_compatibility"][cs_name]: prod.cs_systems.append(normalize_system(cs_name)) prod.cs_systems.sort(reverse=True) prod.store_date = parse_datetime(v0_cont["release_date"]) prod.is_in_development = v0_cont["in_development"]["active"] #prod.is_pre_order = v0_cont["is_pre_order"] # deprecated prod.image_logo = extract_imageid(v0_cont["images"]["logo"]) prod.image_background = extract_imageid(v0_cont["images"]["background"]) prod.image_icon = extract_imageid(v0_cont["images"]["sidebarIcon"]) prod.link_forum = v0_cont["links"]["forum"] prod.link_store = v0_cont["links"]["product_card"] prod.link_support = v0_cont["links"]["support"] prod.screenshots = [x["image_id"] for x in v0_cont.get("screenshots", [])] prod.videos = [ model.Video(video_url=v["video_url"], thumbnail_url=v["thumbnail_url"], provider=v["provider"]) for v in v0_cont.get("videos", []) ] if v0_cont["dlcs"]: prod.dlcs = [x["id"] for x in v0_cont["dlcs"]["products"]] prod.changelog = v0_cont["changelog"] or None def parse_file(file_cont): return model.File(id=str(file_cont["id"]), size=file_cont["size"], downlink=file_cont["downlink"]) def parse_bonusdls(bonus_cont): return [ model.BonusDownload( id=str(dl["id"]), name=dl["name"], total_size=dl["total_size"], bonus_type=dl["type"], count=dl["count"], files=[parse_file(dlfile) for dlfile in dl["files"]]) for dl in bonus_cont ] prod.dl_bonus = parse_bonusdls(v0_cont["downloads"]["bonus_content"]) def parse_softwaredls(software_cont): return [ model.SoftwareDownload( id=dl["id"], name=dl["name"], total_size=dl["total_size"], os=normalize_system(dl["os"]), language=model.Language(dl["language"], dl["language_full"]), version=dl["version"], files=[parse_file(dlfile) for dlfile in dl["files"]]) for dl in software_cont ] prod.dl_installer = parse_softwaredls(v0_cont["downloads"]["installers"]) prod.dl_langpack = parse_softwaredls( v0_cont["downloads"]["language_packs"]) prod.dl_patch = parse_softwaredls(v0_cont["downloads"]["patches"])
def extract_properties_v2(prod, v2_cont): v2_embed = v2_cont["_embedded"] v2_links = v2_cont["_links"] prod.features = [ model.Feature(id=x["id"], name=x["name"]) for x in v2_embed["features"] ] localizations_map = collections.defaultdict(lambda: model.Localization()) for loc in v2_embed["localizations"]: loc_embed = loc["_embedded"] localization = localizations_map[loc_embed["language"]["code"]] localization.code = loc_embed["language"]["code"] localization.name = loc_embed["language"]["name"] if loc_embed["localizationScope"]["type"] == "text": localization.text = True elif loc_embed["localizationScope"]["type"] == "audio": localization.audio = True prod.localizations = list(localizations_map.values()) prod.tags = [ model.Tag(id=x["id"], level=x["level"], name=x["name"], slug=x["slug"]) for x in v2_embed["tags"] ] prod.comp_systems = [ normalize_system(support_entry["operatingSystem"]["name"]) for support_entry in v2_embed["supportedOperatingSystems"] ] prod.comp_systems.sort(reverse=True) prod.is_using_dosbox = v2_cont["isUsingDosBox"] prod.developers = [x["name"] for x in v2_embed["developers"]] prod.publisher = v2_embed["publisher"]["name"] prod.copyright = v2_cont["copyrights"] or None prod.global_date = parse_datetime( v2_embed["product"].get("globalReleaseDate")) if "gogRating" in v2_embed and v2_embed["gogRating"] is not None: prod.age_rating = v2_embed["gogRating"].get("ageRating") if "galaxyBackgroundImage" in v2_links: prod.image_galaxy_background = extract_imageid( v2_links["galaxyBackgroundImage"]["href"]) prod.image_boxart = extract_imageid(v2_links["boxArtImage"]["href"]) prod.image_icon_square = extract_imageid(v2_links["iconSquare"]["href"]) prod.editions = [ model.Edition(id=ed["id"], name=ed["name"], has_product_card=ed["hasProductCard"]) for ed in v2_embed["editions"] ] prod.includes_games = [ extract_prodid(link["href"]) for link in v2_links.get("includesGames", []) ] prod.is_included_in = [ extract_prodid(link["href"]) for link in v2_links.get("isIncludedInGames", []) ] prod.required_by = [ extract_prodid(link["href"]) for link in v2_links.get("isRequiredByGames", []) ] prod.requires = [ extract_prodid(link["href"]) for link in v2_links.get("requiresGames", []) ] if v2_embed["series"]: prod.series = model.Series(id=v2_embed["series"]["id"], name=v2_embed["series"]["name"]) prod.description = v2_cont["description"]
def convert_product(l_prod): n_prod = new_model.Product() n_prod.id = l_prod.id def is_add_product_record(changerec): return changerec.action == "add" and changerec.type_prim == "product" added_record = list(filter(is_add_product_record, l_prod.changes)) if added_record: n_prod.added_on = added_record[0].timestamp.replace(tzinfo=datetime.timezone.utc) #n_prod.last_updated n_prod.title = l_prod.title n_prod.type = l_prod.product_type n_prod.slug = l_prod.slug n_prod.access = l_prod.access #n_prod.features = l_prod. #n_prod.localizations = l_prod. #n_prod.tags = l_prod. n_prod.cs_systems = normalize_system_list(l_prod.cs_systems) n_prod.comp_systems = normalize_system_list(l_prod.comp_systems) #n_prod.is_using_dosbox = l_prod. #n_prod.developers = l_prod. #n_prod.publisher = l_prod. #n_prod.copyright = l_prod. n_prod.global_date = date_to_datetime(l_prod.release_date) n_prod.store_date = date_to_datetime(l_prod.store_date) n_prod.is_in_development = l_prod.development_active n_prod.is_pre_order = l_prod.is_pre_order #n_prod.sale_rank = l_prod. n_prod.image_logo = l_prod.image_logo n_prod.image_background = l_prod.image_background n_prod.image_icon = l_prod.image_icon #n_prod.image_galaxy_background = l_prod. #n_prod.image_boxart = l_prod. #n_prod.image_icon_square = l_prod. #n_prod.link_forum = l_prod. #n_prod.link_store = l_prod. #n_prod.link_support = l_prod. n_prod.screenshots = [l_scr.image_id for l_scr in l_prod.screenshots] n_prod.videos = [ new_model.Video( provider="youtube", video_url=f"https://www.youtube.com/embed/{ l_video.video_id }?wmode=opaque&rel=0", thumbnail_url=f"https://img.youtube.com/vi/{ l_video.video_id }/hqdefault.jpg" ) for l_video in l_prod.videos ] #n_prod.editions = l_prod. #n_prod.includes_games = l_prod. #n_prod.is_included_in = l_prod. #n_prod.required_by = l_prod. if l_prod.base_prod_id: n_prod.requires = [l_prod.base_prod_id] #n_prod.series = l_prod. n_prod.dlcs = [l_dlc.id for l_dlc in l_prod.dlcs] n_prod.description = l_prod.description_full n_prod.changelog = l_prod.changelog or None for l_dl in l_prod.downloads: if l_dl.deleted: continue if l_dl.type == "bonus_content": n_prod.dl_bonus.append(convert_bonusdl(l_dl, l_prod.id)) elif l_dl.type == "installers": n_prod.dl_installer.append(convert_softwaredl(l_dl, l_prod.id)) elif l_dl.type == "language_packs": n_prod.dl_langpack.append(convert_softwaredl(l_dl, l_prod.id)) elif l_dl.type == "patches": n_prod.dl_patch.append(convert_softwaredl(l_dl, l_prod.id)) for l_build in l_prod.builds: if l_build.generation == 1: manifest_url = "https://cdn.gog.com/content-system/v1/manifests/{}/windows/{}/repository.json".format( l_prod.id, l_build.legacy_build_id) elif l_build.generation == 2: manifest_url = "https://cdn.gog.com/content-system/v2/meta/{}/{}/{}".format( l_build.meta_id[0:2], l_build.meta_id[2:4], l_build.meta_id) n_prod.builds.append(new_model.Build( id = l_build.build_id, product_id = l_build.prod_id, os = normalize_system(l_build.os), branch = None, version = l_build.version, tags = l_build.tags, public = l_build.public, date_published = l_build.date_published.astimezone(datetime.timezone.utc), generation = l_build.generation, legacy_build_id = l_build.legacy_build_id, meta_id = l_build.meta_id, link = manifest_url, listed = True )) return n_prod
def convert_changelog(l_prod): l_changes = l_prod.changes if not l_changes: return [] # Group changes by date l_changes_groups = [] last_date = l_changes[0].timestamp l_cur_group = [] for l_changerec in l_changes: if l_changerec.timestamp == last_date: l_cur_group.append(l_changerec) else: l_changes_groups.append(l_cur_group) l_cur_group = [l_changerec] last_date = l_changerec.timestamp l_changes_groups.append(l_cur_group) dummy_prod = DummyProduct() dummy_prod.id = l_prod.id dummy_prod.title = l_prod.title dummy_prod.comp_systems = normalize_system_list(l_prod.comp_systems) dummy_prod.access = l_prod.access dummy_prod.downloads = [] for l_dl in l_prod.downloads: if l_dl.type == "bonus_content": dummy_prod.downloads.append( MetaDownload( type = l_dl.type, deleted = l_dl.deleted, download = convert_bonusdl(l_dl, l_prod.id) ) ) else: dummy_prod.downloads.append( MetaDownload( type = l_dl.type, deleted = l_dl.deleted, download = convert_softwaredl(l_dl, l_prod.id) ) ) # Moving in time backwards, so the previous state is actually in the future future_productstate = dummy_prod changelog = [] for l_ch_group in l_changes_groups: cur_productstate = copy.deepcopy(future_productstate) cur_date = l_ch_group[0].timestamp.replace(tzinfo=datetime.timezone.utc) prod_changelogger = changelogger.Changelogger(future_productstate, cur_productstate, cur_date) for l_change in l_ch_group: action_type = l_change.action_type if action_type == "add product": prod_changelogger.prod_added() # I decided to not convert this entry because it's 99% false positives elif action_type == "change product.access": #cur_productstate.access = int(l_change.old) pass elif action_type == "change product.cs": pass elif action_type == "change product.os": cur_productstate.comp_systems = [normalize_system(s) for s in l_change.old.split(",")] elif action_type == "change product.title": cur_productstate.title = l_change.old elif action_type == "change product.forum_slug": pass elif action_type == "add download": cur_productstate.download_by_id(l_change.resource).deleted = True elif action_type == "del download": cur_productstate.download_by_id(l_change.resource).deleted = False elif action_type == "change download.version": cur_productstate.download_by_id(l_change.resource).download.version = l_change.old elif action_type == "change download.name": cur_productstate.download_by_id(l_change.resource).download.name = l_change.old elif action_type == "change download.total_size": cur_productstate.download_by_id(l_change.resource).download.total_size = int(l_change.old) else: raise RuntimeError("Unknown change type " + l_change.action_type) prod_changelogger.property("title") prod_changelogger.property("comp_systems") prod_changelogger.property("access") prod_changelogger.downloads("bonus") prod_changelogger.downloads("installer") prod_changelogger.downloads("langpack") prod_changelogger.downloads("patch") changelog += prod_changelogger.entries future_productstate = cur_productstate changelog.reverse() # reverse so entries are sorted oldest to most recent return changelog