def main() -> None: """ Main Method """ args: argparse.Namespace = parse_args() mtgjson4.USE_CACHE.set(not args.skip_cache) mtgjson4.PRETTY_OUTPUT.set(4 if args.pretty_output else None) if not mtgjson4.CONFIG_PATH.is_file(): LOGGER.warning( f"No properties file found at {mtgjson4.CONFIG_PATH}. Will download without authentication" ) # Determine set(s) to build args_s = args.s if args.s else [] set_list: List[str] = get_all_sets() if args.a else args_s if args.skip_sets: set_list = sorted(list(set(set_list) - set(args.skip_sets))) LOGGER.info(f"Skipping set(s) by request of user: {args.skip_sets}") LOGGER.info(f"Sets to compile: {set_list}") # If we had to kill mid-build, we can skip the completed set(s) if args.x: sets_compiled_already: List[str] = get_compiled_sets() set_list = [ s for s in set_list if s.lower() not in sets_compiled_already ] LOGGER.info( f"Sets to skip compilation for: {sets_compiled_already}\n\nSets to compile, after cached sets removed: {set_list}" ) for set_code in set_list: sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code) compiled = compile_mtg.build_mtgjson_set(sf_set, set_code, args.skip_keys) # If we have at least 1 card, dump to file SET.json # but first add them to ReferralMap.json if compiled["cards"] or compiled["tokens"]: if not args.skip_keys: for card in compiled["cards"]: add_card_to_referral_map(card) mtgjson4.outputter.write_to_file(set_code.upper(), compiled, set_file=True) # Compile the additional outputs if args.c: LOGGER.info("Compiling additional outputs") mtgjson4.outputter.create_and_write_compiled_outputs() # Compress the output folder if args.z: LOGGER.info("Start compressing for production") compressor.compress_output_folder() LOGGER.info("Finished compressing for production")
def main() -> None: """ Main Method """ parser = argparse.ArgumentParser(description="") parser.add_argument("-a", action="store_true") parser.add_argument("-s", metavar="SET", nargs="*", type=str) parser.add_argument("-c", action="store_true") parser.add_argument("-x", action="store_true") parser.add_argument("--skip-tcgplayer", action="store_true") parser.add_argument("--skip-sets", metavar="SET", nargs="*", type=str) # Ensure there are args if len(sys.argv) < 2: parser.print_usage() sys.exit(1) else: args = parser.parse_args() if not mtgjson4.CONFIG_PATH.is_file(): LOGGER.warning( "No properties file found at {}. Will download without authentication" .format(mtgjson4.CONFIG_PATH)) # Determine set(s) to build args_s = args.s if args.s else [] set_list: List[str] = get_all_sets() if args.a else args_s if args.skip_sets: set_list = list(set(set_list) - set(args.skip_sets)) LOGGER.info("Skipping set(s) by request of user: {}".format( args.skip_sets)) LOGGER.info("Sets to compile: {}".format(set_list)) # If we had to kill mid-build, we can skip the completed set(s) if args.x: sets_compiled_already: List[str] = get_compiled_sets() set_list = [s for s in set_list if s not in sets_compiled_already] LOGGER.info( "Sets to skip compilation for: {}\n\nSets to compile, after cached sets removed: {}" .format(sets_compiled_already, set_list)) for set_code in set_list: sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code) compiled = compile_mtg.build_output_file(sf_set, set_code, args.skip_tcgplayer) # If we have at least 1 card, dump to file SET.json if compiled["cards"] or compiled["tokens"]: mtgjson4.outputter.write_to_file(set_code.upper(), compiled, do_cleanup=True) # Compile the additional outputs if args.c: LOGGER.info("Compiling Additional Outputs") mtgjson4.outputter.create_and_write_compiled_outputs()
def main() -> None: """ Main Method """ args: argparse.Namespace = parse_args() mtgjson4.USE_CACHE.set(not args.skip_cache) if not mtgjson4.CONFIG_PATH.is_file(): LOGGER.warning( "No properties file found at {}. Will download without authentication".format( mtgjson4.CONFIG_PATH ) ) # Determine set(s) to build args_s = args.s if args.s else [] set_list: List[str] = get_all_sets() if args.a else args_s if args.skip_sets: set_list = list(set(set_list) - set(args.skip_sets)) LOGGER.info("Skipping set(s) by request of user: {}".format(args.skip_sets)) LOGGER.info("Sets to compile: {}".format(set_list)) # If we had to kill mid-build, we can skip the completed set(s) if args.x: sets_compiled_already: List[str] = get_compiled_sets() set_list = [s for s in set_list if s not in sets_compiled_already] LOGGER.info( "Sets to skip compilation for: {}\n\nSets to compile, after cached sets removed: {}".format( sets_compiled_already, set_list ) ) for set_code in set_list: sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code) compiled = compile_mtg.build_output_file(sf_set, set_code, args.skip_keys) # If we have at least 1 card, dump to file SET.json # but first add them to ReferralMap.json if compiled["cards"] or compiled["tokens"]: if not args.skip_keys: for card in compiled["cards"]: add_card_to_referral_map(card) mtgjson4.outputter.write_to_file(set_code.upper(), compiled, set_file=True) # Compile the additional outputs if args.c: LOGGER.info("Compiling additional outputs") mtgjson4.outputter.create_and_write_compiled_outputs() # Compress the output folder if args.z: LOGGER.info("Start compressing for production") compressor.compress_output_folder() LOGGER.info("Finished compressing for production")
def main() -> None: """ Main Method """ parser = argparse.ArgumentParser(description="") parser.add_argument("-s", metavar="SET", nargs="*", type=str) parser.add_argument("-a", "--all-sets", action="store_true") parser.add_argument("-c", "--compiled-outputs", action="store_true") parser.add_argument("--skip-rebuild", action="store_true") parser.add_argument("--skip-cached", action="store_true") # Ensure there are args if len(sys.argv) < 2: parser.print_usage() sys.exit(1) else: args = parser.parse_args() if not pathlib.Path(mtgjson4.CONFIG_PATH).is_file(): LOGGER.warning( "No properties file found at {}. Will download without authentication" .format(mtgjson4.CONFIG_PATH)) if not args.skip_rebuild: # Determine sets to build, whether they're passed in as args or all sets in our configs args_s = args.s if args.s else [] set_list: List[str] = get_all_sets() if args.all_sets else args_s LOGGER.info("Sets to compile: {}".format(set_list)) # If we had to kill mid-rebuild, we can skip the sets that already were done if args.skip_cached: sets_compiled_already: List[str] = get_compiled_sets() set_list = [s for s in set_list if s not in sets_compiled_already] LOGGER.info("Sets to skip compilation for: {}".format( sets_compiled_already)) LOGGER.info( "Sets to compile, after cached sets removed: {}".format( set_list)) for set_code in set_list: sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code) compiled: Dict[str, Any] = compile_mtg.build_output_file( sf_set, set_code) # If we have at least 1 card, dump to file SET.json if compiled["cards"] or compiled["tokens"]: mtgjson4.outputter.write_to_file(set_code.upper(), compiled, do_cleanup=True) if args.compiled_outputs: LOGGER.info("Compiling Additional Outputs") mtgjson4.outputter.create_and_write_compiled_outputs()
def build_output_file( sf_cards: List[Dict[str, Any]], set_code: str, skip_tcgplayer: bool ) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param skip_tcgplayer: Skip building TCGPlayer stuff :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ output_file: Dict[str, Any] = {} # Get the set config from Scryfall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error("Set Config for {} was not found, skipping...".format(set_code)) return {"cards": [], "tokens": []} output_file["name"] = set_config.get("name") output_file["code"] = str(set_config.get("code", "")).upper() output_file["mtgoCode"] = str(set_config.get("mtgo_code", "")).upper() output_file["releaseDate"] = set_config.get("released_at") output_file["type"] = set_config.get("set_type") # Add booster info based on boosters resource (manually maintained for the time being) with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open( "r", encoding="utf-8" ) as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"].upper() in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] # Add V3 code for some backwards compatibility with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open( "r", encoding="utf-8" ) as f: json_dict = json.load(f) if output_file["code"].upper() in json_dict.keys(): output_file["codeV3"] = json_dict[output_file["code"]] if set_config.get("block"): output_file["block"] = set_config.get("block") if set_config.get("digital"): output_file["isOnlineOnly"] = True if set_config.get("foil_only"): output_file["isFoilOnly"] = True # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder = convert_to_mtgjson(sf_cards) card_holder = add_start_flag_and_count_modified( set_code, set_config["search_uri"], card_holder ) # Address duplicates in un-sets card_holder = uniquify_duplicates_in_set(card_holder) # Move bogus tokens out card_holder, added_tokens = transpose_tokens(card_holder) # Add TCGPlayer information if "tcgplayer_id" in set_config: output_file["tcgplayerGroupId"] = set_config.get("tcgplayer_id") if not skip_tcgplayer: card_holder = add_tcgplayer_fields( output_file["tcgplayerGroupId"], card_holder ) # Set sizes; BASE SET SIZE WILL BE UPDATED BELOW output_file["totalSetSize"] = len(sf_cards) with mtgjson4.RESOURCE_PATH.joinpath("base_set_sizes.json").open( "r", encoding="utf-8" ) as f: output_file["baseSetSize"] = json.load(f).get(set_code.upper(), 0) output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) # Add UUID to each entry add_uuid_to_cards(output_file["cards"], output_file["tokens"], output_file) # Add Variations to each entry, as well as mark alternatives add_variations_and_alternative_fields(output_file["cards"], output_file) if set_code[:2] == "DD": mark_duel_decks(output_file["cards"]) return output_file
def build_output_file(sf_cards: List[Dict[str, Any]], set_code: str, skip_keys: bool) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param skip_keys: Skip building TCGPlayer & MKM components :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ if not skip_keys and os.environ["MKM_APP_TOKEN"] and os.environ[ "MKM_APP_SECRET"]: MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"])) output_file: Dict[str, Any] = {} # Get the set config from Scryfall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error( "Set Config for {} was not found, skipping...".format(set_code)) return {"cards": [], "tokens": []} output_file["name"] = set_config["name"] output_file["code"] = set_config["code"].upper() output_file["releaseDate"] = set_config["released_at"] output_file["type"] = set_config["set_type"] output_file["keyruneCode"] = (pathlib.Path( set_config["icon_svg_uri"]).name.split(".")[0].upper()) # Try adding MKM Set Name # Then store the card data for future pulling if MKM_API.get(None): mkm_resp = MKM_API.get().market_place.expansions(game=1) if mkm_resp.status_code != 200: LOGGER.error( "Unable to download MKM correctly: {}".format(mkm_resp)) else: for set_content in mkm_resp.json()["expansion"]: if (set_content["enName"].lower() == output_file["name"].lower() or set_content["abbreviation"].lower() == output_file["code"].lower()): output_file["mcmId"] = set_content["idExpansion"] output_file["mcmName"] = set_content["enName"] break initialize_mkm_set_cards(output_file.get("mcmId", None)) # Add translations to the files try: output_file["translations"] = wizards.get_translations( output_file["code"]) except KeyError: LOGGER.warning( "Unable to find set translations for {}".format(set_code)) # Add optionals if they exist if "mtgo_code" in set_config.keys(): output_file["mtgoCode"] = set_config["mtgo_code"].upper() if "parent_set_code" in set_config.keys(): output_file["parentCode"] = set_config["parent_set_code"].upper() if "block" in set_config.keys(): output_file["block"] = set_config["block"] if "digital" in set_config.keys(): output_file["isOnlineOnly"] = set_config["digital"] if "foil_only" in set_config.keys(): output_file["isFoilOnly"] = set_config["foil_only"] if set_code.upper() in mtgjson4.NON_ENGLISH_SETS: output_file["isForeignOnly"] = True # Add booster info based on boosters resource (manually maintained for the time being) with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open( "r", encoding="utf-8") as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"] in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] # Add V3 code for some backwards compatibility with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open( "r", encoding="utf-8") as f: json_dict = json.load(f) if output_file["code"] in json_dict.keys(): output_file["codeV3"] = json_dict[output_file["code"]] # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, "pricesDate": mtgjson4.__PRICE_UPDATE_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards) card_holder = add_start_flag_and_count_modified(set_code, set_config["search_uri"], card_holder) # Address duplicates in un-sets card_holder = uniquify_duplicates_in_set(card_holder) # Move bogus tokens out card_holder, added_tokens = transpose_tokens(card_holder) if not skip_keys: # Add MTGStocks data in card_holder = add_stocks_data(card_holder) # Add TCGPlayer information if "tcgplayer_id" in set_config.keys(): output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"] if not skip_keys: add_purchase_fields(output_file["tcgplayerGroupId"], card_holder) # Set Sizes output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper()) output_file["totalSetSize"] = len(sf_cards) output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) # Handle tokens LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) # Cleanups and UUIDs mtgjson_card.DUEL_DECK_LAND_MARKED.set(False) mtgjson_card.DUEL_DECK_SIDE_COMP.set("a") for card in sorted(output_file["cards"]): card.final_card_cleanup() for token in output_file["tokens"]: token.final_card_cleanup(is_card=False) # Add Variations to each entry, as well as mark alternatives add_variations_and_alternative_fields(output_file["cards"], output_file) return output_file
def build_output_file(sf_cards: List[Dict[str, Any]], set_code: str) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ output_file: Dict[str, Any] = {} # Get the set config from ScryFall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error( "Set Config for {} was not found, skipping...".format(set_code)) return {"cards": []} output_file["name"] = set_config.get("name") output_file["code"] = set_config.get("code") output_file["mtgoCode"] = set_config.get("mtgo_code") output_file["releaseDate"] = set_config.get("released_at") output_file["type"] = set_config.get("set_type") # Add booster info based on boosters resource (manually maintained for the time being) with pathlib.Path(mtgjson4.RESOURCE_PATH, "boosters.json").open("r", encoding="utf-8") as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"].upper() in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] if set_config.get("block"): output_file["block"] = set_config.get("block") if set_config.get("digital"): output_file["isOnlineOnly"] = True if set_config.get("foil_only"): output_file["isFoilOnly"] = True # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder = convert_to_mtgjson(sf_cards) card_holder, non_booster_cards = add_start_flag_and_count_modified( set_code, set_config["search_uri"], card_holder) output_file["totalSetSize"] = len(sf_cards) output_file[ "baseSetSize"] = output_file["totalSetSize"] - non_booster_cards output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) return output_file
def build_output_file( sf_cards: List[Dict[str, Any]], set_code: str, skip_keys: bool ) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param skip_keys: Skip building TCGPlayer & MKM components :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ if not skip_keys: MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"])) output_file: Dict[str, Any] = {} # Get the set config from Scryfall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error("Set Config for {} was not found, skipping...".format(set_code)) return {"cards": [], "tokens": []} output_file["name"] = set_config["name"] output_file["code"] = set_config["code"].upper() output_file["releaseDate"] = set_config["released_at"] output_file["type"] = set_config["set_type"] output_file["keyruneCode"] = ( pathlib.Path(set_config["icon_svg_uri"]).name.split(".")[0].upper() ) # Try adding MKM Set Name # Then store the card data for future pulling if MKM_API.get(None): mkm_resp = MKM_API.get().market_place.expansions(game=1) if mkm_resp.status_code != 200: LOGGER.error("Unable to download MKM correctly: {}".format(mkm_resp)) else: for set_content in mkm_resp.json()["expansion"]: if ( set_content["enName"].lower() == output_file["name"].lower() or set_content["abbreviation"].lower() == output_file["code"].lower() ): output_file["mcmId"] = set_content["idExpansion"] output_file["mcmName"] = set_content["enName"] break initialize_mkm_set_cards(output_file.get("mcmId", None)) # Add translations to the files try: output_file["translations"] = wizards.get_translations(output_file["code"]) except KeyError: LOGGER.warning("Unable to find set translations for {}".format(set_code)) # Add optionals if they exist if "mtgo_code" in set_config.keys(): output_file["mtgoCode"] = set_config["mtgo_code"].upper() if "parent_set_code" in set_config.keys(): output_file["parentCode"] = set_config["parent_set_code"].upper() if "block" in set_config.keys(): output_file["block"] = set_config["block"] if "digital" in set_config.keys(): output_file["isOnlineOnly"] = set_config["digital"] if "foil_only" in set_config.keys(): output_file["isFoilOnly"] = set_config["foil_only"] if set_code.upper() in mtgjson4.NON_ENGLISH_SETS: output_file["isForeignOnly"] = True # Add booster info based on boosters resource (manually maintained for the time being) with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open( "r", encoding="utf-8" ) as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"] in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] # Add V3 code for some backwards compatibility with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open( "r", encoding="utf-8" ) as f: json_dict = json.load(f) if output_file["code"] in json_dict.keys(): output_file["codeV3"] = json_dict[output_file["code"]] # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, "pricesDate": mtgjson4.__PRICE_UPDATE_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards) card_holder = add_start_flag_and_count_modified( set_code, set_config["search_uri"], card_holder ) # Address duplicates in un-sets card_holder = uniquify_duplicates_in_set(card_holder) # Move bogus tokens out card_holder, added_tokens = transpose_tokens(card_holder) if not skip_keys: # Add MTGStocks data in card_holder = add_stocks_data(card_holder) # Add TCGPlayer information if "tcgplayer_id" in set_config.keys(): output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"] if not skip_keys: add_purchase_fields(output_file["tcgplayerGroupId"], card_holder) # Set Sizes output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper()) output_file["totalSetSize"] = len(sf_cards) output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) # Handle tokens LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) # Cleanups and UUIDs mtgjson_card.DUEL_DECK_LAND_MARKED.set(False) mtgjson_card.DUEL_DECK_SIDE_COMP.set("a") for card in sorted(output_file["cards"]): card.final_card_cleanup() for token in output_file["tokens"]: token.final_card_cleanup(is_card=False) # Add Variations to each entry, as well as mark alternatives add_variations_and_alternative_fields(output_file["cards"], output_file) return output_file