def add_start_flag_and_count_modified( set_code: str, search_url: str, mtgjson_cards: List[Dict[str, Any]] ) -> List[Dict[str, Any]]: """ Since SF doesn't provide individual card notices, we can post-process add the starter flag This method will also tell us how many starter cards are in the set :param set_code: Set to address :param search_url: URL to fix up to get non-booster cards :param mtgjson_cards: Modify the argument and return it :return: List of cards, number of cards modified """ starter_card_url = search_url.replace("&unique=", "++not:booster&unique=") starter_cards = scryfall.download(starter_card_url) if starter_cards["object"] == "error": LOGGER.info("All cards in {} are available in boosters".format(set_code)) return mtgjson_cards for sf_card in starter_cards["data"]: # Each card has a unique UUID, even if they're the same card printed twice try: card = next( item for item in mtgjson_cards if item["scryfallId"] == sf_card["id"] ) if card: card["starter"] = True except StopIteration: LOGGER.warning( "Passed on {0} with SF_ID {1}".format( sf_card["name"], sf_card["scryfallId"] ) ) return mtgjson_cards
def get_funny_sets() -> List[str]: """ This will determine all of the "joke" sets and give back a list of their set codes :return: List of joke set codes """ return [ x["code"].upper() for x in scryfall.download(scryfall.SCRYFALL_API_SETS)["data"] if str(x["set_type"]) == "funny" ]
def get_funny_sets() -> List[str]: """ This will determine all of the "joke" sets and give back a list of their set codes :return: List of joke set codes """ return [ x["code"].upper() for x in scryfall.download(scryfall.SCRYFALL_API_SETS)["data"] if str(x["set_type"]) in ["funny", "memorabilia"] ]
def transpose_tokens( cards: List[MTGJSONCard] ) -> Tuple[List[MTGJSONCard], List[Dict[str, Any]]]: """ Sometimes, tokens slip through and need to be transplanted back into their appropriate array. This method will allow us to pluck the tokens out and return them home. :param cards: Cards+Tokens to iterate :return: Cards, Tokens as two separate lists """ # Order matters with these, as if you do cards first # it will shadow the tokens lookup # Single faced tokens are easy tokens = [ scryfall.download(scryfall.SCRYFALL_API_CARD + card.get("scryfallId")) for card in cards if card.get("layout") in ["token", "emblem"] ] # Do not duplicate double faced tokens done_tokens: Set[str] = set() for card in cards: if ( card.get("layout") == "double_faced_token" and card.get("scryfallId") not in done_tokens ): tokens.append( scryfall.download(scryfall.SCRYFALL_API_CARD + card.get("scryfallId")) ) done_tokens.add(card.get("scryfallId")) # Remaining cards, without any kind of token cards = [ card for card in cards if card.get("layout") not in ["token", "double_faced_token"] ] return cards, tokens
def transpose_tokens( cards: List[Dict[str, Any]] ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: """ Sometimes, tokens slip through and need to be transplanted back into their appropriate array. This method will allow us to pluck the tokens out and return them home. :param cards: Cards+Tokens to iterate :return: Cards, Tokens as two separate lists """ # Order matters with these, as if you do cards first # it will shadow the tokens lookup # Single faced tokens are easy tokens = [ scryfall.download(scryfall.SCRYFALL_API_CARD + card["scryfallId"]) for card in cards if card["layout"] == "token" ] # Do not duplicate double faced tokens done_tokens: Set[str] = set() for card in cards: if ( card["layout"] == "double_faced_token" and card["scryfallId"] not in done_tokens ): tokens.append( scryfall.download(scryfall.SCRYFALL_API_CARD + card["scryfallId"]) ) done_tokens.add(card["scryfallId"]) # Remaining cards, without any kind of token cards = [ card for card in cards if card["layout"] not in ["token", "double_faced_token"] ] return cards, tokens
def get_all_sets() -> List[str]: """ Grab the set codes (~3 letters) for all sets found in the config database. :return: List of all set codes found, sorted """ downloaded = scryfall.download(scryfall.SCRYFALL_API_SETS) if downloaded["object"] == "error": LOGGER.error("Downloading Scryfall data failed: {}".format(downloaded)) return [] # Get _ALL_ Scryfall sets set_codes: List[str] = [set_obj["code"] for set_obj in downloaded["data"]] # Remove Scryfall token sets (but leave extra sets) set_codes = [ s for s in set_codes if not (s.startswith("t") and s[1:] in set_codes) ] return sorted(set_codes)
def add_start_flag_and_count_modified( set_code: str, search_url: str, mtgjson_cards: List[MTGJSONCard] ) -> List[MTGJSONCard]: """ Since SF doesn't provide individual card notices, we can post-process add the starter flag This method will also tell us how many starter cards are in the set :param set_code: Set to address :param search_url: URL to fix up to get non-booster cards :param mtgjson_cards: Modify the argument and return it :return: List of cards, number of cards modified """ starter_card_url = search_url.replace("&unique=", "++not:booster&unique=") starter_cards = scryfall.download(starter_card_url) if starter_cards["object"] == "error": LOGGER.info("All cards in {} are available in boosters".format(set_code)) return mtgjson_cards for sf_card in starter_cards["data"]: # Each card has a unique UUID, even if they're the same card printed twice try: card = next( item for item in mtgjson_cards if item.get("scryfallId") == sf_card["id"] ) if card: card.set("isStarter", True) except StopIteration: LOGGER.warning( "Passed on {0} with SF_ID {1}".format( sf_card["name"], sf_card["scryfallId"] ) ) return mtgjson_cards
def build_output_file( sf_cards: List[Dict[str, Any]], set_code: str, skip_tcgplayer: bool ) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param skip_tcgplayer: Skip building TCGPlayer stuff :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ output_file: Dict[str, Any] = {} # Get the set config from Scryfall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error("Set Config for {} was not found, skipping...".format(set_code)) return {"cards": [], "tokens": []} output_file["name"] = set_config.get("name") output_file["code"] = str(set_config.get("code", "")).upper() output_file["mtgoCode"] = str(set_config.get("mtgo_code", "")).upper() output_file["releaseDate"] = set_config.get("released_at") output_file["type"] = set_config.get("set_type") # Add booster info based on boosters resource (manually maintained for the time being) with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open( "r", encoding="utf-8" ) as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"].upper() in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] # Add V3 code for some backwards compatibility with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open( "r", encoding="utf-8" ) as f: json_dict = json.load(f) if output_file["code"].upper() in json_dict.keys(): output_file["codeV3"] = json_dict[output_file["code"]] if set_config.get("block"): output_file["block"] = set_config.get("block") if set_config.get("digital"): output_file["isOnlineOnly"] = True if set_config.get("foil_only"): output_file["isFoilOnly"] = True # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder = convert_to_mtgjson(sf_cards) card_holder = add_start_flag_and_count_modified( set_code, set_config["search_uri"], card_holder ) # Address duplicates in un-sets card_holder = uniquify_duplicates_in_set(card_holder) # Move bogus tokens out card_holder, added_tokens = transpose_tokens(card_holder) # Add TCGPlayer information if "tcgplayer_id" in set_config: output_file["tcgplayerGroupId"] = set_config.get("tcgplayer_id") if not skip_tcgplayer: card_holder = add_tcgplayer_fields( output_file["tcgplayerGroupId"], card_holder ) # Set sizes; BASE SET SIZE WILL BE UPDATED BELOW output_file["totalSetSize"] = len(sf_cards) with mtgjson4.RESOURCE_PATH.joinpath("base_set_sizes.json").open( "r", encoding="utf-8" ) as f: output_file["baseSetSize"] = json.load(f).get(set_code.upper(), 0) output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) # Add UUID to each entry add_uuid_to_cards(output_file["cards"], output_file["tokens"], output_file) # Add Variations to each entry, as well as mark alternatives add_variations_and_alternative_fields(output_file["cards"], output_file) if set_code[:2] == "DD": mark_duel_decks(output_file["cards"]) return output_file
def build_output_file(sf_cards: List[Dict[str, Any]], set_code: str, skip_keys: bool) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param skip_keys: Skip building TCGPlayer & MKM components :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ if not skip_keys and os.environ["MKM_APP_TOKEN"] and os.environ[ "MKM_APP_SECRET"]: MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"])) output_file: Dict[str, Any] = {} # Get the set config from Scryfall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error( "Set Config for {} was not found, skipping...".format(set_code)) return {"cards": [], "tokens": []} output_file["name"] = set_config["name"] output_file["code"] = set_config["code"].upper() output_file["releaseDate"] = set_config["released_at"] output_file["type"] = set_config["set_type"] output_file["keyruneCode"] = (pathlib.Path( set_config["icon_svg_uri"]).name.split(".")[0].upper()) # Try adding MKM Set Name # Then store the card data for future pulling if MKM_API.get(None): mkm_resp = MKM_API.get().market_place.expansions(game=1) if mkm_resp.status_code != 200: LOGGER.error( "Unable to download MKM correctly: {}".format(mkm_resp)) else: for set_content in mkm_resp.json()["expansion"]: if (set_content["enName"].lower() == output_file["name"].lower() or set_content["abbreviation"].lower() == output_file["code"].lower()): output_file["mcmId"] = set_content["idExpansion"] output_file["mcmName"] = set_content["enName"] break initialize_mkm_set_cards(output_file.get("mcmId", None)) # Add translations to the files try: output_file["translations"] = wizards.get_translations( output_file["code"]) except KeyError: LOGGER.warning( "Unable to find set translations for {}".format(set_code)) # Add optionals if they exist if "mtgo_code" in set_config.keys(): output_file["mtgoCode"] = set_config["mtgo_code"].upper() if "parent_set_code" in set_config.keys(): output_file["parentCode"] = set_config["parent_set_code"].upper() if "block" in set_config.keys(): output_file["block"] = set_config["block"] if "digital" in set_config.keys(): output_file["isOnlineOnly"] = set_config["digital"] if "foil_only" in set_config.keys(): output_file["isFoilOnly"] = set_config["foil_only"] if set_code.upper() in mtgjson4.NON_ENGLISH_SETS: output_file["isForeignOnly"] = True # Add booster info based on boosters resource (manually maintained for the time being) with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open( "r", encoding="utf-8") as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"] in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] # Add V3 code for some backwards compatibility with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open( "r", encoding="utf-8") as f: json_dict = json.load(f) if output_file["code"] in json_dict.keys(): output_file["codeV3"] = json_dict[output_file["code"]] # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, "pricesDate": mtgjson4.__PRICE_UPDATE_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards) card_holder = add_start_flag_and_count_modified(set_code, set_config["search_uri"], card_holder) # Address duplicates in un-sets card_holder = uniquify_duplicates_in_set(card_holder) # Move bogus tokens out card_holder, added_tokens = transpose_tokens(card_holder) if not skip_keys: # Add MTGStocks data in card_holder = add_stocks_data(card_holder) # Add TCGPlayer information if "tcgplayer_id" in set_config.keys(): output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"] if not skip_keys: add_purchase_fields(output_file["tcgplayerGroupId"], card_holder) # Set Sizes output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper()) output_file["totalSetSize"] = len(sf_cards) output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) # Handle tokens LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) # Cleanups and UUIDs mtgjson_card.DUEL_DECK_LAND_MARKED.set(False) mtgjson_card.DUEL_DECK_SIDE_COMP.set("a") for card in sorted(output_file["cards"]): card.final_card_cleanup() for token in output_file["tokens"]: token.final_card_cleanup(is_card=False) # Add Variations to each entry, as well as mark alternatives add_variations_and_alternative_fields(output_file["cards"], output_file) return output_file
def build_output_file(sf_cards: List[Dict[str, Any]], set_code: str) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ output_file: Dict[str, Any] = {} # Get the set config from ScryFall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error( "Set Config for {} was not found, skipping...".format(set_code)) return {"cards": []} output_file["name"] = set_config.get("name") output_file["code"] = set_config.get("code") output_file["mtgoCode"] = set_config.get("mtgo_code") output_file["releaseDate"] = set_config.get("released_at") output_file["type"] = set_config.get("set_type") # Add booster info based on boosters resource (manually maintained for the time being) with pathlib.Path(mtgjson4.RESOURCE_PATH, "boosters.json").open("r", encoding="utf-8") as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"].upper() in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] if set_config.get("block"): output_file["block"] = set_config.get("block") if set_config.get("digital"): output_file["isOnlineOnly"] = True if set_config.get("foil_only"): output_file["isFoilOnly"] = True # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder = convert_to_mtgjson(sf_cards) card_holder, non_booster_cards = add_start_flag_and_count_modified( set_code, set_config["search_uri"], card_holder) output_file["totalSetSize"] = len(sf_cards) output_file[ "baseSetSize"] = output_file["totalSetSize"] - non_booster_cards output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) return output_file
def build_output_file( sf_cards: List[Dict[str, Any]], set_code: str, skip_keys: bool ) -> Dict[str, Any]: """ Compile the entire XYZ.json file and pass it off to be written out :param skip_keys: Skip building TCGPlayer & MKM components :param sf_cards: Scryfall cards :param set_code: Set code :return: Completed JSON file """ if not skip_keys: MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"])) output_file: Dict[str, Any] = {} # Get the set config from Scryfall set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code) if set_config["object"] == "error": LOGGER.error("Set Config for {} was not found, skipping...".format(set_code)) return {"cards": [], "tokens": []} output_file["name"] = set_config["name"] output_file["code"] = set_config["code"].upper() output_file["releaseDate"] = set_config["released_at"] output_file["type"] = set_config["set_type"] output_file["keyruneCode"] = ( pathlib.Path(set_config["icon_svg_uri"]).name.split(".")[0].upper() ) # Try adding MKM Set Name # Then store the card data for future pulling if MKM_API.get(None): mkm_resp = MKM_API.get().market_place.expansions(game=1) if mkm_resp.status_code != 200: LOGGER.error("Unable to download MKM correctly: {}".format(mkm_resp)) else: for set_content in mkm_resp.json()["expansion"]: if ( set_content["enName"].lower() == output_file["name"].lower() or set_content["abbreviation"].lower() == output_file["code"].lower() ): output_file["mcmId"] = set_content["idExpansion"] output_file["mcmName"] = set_content["enName"] break initialize_mkm_set_cards(output_file.get("mcmId", None)) # Add translations to the files try: output_file["translations"] = wizards.get_translations(output_file["code"]) except KeyError: LOGGER.warning("Unable to find set translations for {}".format(set_code)) # Add optionals if they exist if "mtgo_code" in set_config.keys(): output_file["mtgoCode"] = set_config["mtgo_code"].upper() if "parent_set_code" in set_config.keys(): output_file["parentCode"] = set_config["parent_set_code"].upper() if "block" in set_config.keys(): output_file["block"] = set_config["block"] if "digital" in set_config.keys(): output_file["isOnlineOnly"] = set_config["digital"] if "foil_only" in set_config.keys(): output_file["isFoilOnly"] = set_config["foil_only"] if set_code.upper() in mtgjson4.NON_ENGLISH_SETS: output_file["isForeignOnly"] = True # Add booster info based on boosters resource (manually maintained for the time being) with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open( "r", encoding="utf-8" ) as f: json_dict: Dict[str, List[Any]] = json.load(f) if output_file["code"] in json_dict.keys(): output_file["boosterV3"] = json_dict[output_file["code"].upper()] # Add V3 code for some backwards compatibility with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open( "r", encoding="utf-8" ) as f: json_dict = json.load(f) if output_file["code"] in json_dict.keys(): output_file["codeV3"] = json_dict[output_file["code"]] # Declare the version of the build in the output file output_file["meta"] = { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, "pricesDate": mtgjson4.__PRICE_UPDATE_DATE__, } LOGGER.info("Starting cards for {}".format(set_code)) card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards) card_holder = add_start_flag_and_count_modified( set_code, set_config["search_uri"], card_holder ) # Address duplicates in un-sets card_holder = uniquify_duplicates_in_set(card_holder) # Move bogus tokens out card_holder, added_tokens = transpose_tokens(card_holder) if not skip_keys: # Add MTGStocks data in card_holder = add_stocks_data(card_holder) # Add TCGPlayer information if "tcgplayer_id" in set_config.keys(): output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"] if not skip_keys: add_purchase_fields(output_file["tcgplayerGroupId"], card_holder) # Set Sizes output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper()) output_file["totalSetSize"] = len(sf_cards) output_file["cards"] = card_holder LOGGER.info("Finished cards for {}".format(set_code)) # Handle tokens LOGGER.info("Starting tokens for {}".format(set_code)) sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code) output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens) LOGGER.info("Finished tokens for {}".format(set_code)) # Cleanups and UUIDs mtgjson_card.DUEL_DECK_LAND_MARKED.set(False) mtgjson_card.DUEL_DECK_SIDE_COMP.set("a") for card in sorted(output_file["cards"]): card.final_card_cleanup() for token in output_file["tokens"]: token.final_card_cleanup(is_card=False) # Add Variations to each entry, as well as mark alternatives add_variations_and_alternative_fields(output_file["cards"], output_file) return output_file