def add_start_flag_and_count_modified(
set_code: str, search_url: str, mtgjson_cards: List[Dict[str, Any]]
) -> List[Dict[str, Any]]:
"""
Since SF doesn't provide individual card notices, we can post-process add the starter flag
This method will also tell us how many starter cards are in the set
:param set_code: Set to address
:param search_url: URL to fix up to get non-booster cards
:param mtgjson_cards: Modify the argument and return it
:return: List of cards, number of cards modified
"""
starter_card_url = search_url.replace("&unique=", "++not:booster&unique=")
starter_cards = scryfall.download(starter_card_url)
if starter_cards["object"] == "error":
LOGGER.info("All cards in {} are available in boosters".format(set_code))
return mtgjson_cards
for sf_card in starter_cards["data"]:
# Each card has a unique UUID, even if they're the same card printed twice
try:
card = next(
item for item in mtgjson_cards if item["scryfallId"] == sf_card["id"]
)
if card:
card["starter"] = True
except StopIteration:
LOGGER.warning(
"Passed on {0} with SF_ID {1}".format(
sf_card["name"], sf_card["scryfallId"]
)
)
return mtgjson_cards
def get_funny_sets() -> List[str]:
"""
This will determine all of the "joke" sets and give
back a list of their set codes
:return: List of joke set codes
"""
return [
x["code"].upper()
for x in scryfall.download(scryfall.SCRYFALL_API_SETS)["data"]
if str(x["set_type"]) == "funny"
]
def get_funny_sets() -> List[str]:
"""
This will determine all of the "joke" sets and give
back a list of their set codes
:return: List of joke set codes
"""
return [
x["code"].upper()
for x in scryfall.download(scryfall.SCRYFALL_API_SETS)["data"]
if str(x["set_type"]) in ["funny", "memorabilia"]
]
def transpose_tokens(
cards: List[MTGJSONCard]
) -> Tuple[List[MTGJSONCard], List[Dict[str, Any]]]:
"""
Sometimes, tokens slip through and need to be transplanted
back into their appropriate array. This method will allow
us to pluck the tokens out and return them home.
:param cards: Cards+Tokens to iterate
:return: Cards, Tokens as two separate lists
"""
# Order matters with these, as if you do cards first
# it will shadow the tokens lookup
# Single faced tokens are easy
tokens = [
scryfall.download(scryfall.SCRYFALL_API_CARD + card.get("scryfallId"))
for card in cards
if card.get("layout") in ["token", "emblem"]
]
# Do not duplicate double faced tokens
done_tokens: Set[str] = set()
for card in cards:
if (
card.get("layout") == "double_faced_token"
and card.get("scryfallId") not in done_tokens
):
tokens.append(
scryfall.download(scryfall.SCRYFALL_API_CARD + card.get("scryfallId"))
)
done_tokens.add(card.get("scryfallId"))
# Remaining cards, without any kind of token
cards = [
card
for card in cards
if card.get("layout") not in ["token", "double_faced_token"]
]
return cards, tokens
def transpose_tokens(
cards: List[Dict[str, Any]]
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
"""
Sometimes, tokens slip through and need to be transplanted
back into their appropriate array. This method will allow
us to pluck the tokens out and return them home.
:param cards: Cards+Tokens to iterate
:return: Cards, Tokens as two separate lists
"""
# Order matters with these, as if you do cards first
# it will shadow the tokens lookup
# Single faced tokens are easy
tokens = [
scryfall.download(scryfall.SCRYFALL_API_CARD + card["scryfallId"])
for card in cards
if card["layout"] == "token"
]
# Do not duplicate double faced tokens
done_tokens: Set[str] = set()
for card in cards:
if (
card["layout"] == "double_faced_token"
and card["scryfallId"] not in done_tokens
):
tokens.append(
scryfall.download(scryfall.SCRYFALL_API_CARD + card["scryfallId"])
)
done_tokens.add(card["scryfallId"])
# Remaining cards, without any kind of token
cards = [
card for card in cards if card["layout"] not in ["token", "double_faced_token"]
]
return cards, tokens
def get_all_sets() -> List[str]:
"""
Grab the set codes (~3 letters) for all sets found
in the config database.
:return: List of all set codes found, sorted
"""
downloaded = scryfall.download(scryfall.SCRYFALL_API_SETS)
if downloaded["object"] == "error":
LOGGER.error("Downloading Scryfall data failed: {}".format(downloaded))
return []
# Get _ALL_ Scryfall sets
set_codes: List[str] = [set_obj["code"] for set_obj in downloaded["data"]]
# Remove Scryfall token sets (but leave extra sets)
set_codes = [
s for s in set_codes if not (s.startswith("t") and s[1:] in set_codes)
]
return sorted(set_codes)
def add_start_flag_and_count_modified(
set_code: str, search_url: str, mtgjson_cards: List[MTGJSONCard]
) -> List[MTGJSONCard]:
"""
Since SF doesn't provide individual card notices, we can post-process add the starter flag
This method will also tell us how many starter cards are in the set
:param set_code: Set to address
:param search_url: URL to fix up to get non-booster cards
:param mtgjson_cards: Modify the argument and return it
:return: List of cards, number of cards modified
"""
starter_card_url = search_url.replace("&unique=", "++not:booster&unique=")
starter_cards = scryfall.download(starter_card_url)
if starter_cards["object"] == "error":
LOGGER.info("All cards in {} are available in boosters".format(set_code))
return mtgjson_cards
for sf_card in starter_cards["data"]:
# Each card has a unique UUID, even if they're the same card printed twice
try:
card = next(
item
for item in mtgjson_cards
if item.get("scryfallId") == sf_card["id"]
)
if card:
card.set("isStarter", True)
except StopIteration:
LOGGER.warning(
"Passed on {0} with SF_ID {1}".format(
sf_card["name"], sf_card["scryfallId"]
)
)
return mtgjson_cards
def build_output_file(
sf_cards: List[Dict[str, Any]], set_code: str, skip_tcgplayer: bool
) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param skip_tcgplayer: Skip building TCGPlayer stuff
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
output_file: Dict[str, Any] = {}
# Get the set config from Scryfall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error("Set Config for {} was not found, skipping...".format(set_code))
return {"cards": [], "tokens": []}
output_file["name"] = set_config.get("name")
output_file["code"] = str(set_config.get("code", "")).upper()
output_file["mtgoCode"] = str(set_config.get("mtgo_code", "")).upper()
output_file["releaseDate"] = set_config.get("released_at")
output_file["type"] = set_config.get("set_type")
# Add booster info based on boosters resource (manually maintained for the time being)
with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open(
"r", encoding="utf-8"
) as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"].upper() in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
# Add V3 code for some backwards compatibility
with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open(
"r", encoding="utf-8"
) as f:
json_dict = json.load(f)
if output_file["code"].upper() in json_dict.keys():
output_file["codeV3"] = json_dict[output_file["code"]]
if set_config.get("block"):
output_file["block"] = set_config.get("block")
if set_config.get("digital"):
output_file["isOnlineOnly"] = True
if set_config.get("foil_only"):
output_file["isFoilOnly"] = True
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder = convert_to_mtgjson(sf_cards)
card_holder = add_start_flag_and_count_modified(
set_code, set_config["search_uri"], card_holder
)
# Address duplicates in un-sets
card_holder = uniquify_duplicates_in_set(card_holder)
# Move bogus tokens out
card_holder, added_tokens = transpose_tokens(card_holder)
# Add TCGPlayer information
if "tcgplayer_id" in set_config:
output_file["tcgplayerGroupId"] = set_config.get("tcgplayer_id")
if not skip_tcgplayer:
card_holder = add_tcgplayer_fields(
output_file["tcgplayerGroupId"], card_holder
)
# Set sizes; BASE SET SIZE WILL BE UPDATED BELOW
output_file["totalSetSize"] = len(sf_cards)
with mtgjson4.RESOURCE_PATH.joinpath("base_set_sizes.json").open(
"r", encoding="utf-8"
) as f:
output_file["baseSetSize"] = json.load(f).get(set_code.upper(), 0)
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
# Add UUID to each entry
add_uuid_to_cards(output_file["cards"], output_file["tokens"], output_file)
# Add Variations to each entry, as well as mark alternatives
add_variations_and_alternative_fields(output_file["cards"], output_file)
if set_code[:2] == "DD":
mark_duel_decks(output_file["cards"])
return output_file
def build_output_file(sf_cards: List[Dict[str, Any]], set_code: str,
skip_keys: bool) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param skip_keys: Skip building TCGPlayer & MKM components
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
if not skip_keys and os.environ["MKM_APP_TOKEN"] and os.environ[
"MKM_APP_SECRET"]:
MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"]))
output_file: Dict[str, Any] = {}
# Get the set config from Scryfall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error(
"Set Config for {} was not found, skipping...".format(set_code))
return {"cards": [], "tokens": []}
output_file["name"] = set_config["name"]
output_file["code"] = set_config["code"].upper()
output_file["releaseDate"] = set_config["released_at"]
output_file["type"] = set_config["set_type"]
output_file["keyruneCode"] = (pathlib.Path(
set_config["icon_svg_uri"]).name.split(".")[0].upper())
# Try adding MKM Set Name
# Then store the card data for future pulling
if MKM_API.get(None):
mkm_resp = MKM_API.get().market_place.expansions(game=1)
if mkm_resp.status_code != 200:
LOGGER.error(
"Unable to download MKM correctly: {}".format(mkm_resp))
else:
for set_content in mkm_resp.json()["expansion"]:
if (set_content["enName"].lower()
== output_file["name"].lower()
or set_content["abbreviation"].lower()
== output_file["code"].lower()):
output_file["mcmId"] = set_content["idExpansion"]
output_file["mcmName"] = set_content["enName"]
break
initialize_mkm_set_cards(output_file.get("mcmId", None))
# Add translations to the files
try:
output_file["translations"] = wizards.get_translations(
output_file["code"])
except KeyError:
LOGGER.warning(
"Unable to find set translations for {}".format(set_code))
# Add optionals if they exist
if "mtgo_code" in set_config.keys():
output_file["mtgoCode"] = set_config["mtgo_code"].upper()
if "parent_set_code" in set_config.keys():
output_file["parentCode"] = set_config["parent_set_code"].upper()
if "block" in set_config.keys():
output_file["block"] = set_config["block"]
if "digital" in set_config.keys():
output_file["isOnlineOnly"] = set_config["digital"]
if "foil_only" in set_config.keys():
output_file["isFoilOnly"] = set_config["foil_only"]
if set_code.upper() in mtgjson4.NON_ENGLISH_SETS:
output_file["isForeignOnly"] = True
# Add booster info based on boosters resource (manually maintained for the time being)
with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open(
"r", encoding="utf-8") as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
# Add V3 code for some backwards compatibility
with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open(
"r", encoding="utf-8") as f:
json_dict = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["codeV3"] = json_dict[output_file["code"]]
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
"pricesDate": mtgjson4.__PRICE_UPDATE_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards)
card_holder = add_start_flag_and_count_modified(set_code,
set_config["search_uri"],
card_holder)
# Address duplicates in un-sets
card_holder = uniquify_duplicates_in_set(card_holder)
# Move bogus tokens out
card_holder, added_tokens = transpose_tokens(card_holder)
if not skip_keys:
# Add MTGStocks data in
card_holder = add_stocks_data(card_holder)
# Add TCGPlayer information
if "tcgplayer_id" in set_config.keys():
output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"]
if not skip_keys:
add_purchase_fields(output_file["tcgplayerGroupId"], card_holder)
# Set Sizes
output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper())
output_file["totalSetSize"] = len(sf_cards)
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
# Handle tokens
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
# Cleanups and UUIDs
mtgjson_card.DUEL_DECK_LAND_MARKED.set(False)
mtgjson_card.DUEL_DECK_SIDE_COMP.set("a")
for card in sorted(output_file["cards"]):
card.final_card_cleanup()
for token in output_file["tokens"]:
token.final_card_cleanup(is_card=False)
# Add Variations to each entry, as well as mark alternatives
add_variations_and_alternative_fields(output_file["cards"], output_file)
return output_file
def build_output_file(sf_cards: List[Dict[str, Any]],
set_code: str) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
output_file: Dict[str, Any] = {}
# Get the set config from ScryFall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error(
"Set Config for {} was not found, skipping...".format(set_code))
return {"cards": []}
output_file["name"] = set_config.get("name")
output_file["code"] = set_config.get("code")
output_file["mtgoCode"] = set_config.get("mtgo_code")
output_file["releaseDate"] = set_config.get("released_at")
output_file["type"] = set_config.get("set_type")
# Add booster info based on boosters resource (manually maintained for the time being)
with pathlib.Path(mtgjson4.RESOURCE_PATH,
"boosters.json").open("r", encoding="utf-8") as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"].upper() in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
if set_config.get("block"):
output_file["block"] = set_config.get("block")
if set_config.get("digital"):
output_file["isOnlineOnly"] = True
if set_config.get("foil_only"):
output_file["isFoilOnly"] = True
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder = convert_to_mtgjson(sf_cards)
card_holder, non_booster_cards = add_start_flag_and_count_modified(
set_code, set_config["search_uri"], card_holder)
output_file["totalSetSize"] = len(sf_cards)
output_file[
"baseSetSize"] = output_file["totalSetSize"] - non_booster_cards
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
return output_file
def build_output_file(
sf_cards: List[Dict[str, Any]], set_code: str, skip_keys: bool
) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param skip_keys: Skip building TCGPlayer & MKM components
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
if not skip_keys:
MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"]))
output_file: Dict[str, Any] = {}
# Get the set config from Scryfall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error("Set Config for {} was not found, skipping...".format(set_code))
return {"cards": [], "tokens": []}
output_file["name"] = set_config["name"]
output_file["code"] = set_config["code"].upper()
output_file["releaseDate"] = set_config["released_at"]
output_file["type"] = set_config["set_type"]
output_file["keyruneCode"] = (
pathlib.Path(set_config["icon_svg_uri"]).name.split(".")[0].upper()
)
# Try adding MKM Set Name
# Then store the card data for future pulling
if MKM_API.get(None):
mkm_resp = MKM_API.get().market_place.expansions(game=1)
if mkm_resp.status_code != 200:
LOGGER.error("Unable to download MKM correctly: {}".format(mkm_resp))
else:
for set_content in mkm_resp.json()["expansion"]:
if (
set_content["enName"].lower() == output_file["name"].lower()
or set_content["abbreviation"].lower()
== output_file["code"].lower()
):
output_file["mcmId"] = set_content["idExpansion"]
output_file["mcmName"] = set_content["enName"]
break
initialize_mkm_set_cards(output_file.get("mcmId", None))
# Add translations to the files
try:
output_file["translations"] = wizards.get_translations(output_file["code"])
except KeyError:
LOGGER.warning("Unable to find set translations for {}".format(set_code))
# Add optionals if they exist
if "mtgo_code" in set_config.keys():
output_file["mtgoCode"] = set_config["mtgo_code"].upper()
if "parent_set_code" in set_config.keys():
output_file["parentCode"] = set_config["parent_set_code"].upper()
if "block" in set_config.keys():
output_file["block"] = set_config["block"]
if "digital" in set_config.keys():
output_file["isOnlineOnly"] = set_config["digital"]
if "foil_only" in set_config.keys():
output_file["isFoilOnly"] = set_config["foil_only"]
if set_code.upper() in mtgjson4.NON_ENGLISH_SETS:
output_file["isForeignOnly"] = True
# Add booster info based on boosters resource (manually maintained for the time being)
with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open(
"r", encoding="utf-8"
) as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
# Add V3 code for some backwards compatibility
with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open(
"r", encoding="utf-8"
) as f:
json_dict = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["codeV3"] = json_dict[output_file["code"]]
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
"pricesDate": mtgjson4.__PRICE_UPDATE_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards)
card_holder = add_start_flag_and_count_modified(
set_code, set_config["search_uri"], card_holder
)
# Address duplicates in un-sets
card_holder = uniquify_duplicates_in_set(card_holder)
# Move bogus tokens out
card_holder, added_tokens = transpose_tokens(card_holder)
if not skip_keys:
# Add MTGStocks data in
card_holder = add_stocks_data(card_holder)
# Add TCGPlayer information
if "tcgplayer_id" in set_config.keys():
output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"]
if not skip_keys:
add_purchase_fields(output_file["tcgplayerGroupId"], card_holder)
# Set Sizes
output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper())
output_file["totalSetSize"] = len(sf_cards)
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
# Handle tokens
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
# Cleanups and UUIDs
mtgjson_card.DUEL_DECK_LAND_MARKED.set(False)
mtgjson_card.DUEL_DECK_SIDE_COMP.set("a")
for card in sorted(output_file["cards"]):
card.final_card_cleanup()
for token in output_file["tokens"]:
token.final_card_cleanup(is_card=False)
# Add Variations to each entry, as well as mark alternatives
add_variations_and_alternative_fields(output_file["cards"], output_file)
return output_file
