def main() -> None:
"""
Main Method
"""
args: argparse.Namespace = parse_args()
mtgjson4.USE_CACHE.set(not args.skip_cache)
mtgjson4.PRETTY_OUTPUT.set(4 if args.pretty_output else None)
if not mtgjson4.CONFIG_PATH.is_file():
LOGGER.warning(
f"No properties file found at {mtgjson4.CONFIG_PATH}. Will download without authentication"
)
# Determine set(s) to build
args_s = args.s if args.s else []
set_list: List[str] = get_all_sets() if args.a else args_s
if args.skip_sets:
set_list = sorted(list(set(set_list) - set(args.skip_sets)))
LOGGER.info(f"Skipping set(s) by request of user: {args.skip_sets}")
LOGGER.info(f"Sets to compile: {set_list}")
# If we had to kill mid-build, we can skip the completed set(s)
if args.x:
sets_compiled_already: List[str] = get_compiled_sets()
set_list = [
s for s in set_list if s.lower() not in sets_compiled_already
]
LOGGER.info(
f"Sets to skip compilation for: {sets_compiled_already}\n\nSets to compile, after cached sets removed: {set_list}"
)
for set_code in set_list:
sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code)
compiled = compile_mtg.build_mtgjson_set(sf_set, set_code,
args.skip_keys)
# If we have at least 1 card, dump to file SET.json
# but first add them to ReferralMap.json
if compiled["cards"] or compiled["tokens"]:
if not args.skip_keys:
for card in compiled["cards"]:
add_card_to_referral_map(card)
mtgjson4.outputter.write_to_file(set_code.upper(),
compiled,
set_file=True)
# Compile the additional outputs
if args.c:
LOGGER.info("Compiling additional outputs")
mtgjson4.outputter.create_and_write_compiled_outputs()
# Compress the output folder
if args.z:
LOGGER.info("Start compressing for production")
compressor.compress_output_folder()
LOGGER.info("Finished compressing for production")
def main() -> None:
"""
Main Method
"""
parser = argparse.ArgumentParser(description="")
parser.add_argument("-a", action="store_true")
parser.add_argument("-s", metavar="SET", nargs="*", type=str)
parser.add_argument("-c", action="store_true")
parser.add_argument("-x", action="store_true")
parser.add_argument("--skip-tcgplayer", action="store_true")
parser.add_argument("--skip-sets", metavar="SET", nargs="*", type=str)
# Ensure there are args
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
else:
args = parser.parse_args()
if not mtgjson4.CONFIG_PATH.is_file():
LOGGER.warning(
"No properties file found at {}. Will download without authentication"
.format(mtgjson4.CONFIG_PATH))
# Determine set(s) to build
args_s = args.s if args.s else []
set_list: List[str] = get_all_sets() if args.a else args_s
if args.skip_sets:
set_list = list(set(set_list) - set(args.skip_sets))
LOGGER.info("Skipping set(s) by request of user: {}".format(
args.skip_sets))
LOGGER.info("Sets to compile: {}".format(set_list))
# If we had to kill mid-build, we can skip the completed set(s)
if args.x:
sets_compiled_already: List[str] = get_compiled_sets()
set_list = [s for s in set_list if s not in sets_compiled_already]
LOGGER.info(
"Sets to skip compilation for: {}\n\nSets to compile, after cached sets removed: {}"
.format(sets_compiled_already, set_list))
for set_code in set_list:
sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code)
compiled = compile_mtg.build_output_file(sf_set, set_code,
args.skip_tcgplayer)
# If we have at least 1 card, dump to file SET.json
if compiled["cards"] or compiled["tokens"]:
mtgjson4.outputter.write_to_file(set_code.upper(),
compiled,
do_cleanup=True)
# Compile the additional outputs
if args.c:
LOGGER.info("Compiling Additional Outputs")
mtgjson4.outputter.create_and_write_compiled_outputs()
def main() -> None:
"""
Main Method
"""
args: argparse.Namespace = parse_args()
mtgjson4.USE_CACHE.set(not args.skip_cache)
if not mtgjson4.CONFIG_PATH.is_file():
LOGGER.warning(
"No properties file found at {}. Will download without authentication".format(
mtgjson4.CONFIG_PATH
)
)
# Determine set(s) to build
args_s = args.s if args.s else []
set_list: List[str] = get_all_sets() if args.a else args_s
if args.skip_sets:
set_list = list(set(set_list) - set(args.skip_sets))
LOGGER.info("Skipping set(s) by request of user: {}".format(args.skip_sets))
LOGGER.info("Sets to compile: {}".format(set_list))
# If we had to kill mid-build, we can skip the completed set(s)
if args.x:
sets_compiled_already: List[str] = get_compiled_sets()
set_list = [s for s in set_list if s not in sets_compiled_already]
LOGGER.info(
"Sets to skip compilation for: {}\n\nSets to compile, after cached sets removed: {}".format(
sets_compiled_already, set_list
)
)
for set_code in set_list:
sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code)
compiled = compile_mtg.build_output_file(sf_set, set_code, args.skip_keys)
# If we have at least 1 card, dump to file SET.json
# but first add them to ReferralMap.json
if compiled["cards"] or compiled["tokens"]:
if not args.skip_keys:
for card in compiled["cards"]:
add_card_to_referral_map(card)
mtgjson4.outputter.write_to_file(set_code.upper(), compiled, set_file=True)
# Compile the additional outputs
if args.c:
LOGGER.info("Compiling additional outputs")
mtgjson4.outputter.create_and_write_compiled_outputs()
# Compress the output folder
if args.z:
LOGGER.info("Start compressing for production")
compressor.compress_output_folder()
LOGGER.info("Finished compressing for production")
def main() -> None:
"""
Main Method
"""
parser = argparse.ArgumentParser(description="")
parser.add_argument("-s", metavar="SET", nargs="*", type=str)
parser.add_argument("-a", "--all-sets", action="store_true")
parser.add_argument("-c", "--compiled-outputs", action="store_true")
parser.add_argument("--skip-rebuild", action="store_true")
parser.add_argument("--skip-cached", action="store_true")
# Ensure there are args
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
else:
args = parser.parse_args()
if not pathlib.Path(mtgjson4.CONFIG_PATH).is_file():
LOGGER.warning(
"No properties file found at {}. Will download without authentication"
.format(mtgjson4.CONFIG_PATH))
if not args.skip_rebuild:
# Determine sets to build, whether they're passed in as args or all sets in our configs
args_s = args.s if args.s else []
set_list: List[str] = get_all_sets() if args.all_sets else args_s
LOGGER.info("Sets to compile: {}".format(set_list))
# If we had to kill mid-rebuild, we can skip the sets that already were done
if args.skip_cached:
sets_compiled_already: List[str] = get_compiled_sets()
set_list = [s for s in set_list if s not in sets_compiled_already]
LOGGER.info("Sets to skip compilation for: {}".format(
sets_compiled_already))
LOGGER.info(
"Sets to compile, after cached sets removed: {}".format(
set_list))
for set_code in set_list:
sf_set: List[Dict[str, Any]] = scryfall.get_set(set_code)
compiled: Dict[str, Any] = compile_mtg.build_output_file(
sf_set, set_code)
# If we have at least 1 card, dump to file SET.json
if compiled["cards"] or compiled["tokens"]:
mtgjson4.outputter.write_to_file(set_code.upper(),
compiled,
do_cleanup=True)
if args.compiled_outputs:
LOGGER.info("Compiling Additional Outputs")
mtgjson4.outputter.create_and_write_compiled_outputs()
def build_output_file(
sf_cards: List[Dict[str, Any]], set_code: str, skip_tcgplayer: bool
) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param skip_tcgplayer: Skip building TCGPlayer stuff
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
output_file: Dict[str, Any] = {}
# Get the set config from Scryfall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error("Set Config for {} was not found, skipping...".format(set_code))
return {"cards": [], "tokens": []}
output_file["name"] = set_config.get("name")
output_file["code"] = str(set_config.get("code", "")).upper()
output_file["mtgoCode"] = str(set_config.get("mtgo_code", "")).upper()
output_file["releaseDate"] = set_config.get("released_at")
output_file["type"] = set_config.get("set_type")
# Add booster info based on boosters resource (manually maintained for the time being)
with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open(
"r", encoding="utf-8"
) as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"].upper() in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
# Add V3 code for some backwards compatibility
with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open(
"r", encoding="utf-8"
) as f:
json_dict = json.load(f)
if output_file["code"].upper() in json_dict.keys():
output_file["codeV3"] = json_dict[output_file["code"]]
if set_config.get("block"):
output_file["block"] = set_config.get("block")
if set_config.get("digital"):
output_file["isOnlineOnly"] = True
if set_config.get("foil_only"):
output_file["isFoilOnly"] = True
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder = convert_to_mtgjson(sf_cards)
card_holder = add_start_flag_and_count_modified(
set_code, set_config["search_uri"], card_holder
)
# Address duplicates in un-sets
card_holder = uniquify_duplicates_in_set(card_holder)
# Move bogus tokens out
card_holder, added_tokens = transpose_tokens(card_holder)
# Add TCGPlayer information
if "tcgplayer_id" in set_config:
output_file["tcgplayerGroupId"] = set_config.get("tcgplayer_id")
if not skip_tcgplayer:
card_holder = add_tcgplayer_fields(
output_file["tcgplayerGroupId"], card_holder
)
# Set sizes; BASE SET SIZE WILL BE UPDATED BELOW
output_file["totalSetSize"] = len(sf_cards)
with mtgjson4.RESOURCE_PATH.joinpath("base_set_sizes.json").open(
"r", encoding="utf-8"
) as f:
output_file["baseSetSize"] = json.load(f).get(set_code.upper(), 0)
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
# Add UUID to each entry
add_uuid_to_cards(output_file["cards"], output_file["tokens"], output_file)
# Add Variations to each entry, as well as mark alternatives
add_variations_and_alternative_fields(output_file["cards"], output_file)
if set_code[:2] == "DD":
mark_duel_decks(output_file["cards"])
return output_file
def build_output_file(sf_cards: List[Dict[str, Any]], set_code: str,
skip_keys: bool) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param skip_keys: Skip building TCGPlayer & MKM components
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
if not skip_keys and os.environ["MKM_APP_TOKEN"] and os.environ[
"MKM_APP_SECRET"]:
MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"]))
output_file: Dict[str, Any] = {}
# Get the set config from Scryfall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error(
"Set Config for {} was not found, skipping...".format(set_code))
return {"cards": [], "tokens": []}
output_file["name"] = set_config["name"]
output_file["code"] = set_config["code"].upper()
output_file["releaseDate"] = set_config["released_at"]
output_file["type"] = set_config["set_type"]
output_file["keyruneCode"] = (pathlib.Path(
set_config["icon_svg_uri"]).name.split(".")[0].upper())
# Try adding MKM Set Name
# Then store the card data for future pulling
if MKM_API.get(None):
mkm_resp = MKM_API.get().market_place.expansions(game=1)
if mkm_resp.status_code != 200:
LOGGER.error(
"Unable to download MKM correctly: {}".format(mkm_resp))
else:
for set_content in mkm_resp.json()["expansion"]:
if (set_content["enName"].lower()
== output_file["name"].lower()
or set_content["abbreviation"].lower()
== output_file["code"].lower()):
output_file["mcmId"] = set_content["idExpansion"]
output_file["mcmName"] = set_content["enName"]
break
initialize_mkm_set_cards(output_file.get("mcmId", None))
# Add translations to the files
try:
output_file["translations"] = wizards.get_translations(
output_file["code"])
except KeyError:
LOGGER.warning(
"Unable to find set translations for {}".format(set_code))
# Add optionals if they exist
if "mtgo_code" in set_config.keys():
output_file["mtgoCode"] = set_config["mtgo_code"].upper()
if "parent_set_code" in set_config.keys():
output_file["parentCode"] = set_config["parent_set_code"].upper()
if "block" in set_config.keys():
output_file["block"] = set_config["block"]
if "digital" in set_config.keys():
output_file["isOnlineOnly"] = set_config["digital"]
if "foil_only" in set_config.keys():
output_file["isFoilOnly"] = set_config["foil_only"]
if set_code.upper() in mtgjson4.NON_ENGLISH_SETS:
output_file["isForeignOnly"] = True
# Add booster info based on boosters resource (manually maintained for the time being)
with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open(
"r", encoding="utf-8") as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
# Add V3 code for some backwards compatibility
with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open(
"r", encoding="utf-8") as f:
json_dict = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["codeV3"] = json_dict[output_file["code"]]
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
"pricesDate": mtgjson4.__PRICE_UPDATE_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards)
card_holder = add_start_flag_and_count_modified(set_code,
set_config["search_uri"],
card_holder)
# Address duplicates in un-sets
card_holder = uniquify_duplicates_in_set(card_holder)
# Move bogus tokens out
card_holder, added_tokens = transpose_tokens(card_holder)
if not skip_keys:
# Add MTGStocks data in
card_holder = add_stocks_data(card_holder)
# Add TCGPlayer information
if "tcgplayer_id" in set_config.keys():
output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"]
if not skip_keys:
add_purchase_fields(output_file["tcgplayerGroupId"], card_holder)
# Set Sizes
output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper())
output_file["totalSetSize"] = len(sf_cards)
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
# Handle tokens
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
# Cleanups and UUIDs
mtgjson_card.DUEL_DECK_LAND_MARKED.set(False)
mtgjson_card.DUEL_DECK_SIDE_COMP.set("a")
for card in sorted(output_file["cards"]):
card.final_card_cleanup()
for token in output_file["tokens"]:
token.final_card_cleanup(is_card=False)
# Add Variations to each entry, as well as mark alternatives
add_variations_and_alternative_fields(output_file["cards"], output_file)
return output_file
def build_output_file(sf_cards: List[Dict[str, Any]],
set_code: str) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
output_file: Dict[str, Any] = {}
# Get the set config from ScryFall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error(
"Set Config for {} was not found, skipping...".format(set_code))
return {"cards": []}
output_file["name"] = set_config.get("name")
output_file["code"] = set_config.get("code")
output_file["mtgoCode"] = set_config.get("mtgo_code")
output_file["releaseDate"] = set_config.get("released_at")
output_file["type"] = set_config.get("set_type")
# Add booster info based on boosters resource (manually maintained for the time being)
with pathlib.Path(mtgjson4.RESOURCE_PATH,
"boosters.json").open("r", encoding="utf-8") as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"].upper() in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
if set_config.get("block"):
output_file["block"] = set_config.get("block")
if set_config.get("digital"):
output_file["isOnlineOnly"] = True
if set_config.get("foil_only"):
output_file["isFoilOnly"] = True
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder = convert_to_mtgjson(sf_cards)
card_holder, non_booster_cards = add_start_flag_and_count_modified(
set_code, set_config["search_uri"], card_holder)
output_file["totalSetSize"] = len(sf_cards)
output_file[
"baseSetSize"] = output_file["totalSetSize"] - non_booster_cards
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
return output_file
def build_output_file(
sf_cards: List[Dict[str, Any]], set_code: str, skip_keys: bool
) -> Dict[str, Any]:
"""
Compile the entire XYZ.json file and pass it off to be written out
:param skip_keys: Skip building TCGPlayer & MKM components
:param sf_cards: Scryfall cards
:param set_code: Set code
:return: Completed JSON file
"""
if not skip_keys:
MKM_API.set(Mkm(_API_MAP["2.0"]["api"], _API_MAP["2.0"]["api_root"]))
output_file: Dict[str, Any] = {}
# Get the set config from Scryfall
set_config = scryfall.download(scryfall.SCRYFALL_API_SETS + set_code)
if set_config["object"] == "error":
LOGGER.error("Set Config for {} was not found, skipping...".format(set_code))
return {"cards": [], "tokens": []}
output_file["name"] = set_config["name"]
output_file["code"] = set_config["code"].upper()
output_file["releaseDate"] = set_config["released_at"]
output_file["type"] = set_config["set_type"]
output_file["keyruneCode"] = (
pathlib.Path(set_config["icon_svg_uri"]).name.split(".")[0].upper()
)
# Try adding MKM Set Name
# Then store the card data for future pulling
if MKM_API.get(None):
mkm_resp = MKM_API.get().market_place.expansions(game=1)
if mkm_resp.status_code != 200:
LOGGER.error("Unable to download MKM correctly: {}".format(mkm_resp))
else:
for set_content in mkm_resp.json()["expansion"]:
if (
set_content["enName"].lower() == output_file["name"].lower()
or set_content["abbreviation"].lower()
== output_file["code"].lower()
):
output_file["mcmId"] = set_content["idExpansion"]
output_file["mcmName"] = set_content["enName"]
break
initialize_mkm_set_cards(output_file.get("mcmId", None))
# Add translations to the files
try:
output_file["translations"] = wizards.get_translations(output_file["code"])
except KeyError:
LOGGER.warning("Unable to find set translations for {}".format(set_code))
# Add optionals if they exist
if "mtgo_code" in set_config.keys():
output_file["mtgoCode"] = set_config["mtgo_code"].upper()
if "parent_set_code" in set_config.keys():
output_file["parentCode"] = set_config["parent_set_code"].upper()
if "block" in set_config.keys():
output_file["block"] = set_config["block"]
if "digital" in set_config.keys():
output_file["isOnlineOnly"] = set_config["digital"]
if "foil_only" in set_config.keys():
output_file["isFoilOnly"] = set_config["foil_only"]
if set_code.upper() in mtgjson4.NON_ENGLISH_SETS:
output_file["isForeignOnly"] = True
# Add booster info based on boosters resource (manually maintained for the time being)
with mtgjson4.RESOURCE_PATH.joinpath("boosters.json").open(
"r", encoding="utf-8"
) as f:
json_dict: Dict[str, List[Any]] = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["boosterV3"] = json_dict[output_file["code"].upper()]
# Add V3 code for some backwards compatibility
with mtgjson4.RESOURCE_PATH.joinpath("gatherer_set_codes.json").open(
"r", encoding="utf-8"
) as f:
json_dict = json.load(f)
if output_file["code"] in json_dict.keys():
output_file["codeV3"] = json_dict[output_file["code"]]
# Declare the version of the build in the output file
output_file["meta"] = {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
"pricesDate": mtgjson4.__PRICE_UPDATE_DATE__,
}
LOGGER.info("Starting cards for {}".format(set_code))
card_holder: List[MTGJSONCard] = convert_to_mtgjson(sf_cards)
card_holder = add_start_flag_and_count_modified(
set_code, set_config["search_uri"], card_holder
)
# Address duplicates in un-sets
card_holder = uniquify_duplicates_in_set(card_holder)
# Move bogus tokens out
card_holder, added_tokens = transpose_tokens(card_holder)
if not skip_keys:
# Add MTGStocks data in
card_holder = add_stocks_data(card_holder)
# Add TCGPlayer information
if "tcgplayer_id" in set_config.keys():
output_file["tcgplayerGroupId"] = set_config["tcgplayer_id"]
if not skip_keys:
add_purchase_fields(output_file["tcgplayerGroupId"], card_holder)
# Set Sizes
output_file["baseSetSize"] = scryfall.get_base_set_size(set_code.upper())
output_file["totalSetSize"] = len(sf_cards)
output_file["cards"] = card_holder
LOGGER.info("Finished cards for {}".format(set_code))
# Handle tokens
LOGGER.info("Starting tokens for {}".format(set_code))
sf_tokens: List[Dict[str, Any]] = scryfall.get_set("t" + set_code)
output_file["tokens"] = build_mtgjson_tokens(sf_tokens + added_tokens)
LOGGER.info("Finished tokens for {}".format(set_code))
# Cleanups and UUIDs
mtgjson_card.DUEL_DECK_LAND_MARKED.set(False)
mtgjson_card.DUEL_DECK_SIDE_COMP.set("a")
for card in sorted(output_file["cards"]):
card.final_card_cleanup()
for token in output_file["tokens"]:
token.final_card_cleanup(is_card=False)
# Add Variations to each entry, as well as mark alternatives
add_variations_and_alternative_fields(output_file["cards"], output_file)
return output_file