def construct_ch_price_dict(url_to_parse: str) -> Dict[str, float]:
"""
Turn CardHoarder API response into MTGJSON
consumable format.
:param url_to_parse: URL to pull CH data from
:return: MTGJSON dict
"""
response: Any = __get_session().get(url=url_to_parse.format(
SESSION_TOKEN.get("")),
timeout=5.0)
# Get data and log response
request_api_response: str = response.content.decode()
util.print_download_status(response)
mtgjson_to_price = {}
# All Entries from CH, cutting off headers
card_rows: List[str] = request_api_response.split("\n")[2:]
for card_row in card_rows:
split_row = card_row.split("\t")
# We're only indexing cards with MTGJSON UUIDs
if len(split_row[-1]) > 3:
# Last Row = UUID, 5th Row = Price
mtgjson_to_price[split_row[-1]] = float(split_row[5])
return mtgjson_to_price
def __get_stocks_data() -> Dict[str, Any]:
"""
Get the stocks data for later use
:return: All stocks data
"""
if not STOCKS_DATA.get(None):
stocks_file = mtgjson4.RESOURCE_PATH.joinpath("stocks_data.json")
is_file = stocks_file.is_file()
cache_expired = (
is_file
and time.time() - stocks_file.stat().st_mtime
> mtgjson4.SESSION_CACHE_EXPIRE_GENERAL
)
if (not is_file) or cache_expired:
# Rebuild set translations
session = __get_session()
response: Any = session.get(
url=MTG_STOCKS_API_URL.format(SESSION_TOKEN.get("")), timeout=5.0
)
request_api_json: List[Dict[str, Any]] = response.json()
util.print_download_status(response)
save_dictionary = {}
for row in request_api_json:
save_dictionary[row["tcg_id"]] = row
with stocks_file.open("w") as f:
json.dump(save_dictionary, f, indent=4)
f.write("\n")
STOCKS_DATA.set(json.load(stocks_file.open("r")))
return dict(STOCKS_DATA.get())
def build_and_write_decks(decks_url: str) -> Iterator[Dict[str, Any]]:
"""
Given the URL to the precons list, this will
compile them in MTGJSONv4 format and write out
the decks to the "decks/" folder.
:return Each deck completed, one by one
"""
session = util.get_generic_session()
response: Any = session.get(url=decks_url, timeout=5.0)
util.print_download_status(response)
# Location of AllSets.json -- Must be compiled before decks!
all_sets_path: pathlib.Path = mtgjson4.COMPILED_OUTPUT_DIR.joinpath(
mtgjson4.ALL_SETS_OUTPUT + ".json"
)
file_loaded: bool = False
# Does the file exist
if all_sets_path.is_file():
# Is the file > 100MB? (Ensure we have all sets in it)
if all_sets_path.stat().st_size > 1e8:
with all_sets_path.open("r") as f:
SET_SESSION.set(json.load(f))
file_loaded = True
if not file_loaded:
LOGGER.warning("AllSets must be fully compiled before decks. Aborting.")
return
with multiprocessing.Pool(processes=8) as pool:
for deck in response.json():
deck_to_output = {
"name": deck["name"],
"code": deck["set_code"].upper(),
"type": deck["type"],
"releaseDate": deck["release_date"],
"mainBoard": [],
"sideBoard": [],
"meta": {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
"pricesDate": mtgjson4.__PRICE_UPDATE_DATE__,
},
}
# Pool main board first
results: List[Any] = pool.map(build_single_card, deck["cards"])
for cards in results:
for card in cards:
deck_to_output["mainBoard"].append(card)
# Now pool side board
results = pool.map(build_single_card, deck["sideboard"])
for cards in results:
for card in cards:
deck_to_output["sideBoard"].append(card)
LOGGER.info("Finished deck {}".format(deck["name"]))
yield deck_to_output
def build_and_write_decks(decks_url: str) -> Iterator[Dict[str, Any]]:
"""
Given the URL to the precons list, this will
compile them in MTGJSONv4 format and write out
the decks to the "decks/" folder.
:return Each deck completed, one by one
"""
session = util.get_generic_session()
response: Any = session.get(url=decks_url, timeout=5.0)
util.print_download_status(response)
# Location of AllSets.json -- Must be compiled before decks!
all_sets_path: pathlib.Path = mtgjson4.COMPILED_OUTPUT_DIR.joinpath(
mtgjson4.ALL_SETS_OUTPUT + ".json"
)
file_loaded: bool = False
# Does the file exist
if all_sets_path.is_file():
# Is the file > 100MB? (Ensure we have all sets in it)
if all_sets_path.stat().st_size > 1e8:
with all_sets_path.open("r") as f:
SET_SESSION.set(json.load(f))
file_loaded = True
if not file_loaded:
LOGGER.warning("AllSets must be fully compiled before decks. Aborting.")
return
with multiprocessing.Pool(processes=8) as pool:
for deck in response.json():
deck_to_output = {
"name": deck["name"],
"code": deck["set_code"].upper(),
"type": deck["type"],
"releaseDate": deck["release_date"],
"mainBoard": [],
"sideBoard": [],
"meta": {
"version": mtgjson4.__VERSION__,
"date": mtgjson4.__VERSION_DATE__,
"pricesDate": mtgjson4.__PRICE_UPDATE_DATE__,
},
}
# Pool main board first
results: List[Any] = pool.map(build_single_card, deck["cards"])
for cards in results:
for card in cards:
deck_to_output["mainBoard"].append(card)
# Now pool side board
results = pool.map(build_single_card, deck["sideboard"])
for cards in results:
for card in cards:
deck_to_output["sideBoard"].append(card)
LOGGER.info("Finished deck {}".format(deck["name"]))
yield deck_to_output
def download_from_wizards(url: str) -> str:
"""
Generic download class for Wizards URLs
:param url: URL to download (prob from Wizards website)
:return: Text from page
"""
session = util.get_generic_session()
response: Any = session.get(url=url, timeout=5.0)
util.print_download_status(response)
session.close()
return str(response.text)
def download_from_wizards(url: str) -> str:
"""
Generic download class for Wizards URLs
:param url: URL to download (prob from Wizards website)
:return: Text from page
"""
session = util.get_generic_session()
response: Any = session.get(url=url, timeout=5.0)
util.print_download_status(response)
session.close()
return str(response.text)
def download(scryfall_url: str) -> Dict[str, Any]:
"""
Get the data from Scryfall in JSON format using our secret keys
:param scryfall_url: URL to download JSON data from
:return: JSON object of the Scryfall data
"""
session = __get_session()
response: Any = session.get(url=scryfall_url, timeout=5.0)
request_api_json: Dict[str, Any] = response.json()
util.print_download_status(response)
session.close()
return request_api_json
def download(scryfall_url: str) -> Dict[str, Any]:
"""
Get the data from Scryfall in JSON format using our secret keys
:param scryfall_url: URL to download JSON data from
:return: JSON object of the Scryfall data
"""
session = __get_session()
response: Any = session.get(url=scryfall_url, timeout=5.0)
request_api_json: Dict[str, Any] = response.json()
util.print_download_status(response)
session.close()
return request_api_json
def get_cards(multiverse_id: str) -> List[GathererCard]:
"""Get card(s) matching a given multiverseId."""
session = util.get_generic_session()
response: Any = session.get(
url=GATHERER_CARD,
params={"multiverseid": multiverse_id, "printed": "true"},
timeout=8.0,
)
util.print_download_status(response)
session.close()
return parse_cards(response.text)
def download(url: str, encoding: Optional[str] = None) -> str:
"""
Download a file from a specified source using
our generic session.
:param url: URL to download
:param encoding: URL encoding (if necessary)
:return: URL content
"""
session = util.get_generic_session()
response: Any = session.get(url)
if encoding:
response.encoding = encoding
util.print_download_status(response)
return str(response.text)
def download(url: str, encoding: Optional[str] = None) -> str:
"""
Download a file from a specified source using
our generic session.
:param url: URL to download
:param encoding: URL encoding (if necessary)
:return: URL content
"""
session = util.get_generic_session()
response: Any = session.get(url)
if encoding:
response.encoding = encoding
util.print_download_status(response)
return str(response.text)
def get_cards(multiverse_id: str) -> List[GathererCard]:
"""Get card(s) matching a given multiverseId."""
session = util.get_generic_session()
response: Any = session.get(
url=GATHERER_CARD,
params={
"multiverseid": multiverse_id,
"printed": "true"
},
timeout=8.0,
)
util.print_download_status(response)
session.close()
return parse_cards(response.text)
def download(tcgplayer_url: str, params_str: Dict[str, Any] = None) -> Optional[str]:
"""
Download content from TCGPlayer with a given URL that
can include a wildcard for default API version, as well
as a way to pass in custom params to the URL
:param tcgplayer_url: URL to get information from
:param params_str: Additional params to pass to TCGPlayer call
:return: Data from TCGPlayer API Call
"""
if params_str is None:
params_str = {}
try:
session = __get_session()
except configparser.NoOptionError:
return None
response: Any = session.get(
url=tcgplayer_url.replace("[API_VERSION]", TCGPLAYER_API_VERSION.get("")),
params=params_str,
timeout=5.0,
)
util.print_download_status(response)
session.close()
if response.status_code != 200:
if response.status_code == 404:
LOGGER.info(
"Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}".format(
response.status_code, response.url, params_str
)
)
else:
LOGGER.warning(
"Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}".format(
response.status_code, response.url, params_str
)
)
return None
return str(response.text)
def get_modern_sets() -> List[str]:
"""
Pull the modern legal page from Gamepedia and parse it out
to get the sets that are legal in modern
:return: List of set codes legal in modern
"""
session = util.get_generic_session()
response: Any = session.get(url=MODERN_GAMEPEDIA_URL, timeout=5.0)
util.print_download_status(response)
soup = bs4.BeautifulSoup(response.text, "html.parser")
soup = soup.find("div", class_="div-col columns column-width")
soup = soup.find_all("a")
modern_legal_sets = [
scryfall.get_set_header(strip_bad_sf_chars(x.text)).get("code", "").upper()
for x in soup
]
return modern_legal_sets
def download(tcgplayer_url: str,
params_str: Dict[str, Any] = None) -> Optional[str]:
"""
Download content from TCGPlayer with a given URL that
can include a wildcard for default API version, as well
as a way to pass in custom params to the URL
:param tcgplayer_url: URL to get information from
:param params_str: Additional params to pass to TCGPlayer call
:return: Data from TCGPlayer API Call
"""
if params_str is None:
params_str = {}
try:
session = __get_session()
except configparser.NoOptionError:
return None
response: Any = session.get(
url=tcgplayer_url.replace("[API_VERSION]",
TCGPLAYER_API_VERSION.get("")),
params=params_str,
timeout=5.0,
)
util.print_download_status(response)
session.close()
if response.status_code != 200:
if response.status_code == 404:
LOGGER.info(
"Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}"
.format(response.status_code, response.url, params_str))
else:
LOGGER.warning(
"Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}"
.format(response.status_code, response.url, params_str))
return None
return str(response.text)
