def construct_ch_price_dict(url_to_parse: str) -> Dict[str, float]: """ Turn CardHoarder API response into MTGJSON consumable format. :param url_to_parse: URL to pull CH data from :return: MTGJSON dict """ response: Any = __get_session().get(url=url_to_parse.format( SESSION_TOKEN.get("")), timeout=5.0) # Get data and log response request_api_response: str = response.content.decode() util.print_download_status(response) mtgjson_to_price = {} # All Entries from CH, cutting off headers card_rows: List[str] = request_api_response.split("\n")[2:] for card_row in card_rows: split_row = card_row.split("\t") # We're only indexing cards with MTGJSON UUIDs if len(split_row[-1]) > 3: # Last Row = UUID, 5th Row = Price mtgjson_to_price[split_row[-1]] = float(split_row[5]) return mtgjson_to_price
def __get_stocks_data() -> Dict[str, Any]: """ Get the stocks data for later use :return: All stocks data """ if not STOCKS_DATA.get(None): stocks_file = mtgjson4.RESOURCE_PATH.joinpath("stocks_data.json") is_file = stocks_file.is_file() cache_expired = ( is_file and time.time() - stocks_file.stat().st_mtime > mtgjson4.SESSION_CACHE_EXPIRE_GENERAL ) if (not is_file) or cache_expired: # Rebuild set translations session = __get_session() response: Any = session.get( url=MTG_STOCKS_API_URL.format(SESSION_TOKEN.get("")), timeout=5.0 ) request_api_json: List[Dict[str, Any]] = response.json() util.print_download_status(response) save_dictionary = {} for row in request_api_json: save_dictionary[row["tcg_id"]] = row with stocks_file.open("w") as f: json.dump(save_dictionary, f, indent=4) f.write("\n") STOCKS_DATA.set(json.load(stocks_file.open("r"))) return dict(STOCKS_DATA.get())
def build_and_write_decks(decks_url: str) -> Iterator[Dict[str, Any]]: """ Given the URL to the precons list, this will compile them in MTGJSONv4 format and write out the decks to the "decks/" folder. :return Each deck completed, one by one """ session = util.get_generic_session() response: Any = session.get(url=decks_url, timeout=5.0) util.print_download_status(response) # Location of AllSets.json -- Must be compiled before decks! all_sets_path: pathlib.Path = mtgjson4.COMPILED_OUTPUT_DIR.joinpath( mtgjson4.ALL_SETS_OUTPUT + ".json" ) file_loaded: bool = False # Does the file exist if all_sets_path.is_file(): # Is the file > 100MB? (Ensure we have all sets in it) if all_sets_path.stat().st_size > 1e8: with all_sets_path.open("r") as f: SET_SESSION.set(json.load(f)) file_loaded = True if not file_loaded: LOGGER.warning("AllSets must be fully compiled before decks. Aborting.") return with multiprocessing.Pool(processes=8) as pool: for deck in response.json(): deck_to_output = { "name": deck["name"], "code": deck["set_code"].upper(), "type": deck["type"], "releaseDate": deck["release_date"], "mainBoard": [], "sideBoard": [], "meta": { "version": mtgjson4.__VERSION__, "date": mtgjson4.__VERSION_DATE__, "pricesDate": mtgjson4.__PRICE_UPDATE_DATE__, }, } # Pool main board first results: List[Any] = pool.map(build_single_card, deck["cards"]) for cards in results: for card in cards: deck_to_output["mainBoard"].append(card) # Now pool side board results = pool.map(build_single_card, deck["sideboard"]) for cards in results: for card in cards: deck_to_output["sideBoard"].append(card) LOGGER.info("Finished deck {}".format(deck["name"])) yield deck_to_output
def download_from_wizards(url: str) -> str: """ Generic download class for Wizards URLs :param url: URL to download (prob from Wizards website) :return: Text from page """ session = util.get_generic_session() response: Any = session.get(url=url, timeout=5.0) util.print_download_status(response) session.close() return str(response.text)
def download(scryfall_url: str) -> Dict[str, Any]: """ Get the data from Scryfall in JSON format using our secret keys :param scryfall_url: URL to download JSON data from :return: JSON object of the Scryfall data """ session = __get_session() response: Any = session.get(url=scryfall_url, timeout=5.0) request_api_json: Dict[str, Any] = response.json() util.print_download_status(response) session.close() return request_api_json
def get_cards(multiverse_id: str) -> List[GathererCard]: """Get card(s) matching a given multiverseId.""" session = util.get_generic_session() response: Any = session.get( url=GATHERER_CARD, params={"multiverseid": multiverse_id, "printed": "true"}, timeout=8.0, ) util.print_download_status(response) session.close() return parse_cards(response.text)
def download(url: str, encoding: Optional[str] = None) -> str: """ Download a file from a specified source using our generic session. :param url: URL to download :param encoding: URL encoding (if necessary) :return: URL content """ session = util.get_generic_session() response: Any = session.get(url) if encoding: response.encoding = encoding util.print_download_status(response) return str(response.text)
def get_cards(multiverse_id: str) -> List[GathererCard]: """Get card(s) matching a given multiverseId.""" session = util.get_generic_session() response: Any = session.get( url=GATHERER_CARD, params={ "multiverseid": multiverse_id, "printed": "true" }, timeout=8.0, ) util.print_download_status(response) session.close() return parse_cards(response.text)
def download(tcgplayer_url: str, params_str: Dict[str, Any] = None) -> Optional[str]: """ Download content from TCGPlayer with a given URL that can include a wildcard for default API version, as well as a way to pass in custom params to the URL :param tcgplayer_url: URL to get information from :param params_str: Additional params to pass to TCGPlayer call :return: Data from TCGPlayer API Call """ if params_str is None: params_str = {} try: session = __get_session() except configparser.NoOptionError: return None response: Any = session.get( url=tcgplayer_url.replace("[API_VERSION]", TCGPLAYER_API_VERSION.get("")), params=params_str, timeout=5.0, ) util.print_download_status(response) session.close() if response.status_code != 200: if response.status_code == 404: LOGGER.info( "Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}".format( response.status_code, response.url, params_str ) ) else: LOGGER.warning( "Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}".format( response.status_code, response.url, params_str ) ) return None return str(response.text)
def get_modern_sets() -> List[str]: """ Pull the modern legal page from Gamepedia and parse it out to get the sets that are legal in modern :return: List of set codes legal in modern """ session = util.get_generic_session() response: Any = session.get(url=MODERN_GAMEPEDIA_URL, timeout=5.0) util.print_download_status(response) soup = bs4.BeautifulSoup(response.text, "html.parser") soup = soup.find("div", class_="div-col columns column-width") soup = soup.find_all("a") modern_legal_sets = [ scryfall.get_set_header(strip_bad_sf_chars(x.text)).get("code", "").upper() for x in soup ] return modern_legal_sets
def download(tcgplayer_url: str, params_str: Dict[str, Any] = None) -> Optional[str]: """ Download content from TCGPlayer with a given URL that can include a wildcard for default API version, as well as a way to pass in custom params to the URL :param tcgplayer_url: URL to get information from :param params_str: Additional params to pass to TCGPlayer call :return: Data from TCGPlayer API Call """ if params_str is None: params_str = {} try: session = __get_session() except configparser.NoOptionError: return None response: Any = session.get( url=tcgplayer_url.replace("[API_VERSION]", TCGPLAYER_API_VERSION.get("")), params=params_str, timeout=5.0, ) util.print_download_status(response) session.close() if response.status_code != 200: if response.status_code == 404: LOGGER.info( "Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}" .format(response.status_code, response.url, params_str)) else: LOGGER.warning( "Status Code: {} Failed to download from TCGPlayer with URL: {}, Params: {}" .format(response.status_code, response.url, params_str)) return None return str(response.text)