def mine_set(setcode, releasedate, magiccardsinfocode): """Mine the date for a magic set""" html = miner.download(con, url_set % magiccardsinfocode) # Get set name result = re_set.search(html) if result is None: raise RuntimeError(_("Pattern match failed.")) setname, code = result.groups() # Get set cards cids = re_set2.findall(html) if cids is None: raise RuntimeError(_("Pattern (2) match failed.")) # Get full spoilers html = miner.download(con, url_spoiler % magiccardsinfocode) spoilers = re_set3.findall(html) if spoilers is None: raise RuntimeError(_("Pattern (3) match failed.")) if len(cids) != len(spoilers): missing = [] for cid, name in cids: for spoiler in spoilers: if name == spoiler[0]: break else: missing.append(name) raise RuntimeError(_("Missing cards: " + ", ".join(missing))) cardlist = [] for i in range(len(cids)): card = cards.Card() card.name, card.rarity, cardtype, card.power, card.toughness, \ card.manacost, card.text, card.flavor, card.artist = spoilers[i] for i in range(len(cids)): collectorsid, name = cids[i] if name == card.name: card.collectorsid = collectorsid cids[i:i+1] = [] break else: raise RuntimeError(_("Missing card: '%s'") % card.name) if card.manacost is None: card.manacost = "" card.text = card.text.replace("<br>", "\n") card.flavor = card.flavor.replace("<br>", "\n") card.flavor = card.flavor.replace("<i>", "").replace("</i>", "") t = cardtype.split(" - ") card.cardtype = t[0].strip() card.subtype = "" if len(t) <= 1 else t[1].strip() card.converted_cost = cards.convert_mana(card.manacost) card.setid = setcode card.setname = setname.strip() card.releasedate = releasedate.toordinal() card.derive_id() card.derive_colors() cardlist.append(card) return setname.strip(), cardlist
def mine_pricelist(setname): """Get the average price for a card""" url = urllib.quote(url_price % setname, "/?=") html = miner.download(con, url, convert_to_unicode=False) pricelist = [] for part in html.split("<TR height=20>"): match = re_name.search(part) if match is None: continue cardname = unicode(match.group(1), errors="ignore") cardname = cardname.replace(u"AE", u"\xc3") prices = re_price.findall(part) if prices is None or len(prices) < 3: continue lowest_price, average_price, highest_price = map(float, prices) pricelist.append((cardname, int(100 * average_price))) return pricelist
def mine_tokens(): """Mine all tokens""" html = miner.download(con, url_tokens) tokens = [] for part in html.split('</table>'): setname_match = re_token.search(part) if setname_match is None: continue setname = setname_match.group(1) # Get the set id cards._cursor.execute('SELECT * FROM "sets" WHERE "name" = ?', (setname,)) row = cards._cursor.fetchone() if row is None: continue # raise RuntimeError(_("Set not found in the database: '%s'.") # % setname) setid = row[0] releasedate = row[3] tokens_ = re_token2.findall(part) if tokens_ is None: continue for link, name, power, toughness, number, artist in tokens_: link = "/extras/" + link[7:-4] + "jpg" token = cards.Token() token.subtype = name token.cardtype = "Creature" token.power = power token.toughness = toughness token.collectorsid = number if number != "" else "0" token.artist = artist token.setname = setname token.setid = setid token.releasedate = releasedate token.derive_id() tokens.append((link, token)) if len(tokens) == 0: raise RuntimeError(_("Pattern (4) match failed.")) return tokens
def mine_pic(url, filename): """Download the card jpg""" pic = miner.download(con, url, False) with open(filename, 'wb') as f: f.write(pic)