def mine_set(setcode, releasedate, magiccardsinfocode):
"""Mine the date for a magic set"""
html = miner.download(con, url_set % magiccardsinfocode)
# Get set name
result = re_set.search(html)
if result is None:
raise RuntimeError(_("Pattern match failed."))
setname, code = result.groups()
# Get set cards
cids = re_set2.findall(html)
if cids is None:
raise RuntimeError(_("Pattern (2) match failed."))
# Get full spoilers
html = miner.download(con, url_spoiler % magiccardsinfocode)
spoilers = re_set3.findall(html)
if spoilers is None:
raise RuntimeError(_("Pattern (3) match failed."))
if len(cids) != len(spoilers):
missing = []
for cid, name in cids:
for spoiler in spoilers:
if name == spoiler[0]:
break
else:
missing.append(name)
raise RuntimeError(_("Missing cards: " + ", ".join(missing)))
cardlist = []
for i in range(len(cids)):
card = cards.Card()
card.name, card.rarity, cardtype, card.power, card.toughness, \
card.manacost, card.text, card.flavor, card.artist = spoilers[i]
for i in range(len(cids)):
collectorsid, name = cids[i]
if name == card.name:
card.collectorsid = collectorsid
cids[i:i+1] = []
break
else:
raise RuntimeError(_("Missing card: '%s'") % card.name)
if card.manacost is None:
card.manacost = ""
card.text = card.text.replace("<br>", "\n")
card.flavor = card.flavor.replace("<br>", "\n")
card.flavor = card.flavor.replace("<i>", "").replace("</i>", "")
t = cardtype.split(" - ")
card.cardtype = t[0].strip()
card.subtype = "" if len(t) <= 1 else t[1].strip()
card.converted_cost = cards.convert_mana(card.manacost)
card.setid = setcode
card.setname = setname.strip()
card.releasedate = releasedate.toordinal()
card.derive_id()
card.derive_colors()
cardlist.append(card)
return setname.strip(), cardlist
def mine_pricelist(setname):
"""Get the average price for a card"""
url = urllib.quote(url_price % setname, "/?=")
html = miner.download(con, url, convert_to_unicode=False)
pricelist = []
for part in html.split("<TR height=20>"):
match = re_name.search(part)
if match is None:
continue
cardname = unicode(match.group(1), errors="ignore")
cardname = cardname.replace(u"AE", u"\xc3")
prices = re_price.findall(part)
if prices is None or len(prices) < 3:
continue
lowest_price, average_price, highest_price = map(float, prices)
pricelist.append((cardname, int(100 * average_price)))
return pricelist
def mine_tokens():
"""Mine all tokens"""
html = miner.download(con, url_tokens)
tokens = []
for part in html.split('</table>'):
setname_match = re_token.search(part)
if setname_match is None:
continue
setname = setname_match.group(1)
# Get the set id
cards._cursor.execute('SELECT * FROM "sets" WHERE "name" = ?',
(setname,))
row = cards._cursor.fetchone()
if row is None:
continue
# raise RuntimeError(_("Set not found in the database: '%s'.")
# % setname)
setid = row[0]
releasedate = row[3]
tokens_ = re_token2.findall(part)
if tokens_ is None:
continue
for link, name, power, toughness, number, artist in tokens_:
link = "/extras/" + link[7:-4] + "jpg"
token = cards.Token()
token.subtype = name
token.cardtype = "Creature"
token.power = power
token.toughness = toughness
token.collectorsid = number if number != "" else "0"
token.artist = artist
token.setname = setname
token.setid = setid
token.releasedate = releasedate
token.derive_id()
tokens.append((link, token))
if len(tokens) == 0:
raise RuntimeError(_("Pattern (4) match failed."))
return tokens
def mine_pic(url, filename): """Download the card jpg""" pic = miner.download(con, url, False) with open(filename, 'wb') as f: f.write(pic)
