def get_deck(id):
id = str(id)
url = 'http://api.gracefulstats.com/1.0/deck/view?id=' + id + '&cards=true'
try:
con = core.urlopen(url)
except urllib2.HTTPError as e:
logging.warning('Someone posted a bad URL: ' + url + ' (%s) ' % str(e))
return None
deck = set()
deckObject = json.loads(con)
colorIdentity = deckObject['deck']['color_identity']
name = deckObject['deck']['name']
deckFormat = deckObject['deck']['format']['name']
if (deckFormat != 'Commander'):
raise ValueError("This isn't a commander deck, try to change the type to commander")
commander = deckObject['deck']['commander']['name']
for card in deckObject['deck']['cards']:
deck.add(core.sanitize_cardname(card['name']))
out_deck = {
'commander': core.sanitize_cardname(commander),
'cards': sorted(list(deck)),
'date': deckObject['deck']['created'],
'ref': 'gracefulstats'
}
return out_deck
def scrapedeck(url_str):
logging.debug('attempting to scrape the deckstats url: %s ' % url_str)
url_fetch = url_str + EXPORT_APPEND
logging.debug("going to go fetch '%s'" %url_fetch)
try:
content = urllib2.urlopen(url_fetch).readlines()
except:
raise ValueError("Invalid URL '%s'" % url_str)
text = content[0][len('//NAME: '):-len('from DeckStats.net') - 2]
logging.debug('The name of this deck is: %s' % text)
cards = set()
sideboard = set()
for line in content:
line = line.split('//')[0]
line = line.split('#')[0]
line = line.strip()
if len(line) == 0:
continue
if line.startswith('SB:'):
sideboard.add(core.sanitize_cardname(line.split(' ', 2)[2]))
line = line[4:]
if not line[0] in '0123456789':
raise ValueError("This isn't a valid line of the form '# Card Name': %s " % line)
cardname = core.sanitize_cardname(line.split(' ', 1)[1])
cards.add(cardname)
commander = None
if len(sideboard) == 1:
cardname = list(sideboard)[0]
card = core.lookup_card(cardname)
if card.has_key('supertypes') and 'Legendary' in card['supertypes']:
commander = list(sideboard)[0]
if commander is None:
commander = guess_commander(cards, text)
out = {}
out['url'] = url_str
out['scrapedate'] = str(datetime.datetime.now())
out['commander'] = commander
out['cards'] = sorted( cards )
out['ref'] = 'deckstats'
return out
def get_deck(url):
try:
# I tack on /?fmt=txt because it gives th list of cards in a somewhat nice
# text format. If only there was an API...
con = core.urlopen(url.rstrip('/') + '/?fmt=txt')
except urllib2.HTTPError as e:
# This will happen on 404 or any other error.
logging.warning("Someone posted a bad URL: " + url + " (%s)" % str(e))
return None
deck = set()
# For each line in the content of the web page....
for line in con.splitlines():
line = line.strip()
if len(line) == 0:
continue
if not line[0] in '0123456789':
continue
# At this point, the line is not empty and the line starts with a number
# This, we know, is a card
# The line is tab delimited like this: "1\tAustere Command\n"
card = line.split('\t')[1]
try:
deck.add(core.sanitize_cardname(card))
except KeyError:
pass
except ValueError as e:
logging.warning("Ignored this card because of some sort of bad value")
# Call out to get_tappedout_info to grab the deck info
cmdr, colors, date = get_tappedout_info(url)
# if they didn't post the commander, i'm going to try to figure out who it is
if cmdr is None:
for card in deck:
cd = core.lookup_card(card)
if not cd.has_key('supertypes'):
continue
if 'Legendary' in cd['supertypes'] and sorted(list(core.color_identity(card))) == sorted(list(colors)):
# ok, we've got a legenadry with the colors i think the deck should be. i'll just stop here.
cmdr = card
break
else:
logging.warn("there was no legendary creature here.... and none was specified... something f'd up is going on")
cmdr = 'jedit ojanen'
deck.add(cmdr)
out_deck = { 'commander' : cmdr, 'cards' : sorted(list(deck)), 'date' : date }
return out_deck
def get_deck(url):
try:
# I tack on /?fmt=txt because it gives th list of cards in a somewhat nice
# text format. If only there was an API...
con = core.urlopen(url.rstrip('/') + '/?fmt=txt')
except urllib2.HTTPError as e:
# This will happen on 404 or any other error.
logging.warning("Someone posted a bad URL: " + url + " (%s)" % str(e))
return None
deck = set()
# For each line in the content of the web page....
for line in con.splitlines():
line = line.strip()
if len(line) == 0:
continue
if not line[0] in '0123456789':
continue
# At this point, the line is not empty and the line starts with a number
# This, we know, is a card
# The line is tab delimited like this: "1\tAustere Command\n"
card = line.split('\t')[1]
try:
deck.add(core.sanitize_cardname(card))
except KeyError:
pass
except ValueError as e:
logging.warning("Ignored this card because of some sort of bad value")
# Call out to get_tappedout_info to grab the deck info
cmdr, colors, date = get_tappedout_info(url)
# if they didn't post the commander, i'm going to try to figure out who it is
if cmdr is None:
for card in deck:
cd = core.lookup_card(card)
if not cd.has_key('supertypes'):
continue
if 'Legendary' in cd['supertypes'] and sorted(list(core.color_identity(card))) == sorted(list(colors)):
# ok, we've got a legenadry with the colors i think the deck should be. i'll just stop here.
cmdr = card
break
else:
logging.warn("there was no legendary creature here.... and none was specified... something f'd up is going on")
cmdr = 'jedit ojanen'
deck.add(cmdr)
out_deck = { 'commander' : cmdr, 'cards' : sorted(list(deck)), 'date' : date }
return out_deck
def cmdrdeck(self, commander):
commander = commander[:50]
cherrypy.response.headers['Access-Control-Allow-Origin'] = "*"
cherrypy.response.headers['Content-Type']= 'application/json'
try:
cmdr_out = json.loads(self.cmdr(commander))
except ZeroDivisionError:
return "Unfortunately, there are not enough decks in my database for '%s', so I can't generate a list" % commander
colors = [ clr for clr, cnt in cmdr_out['stats']['colors'].items() if cnt > 0 ]
lands = cmdr_out['stats']['lands']
nonlands = cmdr_out['stats']['nonlands']
outdeck = []
landmap = {'Red' : 'Mountain', 'Blue' : 'Island', 'Black' : 'Swamp', 'White' : 'Plains', 'Green' : 'Forest' }
lands -= len(colors)
for rec_dict in cmdr_out['recs']:
if 'Land' in rec_dict['card_info']['types'] and lands > 0:
outdeck.append(rec_dict)
lands -= 1
continue
if (not 'Land' in rec_dict['card_info']['types']) and nonlands > 0:
outdeck.append(rec_dict)
nonlands -= 1
continue
# fill out the lands
total = sum( cnt for clr, cnt in cmdr_out['stats']['colors'].items() )
old_lands = lands
basics = dict(zip(colors, [0] * len(colors)))
for color, count in cmdr_out['stats']['colors'].items():
if count == 0 : continue
num = int( float(count) / total * old_lands ) + 1
lands -= num
basics[color] += num
# tack on a card if we are at 99
if lands + nonlands > 2:
logging.warn('deck built had less than 98 cards... thats weird... %d' % len(outdeck))
while lands + nonlands > 0:
basics[random.choice(colors)] += 1
lands -= 1
out = {}
out['cards'] = outdeck
out['basics'] = [ (landmap[color], count) for color, count in basics.items()]
out['commander'] = cmdr_out['commander']
out['stats'] = deckstats.tally([ { 'cards' : [ core.sanitize_cardname(c['card_info']['name']) for c in out['cards'] ] } ])
return json.dumps(out)
def load_cards_from_json(file_path):
r = core.get_redis()
writecount = 0
# Go through each card in the set...
for card, conts in json.load(open(file_path)).items():
# If it has names, it's a split card or fuse card or something
if conts.has_key('names'):
cardname = core.sanitize_cardname('/'.join(conts['names'])).lower()
for name in conts['names']:
r.hset('CARDS_JSON', core.sanitize_cardname(name), json.dumps(conts))
r.hset('CARDS_JSON', core.sanitize_cardname(cardname), json.dumps(conts))
r.hset('CARDS_JSON', core.sanitize_cardname(cardname.replace('/', ' // ')), json.dumps(conts))
r.hset('CARDS_JSON', core.sanitize_cardname(cardname.replace('/', ' / ')), json.dumps(conts))
else:
cardname = core.sanitize_cardname(conts['name']).lower()
r.hset('CARDS_JSON', core.sanitize_cardname(cardname), json.dumps(conts))
writecount += 1
logging.debug('We just wrote ' + str(writecount) + ' card entries into Redis.')
def load_cards_from_json(file_path):
r = core.get_redis()
writecount = 0
# Go through each card in the set...
for card, conts in json.load(open(file_path)).items():
# If it has names, it's a split card or fuse card or something
if conts.has_key('names'):
cardname = core.sanitize_cardname('/'.join(conts['names'])).lower()
for name in conts['names']:
r.hset('CARDS_JSON', core.sanitize_cardname(name),
json.dumps(conts))
r.hset('CARDS_JSON', core.sanitize_cardname(cardname),
json.dumps(conts))
r.hset('CARDS_JSON',
core.sanitize_cardname(cardname.replace('/', ' // ')),
json.dumps(conts))
r.hset('CARDS_JSON',
core.sanitize_cardname(cardname.replace('/', ' / ')),
json.dumps(conts))
else:
cardname = core.sanitize_cardname(conts['name']).lower()
r.hset('CARDS_JSON', core.sanitize_cardname(cardname),
json.dumps(conts))
writecount += 1
logging.debug('We just wrote ' + str(writecount) +
' card entries into Redis.')
def get_commander_stats(commander):
ds = []
for deck in core.get_decks(core.color_identity(commander)):
if deck['commander'] == core.sanitize_cardname(commander):
ds.append(deck)
out = tally(ds)
out['commander'] = core.cap_cardname(commander)
return out
def get_commander_stats(commander):
ds = []
for deck in core.get_decks(core.color_identity(commander)):
if deck['commander'] == core.sanitize_cardname(commander):
ds.append(deck)
out = tally(ds)
out['commander'] = core.cap_cardname(commander)
return out
def closest_commander(partial_name):
pn = core.sanitize_cardname(partial_name)
for cn in COMMANDERS:
if pn == cn:
return cn
for cn in COMMANDERS:
if cn.startswith(pn):
return cn
for cn in COMMANDERS:
if pn in cn:
return cn
def closest_commander(partial_name):
pn = core.sanitize_cardname(partial_name)
for cn in COMMANDERS:
if pn == cn:
return cn
for cn in COMMANDERS:
if cn.startswith(pn):
return cn
for cn in COMMANDERS:
if pn in cn:
return cn
def get_deck(id):
id = str(id)
url = 'http://api.gracefulstats.com/1.0/deck/view?id=' + id + '&cards=true'
try:
con = core.urlopen(url)
except urllib2.HTTPError as e:
logging.warning('Someone posted a bad URL: ' + url + ' (%s) ' % str(e))
return None
deck = set()
deckObject = json.loads(con)
colorIdentity = deckObject['deck']['color_identity']
name = deckObject['deck']['name']
deckFormat = deckObject['deck']['format']['name']
if (deckFormat != 'Commander'):
raise ValueError(
"This isn't a commander deck, try to change the type to commander")
commander = deckObject['deck']['commander']['name']
for card in deckObject['deck']['cards']:
deck.add(core.sanitize_cardname(card['name']))
out_deck = {
'commander': core.sanitize_cardname(commander),
'cards': sorted(list(deck)),
'date': deckObject['deck']['created'],
'ref': 'gracefulstats'
}
return out_deck
def cmdr(self, commander):
commander = commander[:50]
cherrypy.response.headers['Access-Control-Allow-Origin'] = "*"
r = core.get_redis()
ckey = 'CACHE_COMMANDER_' + commander.replace(' ', '_')
if r.exists(ckey):
return r.get(ckey)
commander = core.sanitize_cardname(commander)
commander = closest_commander(commander)
colors = core.color_identity(commander)
decks = [ deck for deck in core.get_decks(colors) if deck['commander'] == commander]
out = {}
out['numdecks'] = len(decks)
cards = {}
for deck in decks:
for card in deck['cards']:
cards[card] = {'count' : 0, 'cardname' : card, 'card_info' : core.lookup_card(card)}
for deck in decks:
for card in deck['cards']:
if card == commander: continue
if card in ['swamp', 'island', 'mountain', 'forest', 'plains']: continue
cards[card]['count'] += 1
out['recs'] = [ pp for pp in sorted(cards.values(), key = (lambda x: -1 * x['count'])) if pp['count'] > 1 and pp['count'] > .1 * len(decks) ]
out['commander'] = core.cap_cardname(commander)
r.set(ckey, json.dumps(out), ex=60*60*24*7) # 7 day cache
return json.dumps(out)
def scrape_deck(url_str):
logging.debug('scraping a deck for %s' % url_str)
content = urllib2.urlopen(url_str).read()
parsed = bs.BeautifulSoup(content)
tables = parsed.findAll('table')
deck = []
# find the deck
for t in tables:
attrs = dict(t.attrs)
if attrs['class'] != u'deck':
continue
data = json.loads(attrs['data-card-list'])
num_cards = 0
for card in data['Deck']:
num_cards += card['Qty']
deck.append(core.sanitize_cardname(card['CardName']))
if num_cards < 95 or num_cards > 102:
# raise ValueError("This deck has %d cards... that's bad." % num_cards)
pass
if not core.lookup_card(deck[0]).has_key(u'supertypes') or not u'Legendary' in core.lookup_card(deck[0])[u'supertypes']:
raise ValueError("The first card in this deck is not legendary.")
break
else:
raise ValueError("I couldn't find a deck in this post")
out = {}
out['url'] = url_str
out['mtgsalvation'] = url_str
out['date'] = datetime.datetime.now().toordinal()
out['scrapedate'] = str(datetime.datetime.now())
out['commander'] = deck[0]
out['cards'] = sorted(deck)
out['ref'] = 'mtgsalvation'
return out
def get_tappedout_info(url, assume_now = True):
con = core.urlopen(url).splitlines()
# GET COMMANDER
cmdr = None
for line in con:
# First, we need to find the commander we're talking about here.
if line.strip().startswith('<a href="/mtg-card/'):
cmdr_url = 'http://tappedout.net' + line.split('"')[1]
# Unfortunately, it's not easy to grab the name from the commander immage
# We have to go deeper... (the commander's specific page)
ccon = core.urlopen(cmdr_url).splitlines()
# Look for the title because that contains the commander name
for cline in ccon:
if "<title>" in cline:
cmdr = core.sanitize_cardname(cline.split('>')[1].split('(')[0])
break
break
# GET COLORS
# GET COLORS from the pie on the right
colors = set([])
for line in con:
if line.strip().startswith('buildColorChart'):
if 'Green' in line: colors.add('GREEN')
if 'Red' in line: colors.add('RED')
if 'Blue' in line: colors.add('BLUE')
if 'Black' in line: colors.add('BLACK')
if 'White' in line: colors.add('WHITE')
break
colors = sorted(list(colors))
# override pie colors if we have a good commander
if cmdr is not None:
try:
colors = core.color_identity(cmdr)
except ValueError:
logging.warn('I have a commander that I don\'t think should exist')
cmdr = None
# this will happen if the commander is one that tappedout sees but is not in allcards.json (i.e. if it is new)
pass
# GET DATE
if assume_now:
date = datetime.datetime.now().toordinal() # Note: we use ordinal dates to represent the day.
else:
# Go fetch the time. Tappedout makes it hard because they say "3 days ago" or "2 years ago"
# and it's got so many options. So, this scraping is pretty fickle but seems to work fine.
for line in con:
line = line.lower()
if '<td>' in line and len(line) < 21 and ('day' in line or 'hour' in line or 'month' in line or 'year' in line):
num, unit = line.strip()[4:].split('<')[0].split()
num = int(num)
unit = unit.strip('s')
now = datetime.datetime.now().toordinal()
if unit == 'hour':
date = now
elif unit == 'day':
date = now - num
elif unit == 'month':
date = int(now - num * (365. / 12.))
elif unit == 'year':
date = now - num * 365
break
else:
date = now
return cmdr, colors, date
# Go through each card in the set...
for card, conts in json.load(open(file_path)).items():
# If it has names, it's a split card or fuse card or something
if conts.has_key('names'):
cardname = core.sanitize_cardname('/'.join(conts['names'])).lower()
for name in conts['names']:
r.hset('CARDS_JSON', core.sanitize_cardname(name), json.dumps(conts))
r.hset('CARDS_JSON', core.sanitize_cardname(cardname), json.dumps(conts))
r.hset('CARDS_JSON', core.sanitize_cardname(cardname.replace('/', ' // ')), json.dumps(conts))
r.hset('CARDS_JSON', core.sanitize_cardname(cardname.replace('/', ' / ')), json.dumps(conts))
else:
cardname = core.sanitize_cardname(conts['name']).lower()
r.hset('CARDS_JSON', core.sanitize_cardname(cardname), json.dumps(conts))
writecount += 1
logging.debug('We just wrote ' + str(writecount) + ' card entries into Redis.')
load_cards_from_json('AllCards.json')
for deck in open('decks_sample.json').readlines():
core.add_deck(json.loads(deck))
for cc in [ core.sanitize_cardname(c) for c in open('banlist.txt').read().strip().split('\n') ]:
core.get_redis().sadd('BANNED', cc)
def cmdr(self, commander, nolog=False):
commander = commander[:50]
cherrypy.response.headers['Content-Type']= 'application/json'
cherrypy.response.headers['Access-Control-Allow-Origin'] = "*"
r = core.get_redis()
commander = core.sanitize_cardname(commander)
commander = closest_commander(commander)
r = core.get_redis()
if not cherrypy.session.has_key('id'):
cherrypy.session['id'] = ''.join(random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for i in range(8))
if not nolog:
r.sadd("SESSION_CMDRSEARCH_" +cherrypy.session['id'], commander)
ckey = 'CACHE_COMMANDER_' + commander.replace(' ', '_')
if r.exists(ckey):
return r.get(ckey)
colors = core.color_identity(commander)
decks = [ deck for deck in core.get_decks(colors) if deck['commander'] == commander]
if len(decks) < 3:
return json.dumps({'error_code' : 'NOT_ENOUGH_DATA', 'message' : 'There are not enough decks in my database to generate recommendations for %s' % commander})
out = {}
out['numdecks'] = len(decks)
cards = {}
for deck in decks:
for card in deck['cards']:
try:
cards[card] = {'count' : 0, 'card_info' : {'name' : core.lookup_card(card)['name'], \
'types' : core.lookup_card(card)['types'], \
'colors' : core.lookup_card(card).get('colors', []), \
'cmc' : core.lookup_card(card).get('cmc', 0) \
} }
except TypeError:
logging.warn("for some reason card %s could not be looked up, ignoring." % card)
continue
for deck in decks:
for card in deck['cards']:
if card == commander: continue
if card in ['swamp', 'island', 'mountain', 'forest', 'plains']: continue
try:
cards[card]['count'] += 1
except KeyError:
continue
#out['recs'] = [ pp for pp in sorted(cards.values(), key = (lambda x: -1 * x['count'])) if pp['count'] > 1 and pp['count'] > .1 * len(decks) ]
out['recs'] = [ pp for pp in sorted(cards.values(), key = (lambda x: -1 * x['count'])) if pp['count'] > 1 ][:125]
out['commander'] = core.cap_cardname(commander)
out['stats'] = deckstats.get_commander_stats(commander)
# kmeans output for subtopics
if len(decks) > 15:
out['archetypes'] = kmeans.kmeans(commander)
r.set(ckey, json.dumps(out), ex=60*60*24*2) # 2 day cache
return json.dumps(out)
def cmdrdeck(self, commander):
commander = commander[:50]
cherrypy.response.headers['Access-Control-Allow-Origin'] = "*"
cherrypy.response.headers['Content-Type'] = 'application/json'
try:
cmdr_out = json.loads(self.cmdr(commander))
except ZeroDivisionError:
return "Unfortunately, there are not enough decks in my database for '%s', so I can't generate a list" % commander
colors = [
clr for clr, cnt in cmdr_out['stats']['colors'].items() if cnt > 0
]
lands = cmdr_out['stats']['lands']
nonlands = cmdr_out['stats']['nonlands']
outdeck = []
landmap = {
'Red': 'Mountain',
'Blue': 'Island',
'Black': 'Swamp',
'White': 'Plains',
'Green': 'Forest'
}
lands -= len(colors)
for rec_dict in cmdr_out['recs']:
if 'Land' in rec_dict['card_info']['types'] and lands > 0:
outdeck.append(rec_dict)
lands -= 1
continue
if (not 'Land' in rec_dict['card_info']['types']) and nonlands > 0:
outdeck.append(rec_dict)
nonlands -= 1
continue
# fill out the lands
total = sum(cnt for clr, cnt in cmdr_out['stats']['colors'].items())
old_lands = lands
basics = dict(zip(colors, [0] * len(colors)))
for color, count in cmdr_out['stats']['colors'].items():
if count == 0: continue
num = int(float(count) / total * old_lands) + 1
lands -= num
basics[color] += num
# tack on a card if we are at 99
if lands + nonlands > 2:
logging.warn(
'deck built had less than 98 cards... thats weird... %d' %
len(outdeck))
while lands + nonlands > 0:
basics[random.choice(colors)] += 1
lands -= 1
out = {}
out['cards'] = outdeck
out['basics'] = [(landmap[color], count)
for color, count in basics.items()]
out['commander'] = cmdr_out['commander']
out['stats'] = deckstats.tally([{
'cards': [
core.sanitize_cardname(c['card_info']['name'])
for c in out['cards']
]
}])
return json.dumps(out)
def cmdr(self, commander, nolog=False):
commander = commander[:50]
cherrypy.response.headers['Content-Type'] = 'application/json'
cherrypy.response.headers['Access-Control-Allow-Origin'] = "*"
r = core.get_redis()
commander = core.sanitize_cardname(commander)
commander = closest_commander(commander)
r = core.get_redis()
if not cherrypy.session.has_key('id'):
cherrypy.session['id'] = ''.join(
random.choice('0123456789abcdefghijklmnopqrstuvwxyz')
for i in range(8))
if not nolog:
r.sadd("SESSION_CMDRSEARCH_" + cherrypy.session['id'], commander)
ckey = 'CACHE_COMMANDER_' + commander.replace(' ', '_')
if r.exists(ckey):
return r.get(ckey)
colors = core.color_identity(commander)
decks = [
deck for deck in core.get_decks(colors)
if deck['commander'] == commander
]
if len(decks) < 3:
return json.dumps({
'error_code':
'NOT_ENOUGH_DATA',
'message':
'There are not enough decks in my database to generate recommendations for %s'
% commander
})
out = {}
out['numdecks'] = len(decks)
cards = {}
for deck in decks:
for card in deck['cards']:
try:
cards[card] = {'count' : 0, 'card_info' : {'name' : core.lookup_card(card)['name'], \
'types' : core.lookup_card(card)['types'], \
'colors' : core.lookup_card(card).get('colors', []), \
'cmc' : core.lookup_card(card).get('cmc', 0) \
} }
except TypeError:
logging.warn(
"for some reason card %s could not be looked up, ignoring."
% card)
continue
for deck in decks:
for card in deck['cards']:
if card == commander: continue
if card in ['swamp', 'island', 'mountain', 'forest', 'plains']:
continue
try:
cards[card]['count'] += 1
except KeyError:
continue
#out['recs'] = [ pp for pp in sorted(cards.values(), key = (lambda x: -1 * x['count'])) if pp['count'] > 1 and pp['count'] > .1 * len(decks) ]
out['recs'] = [
pp
for pp in sorted(cards.values(), key=(lambda x: -1 * x['count']))
if pp['count'] > 1
][:125]
out['commander'] = core.cap_cardname(commander)
out['stats'] = deckstats.get_commander_stats(commander)
# kmeans output for subtopics
if len(decks) > 15:
out['archetypes'] = kmeans.kmeans(commander)
r.set(ckey, json.dumps(out), ex=60 * 60 * 24 * 2) # 2 day cache
return json.dumps(out)
import cherrypy
import json
import core
import tappedout
import datetime
import logging
import deckstats
import random
import kmeans
import mtgsalvation
import deckstatscom
import gracefulstats
COMMANDERS = sorted(
core.sanitize_cardname(cn.decode('utf-8').strip().lower())
for cn in open('commanders.txt').readlines())
def closest_commander(partial_name):
pn = core.sanitize_cardname(partial_name)
for cn in COMMANDERS:
if pn == cn:
return cn
for cn in COMMANDERS:
if cn.startswith(pn):
return cn
for cn in COMMANDERS:
import cherrypy
import json
import core
import tappedout
import datetime
import logging
logging.basicConfig(filename='api.log')
COMMANDERS = sorted( core.sanitize_cardname(cn.decode('utf-8').strip().lower()) for cn in open('commanders.txt').readlines() )
def closest_commander(partial_name):
pn = core.sanitize_cardname(partial_name)
for cn in COMMANDERS:
if pn == cn:
return cn
for cn in COMMANDERS:
if cn.startswith(pn):
return cn
for cn in COMMANDERS:
if pn in cn:
return cn
class API(object):
_cp_config = {'tools.staticdir.on' : True,
'tools.staticdir.dir' : '/home/ubuntu/edhrec-site',
'tools.staticdir.index' : 'index.html',
def kmeans(cmdr, k=4):
# random.seed(52485)
cmdr = core.sanitize_cardname(cmdr)
card_to_idx = {}
idx_to_card = {}
dims = None
decks = []
i = 0
for deck in core.get_all_decks():
if deck['commander'] != cmdr:
continue
for card in deck['cards']:
if card in ['island', 'swamp', 'mountain', 'forest', 'plains', cmdr]:
continue
lo = core.lookup_card(card)
if lo is None or 'Land' in lo['types']: continue
if card_to_idx.has_key(card): continue
card_to_idx[card] = i
idx_to_card[i] = card
i += 1
ll = numpy.zeros(i, dtype=int)
idxs = []
for card in deck['cards']:
try:
idxs.append(card_to_idx[card])
except KeyError:
continue
for idx in idxs:
ll[idx] = 1
decks.append(ll)
for idx, deck in enumerate(decks):
decks[idx].resize(i, refcheck=False)
decks = numpy.array(decks, dtype=int)
kmc = sklearn.cluster.KMeans(n_clusters=k, init='k-means++', n_init=25, max_iter=300, tol=0.000001, precompute_distances=True, verbose=0, random_state=None, n_jobs=1)
kmc.fit(decks)
clusters = [ [] for i in range(k) ]
out = []
for idx, deck in enumerate(decks):
clusters[kmc.labels_[idx]].append([idx_to_card[idx] for idx, v in enumerate(deck) if v == 1])
for idx, cluster in enumerate(kmc.cluster_centers_):
outc = {}
sumdiff = sum([ cluster - other for other in kmc.cluster_centers_ ])
defining = sorted( enumerate(sumdiff), key=lambda x: x[1], reverse=True)[:12]
defining = [ {'score' : val, 'card_info' : {'name' : core.lookup_card(idx_to_card[jdx])['name'], \
'types' : core.lookup_card(idx_to_card[jdx])['types'], \
'colors' : core.lookup_card(idx_to_card[jdx]).get('colors', []), \
'cmc' : core.lookup_card(idx_to_card[jdx]).get('cmc', 0) } } for jdx, val in defining ]
topc = sorted( [(val, idx_to_card[jdx] ) for jdx, val in enumerate(cluster)], reverse=True)[:125]
topc = [ {'score' : val, 'card_info' : {'name' : core.lookup_card(card)['name'], \
'types' : core.lookup_card(card)['types'], \
'colors' : core.lookup_card(card).get('colors', []), \
'cmc' : core.lookup_card(card).get('cmc', 0) } } for val, card in topc ]
outc['defining'] = defining
outc['recs'] = topc
outc['numdecks'] = len(clusters[idx])
outc['percentdecks'] = int( len(clusters[idx]) / float(len(decks)) * 100 )
outc['commander'] = cmdr
outc['stats'] = deckstats.tally([ {'cards' : d } for d in clusters[idx] ])
out.append(outc)
return sorted(out, key=lambda x: x['percentdecks'], reverse=True)
def get_tappedout_info(url, assume_now = True):
con = core.urlopen(url).splitlines()
# GET COMMANDER
cmdr = None
for line in con:
# First, we need to find the commander we're talking about here.
if line.strip().startswith('<a href="/mtg-card/'):
cmdr_url = 'http://tappedout.net' + line.split('"')[1]
# Unfortunately, it's not easy to grab the name from the commander immage
# We have to go deeper... (the commander's specific page)
ccon = core.urlopen(cmdr_url).splitlines()
# Look for the title because that contains the commander name
for cline in ccon:
if "<title>" in cline:
cmdr = core.sanitize_cardname(cline.split('>')[1].split('(')[0])
break
break
# GET COLORS
# GET COLORS from the pie on the right
colors = set([])
for line in con:
if line.strip().startswith('buildColorChart'):
if 'Green' in line: colors.add('GREEN')
if 'Red' in line: colors.add('RED')
if 'Blue' in line: colors.add('BLUE')
if 'Black' in line: colors.add('BLACK')
if 'White' in line: colors.add('WHITE')
break
colors = sorted(list(colors))
# override pie colors if we have a good commander
if cmdr is not None:
try:
colors = core.color_identity(cmdr)
except ValueError:
logging.warn('I have a commander that I don\'t think should exist')
cmdr = None
# this will happen if the commander is one that tappedout sees but is not in allcards.json (i.e. if it is new)
pass
# GET DATE
if assume_now:
date = datetime.datetime.now().toordinal() # Note: we use ordinal dates to represent the day.
else:
# Go fetch the time. Tappedout makes it hard because they say "3 days ago" or "2 years ago"
# and it's got so many options. So, this scraping is pretty fickle but seems to work fine.
for line in con:
line = line.lower()
if '<td>' in line and len(line) < 21 and ('day' in line or 'hour' in line or 'month' in line or 'year' in line):
num, unit = line.strip()[4:].split('<')[0].split()
num = int(num)
unit = unit.strip('s')
now = datetime.datetime.now().toordinal()
if unit == 'hour':
date = now
elif unit == 'day':
date = now - num
elif unit == 'month':
date = int(now - num * (365. / 12.))
elif unit == 'year':
date = now - num * 365
break
else:
date = now
return cmdr, colors, date
json.dumps(conts))
r.hset('CARDS_JSON',
core.sanitize_cardname(cardname.replace('/', ' // ')),
json.dumps(conts))
r.hset('CARDS_JSON',
core.sanitize_cardname(cardname.replace('/', ' / ')),
json.dumps(conts))
else:
cardname = core.sanitize_cardname(conts['name']).lower()
r.hset('CARDS_JSON', core.sanitize_cardname(cardname),
json.dumps(conts))
writecount += 1
logging.debug('We just wrote ' + str(writecount) +
' card entries into Redis.')
load_cards_from_json('AllCards.json')
for deck in open('decks_sample.json').readlines():
core.add_deck(json.loads(deck))
for cc in [
core.sanitize_cardname(c)
for c in open('banlist.txt').read().strip().split('\n')
]:
core.get_redis().sadd('BANNED', cc)