def run_trueskill_openings():
con = get_mongo_connection()
db = con.test
games = db.games
collection = db.trueskill_openings
player_collection = db.trueskill_players
# player_collection.remove()
# collection.remove()
setup_openings_collection(collection)
# setup_openings_collection(player_collection)
opening_skill_table = DbBackedSkillTable(collection)
# player_skill_table = DbBackedSkillTable(player_collection)
args = utils.incremental_max_parser().parse_args()
scanner = incremental_scanner.IncrementalScanner('trueskill', db)
if not args.incremental:
scanner.reset()
collection.drop()
for ind, game in enumerate(
progress_meter(scanner.scan(db.games, {}), 100)):
if len(game['decks']) >= 2 and len(game['decks'][1]['turns']) >= 5:
update_skills_for_game(game, opening_skill_table)
if ind == args.max_games:
break
#player_skill_table.save()
opening_skill_table.save()
scanner.save()
print scanner.status_msg()
def delete_user(username=None):
"""
Deletes a user
:param username: Username to delete
"""
mongo = get_mongo_connection()
return mongo.db.accounts.remove({"username": username}, safe=True)
def delete_user(username=None):
"""
Deletes a user
:param username: Username to delete
"""
mongo = get_mongo_connection()
return mongo.db.accounts.remove({'username': username}, safe=True)
def update_user(username=None, data={}):
"""
Updates a user with the specified data
:param username: Username to update
:param data: Data to update as a dict
"""
mongo = get_mongo_connection()
return mongo.db.accounts.update({"username": username}, {"$set": data})
def update_user(username=None, data={}):
"""
Updates a user with the specified data
:param username: Username to update
:param data: Data to update as a dict
"""
mongo = get_mongo_connection()
return mongo.db.accounts.update({'username': username}, {'$set': data})
def get_user(data={}):
"""
Returns a user object from the datastore
:param data: Data to use for query
"""
mongo = get_mongo_connection()
user = mongo.db.accounts.find_one(data)
return user
def get_user(data={}):
"""
Returns a user object from the datastore
:param data: Data to use for query
"""
mongo = get_mongo_connection()
user = mongo.db.accounts.find_one(data)
return user
def check_for_work():
"""Examine the state of the database and generate tasks for necessary work.
This task is intended to be called on a regular basis, e.g., from
a frequently run cron job. It is intended to scan through the
database, identify what needs to be done to bring it to a current
state, and then create the needed tasks.
"""
connection = utils.get_mongo_connection()
db = connection.test
# Scrape isotropic for raw games
for date in isotropic.dates_needing_scraping(db):
scrape_raw_games.delay(date)
def check_for_work():
"""Examine the state of the database and generate tasks for necessary work.
This task is intended to be called on a regular basis, e.g., from
a frequently run cron job. It is intended to scan through the
database, identify what needs to be done to bring it to a current
state, and then create the needed tasks.
"""
connection = utils.get_mongo_connection()
db = connection.test
# Scrape goko for raw games
for date in goko.dates_needing_scraping(db):
scrape_raw_games.delay(date)
def calc_goals(game_ids, day):
""" Calculate the goals achieved in the passed list of games """
log.info("Calculating goals for %d game IDs from %s", len(game_ids), day)
connection = utils.get_mongo_connection()
db = connection.test
games_col = db.games
goals_col = db.goals
goals_error_col = db.goals_error
games = []
for game_id in game_ids:
game = games_col.find_one({'_id': game_id})
if game:
games.append(game)
else:
log.warning('Found nothing for game id %s', game_id)
return calculate_goals(games, goals_col, goals_error_col, day)
def parse_games(games, day):
"""Takes list of game ids and a game date and parses them out."""
log.info("Parsing %d games for %s", len(games), day)
connection = utils.get_mongo_connection()
db = connection.test
raw_games_col = db.raw_games
parsed_games_col = db.games
parse_error_col = db.parse_error
raw_games = []
for game_id in games:
game = raw_games_col.find_one({'_id': game_id})
if game:
raw_games.append(game)
else:
log.warning('Found nothing for game id %s', game_id)
return parse_and_insert(log, raw_games, parsed_games_col, parse_error_col, day)
def calc_goals(game_ids, day):
""" Calculate the goals achieved in the passed list of games """
log.info("Calculating goals for %d game IDs from %s", len(game_ids), day)
connection = utils.get_mongo_connection()
db = connection.test
games_col = db.games
goals_col = db.goals
goals_error_col = db.goals_error
games = []
for game_id in game_ids:
game = games_col.find_one({'_id': game_id})
if game:
games.append(game)
else:
log.warning('Found nothing for game id %s', game_id)
return calculate_goals(games, goals_col, goals_error_col, day)
def main():
args = parser.parse_args()
connection = utils.get_mongo_connection()
games_table = connection.test.games
games_table.ensure_index('players')
games_table.ensure_index('supply')
data_files_to_load = os.listdir('parsed_out')
data_files_to_load.sort()
find_id = re.compile('game-.*.html')
done = set()
for fn in data_files_to_load:
yyyymmdd = fn[:8]
print yyyymmdd
if not utils.includes_day(args, yyyymmdd):
print 'skipping', fn, 'because not in range'
continue
if args.incremental:
if yyyymmdd in done:
print 'skipping', fn, 'because done'
continue
contents = open('parsed_out/' + fn, 'r').read(100)
if contents.strip() == '[]':
print "empty contents (make parser not dump empty files?)", \
fn
continue
first_game_id_match = find_id.search(contents)
assert first_game_id_match is not None, (
'could not get id from %s in file %s' % (contents, fn))
first_game_id = first_game_id_match.group(0)
query = {'_id': first_game_id}
if games_table.find(query).count():
done.add(yyyymmdd)
print 'skipping', yyyymmdd, 'and marking as done'
continue
else:
print first_game_id, str(query), 'not in db, importing'
cmd = ('mongoimport -h localhost parsed_out/%s -c '
'games --jsonArray' % fn)
print cmd
os.system(cmd)
def main():
args = parser.parse_args()
connection = utils.get_mongo_connection()
games_table = connection.test.games
games_table.ensure_index('players')
games_table.ensure_index('supply')
data_files_to_load = os.listdir('parsed_out')
data_files_to_load.sort()
find_id = re.compile('game-.*.html')
done = set()
for fn in data_files_to_load:
yyyymmdd = fn[:8]
print yyyymmdd
if not utils.includes_day(args, yyyymmdd):
print 'skipping', fn, 'because not in range'
continue
if args.incremental:
if yyyymmdd in done:
print 'skipping', fn, 'because done'
continue
contents = open('parsed_out/' + fn, 'r').read(100)
if contents.strip() == '[]':
print "empty contents (make parser not dump empty files?)", \
fn
continue
first_game_id_match = find_id.search(contents)
assert first_game_id_match is not None, (
'could not get id from %s in file %s' % (contents, fn))
first_game_id = first_game_id_match.group(0)
query = {'_id': first_game_id}
if games_table.find(query).count():
done.add(yyyymmdd)
print 'skipping', yyyymmdd, 'and marking as done'
continue
else:
print first_game_id, str(query), 'not in db, importing'
cmd = ('mongoimport -h localhost parsed_out/%s -c '
'games --jsonArray' % fn)
print cmd
os.system(cmd)
def parse_games(games, day):
"""Takes list of game ids and a game date and parses them out."""
log.info("Parsing %d games for %s", len(games), day)
connection = utils.get_mongo_connection()
db = connection.test
raw_games_col = db.raw_games
parsed_games_col = db.games
parse_error_col = db.parse_error
raw_games = []
for game_id in games:
game = raw_games_col.find_one({'_id': game_id})
if game:
raw_games.append(game)
else:
log.warning('Found nothing for game id %s', game_id)
return parse_and_insert(log, raw_games, parsed_games_col, parse_error_col,
day)
def main():
c = utils.get_mongo_connection()
force_classification = True
prefix = 'data/test_small_'
# limit = 10000
r_output_file = open(prefix + 'r_format.data', 'w')
weka_output_file = open(prefix + 'games.arff', 'w')
librf_output_file = open(prefix + 'librf_games.csv', 'w')
librf_labels_file = open(prefix + 'librf_games_labels.txt', 'w')
libsvm_output_file = open(prefix + 'libsvm_games.txt', 'w')
write_r_header(r_output_file)
write_weka_header(weka_output_file, force_classification)
for raw_game in utils.progress_meter(
c.test.games.find({
'_id': {
'$gt': 'game-2010-10'
}
}).limit(20000), 100):
g = game.Game(raw_game)
if g.dubious_quality() or len(g.get_player_decks()) != 2:
continue
if force_classification and g.get_player_decks()[0].WinPoints() == 1.0:
print 'skipping tie'
saved_turn_ind = random.randint(0, len(g.get_turns()) - 1)
for ind, game_state in enumerate(g.game_state_iterator()):
# if ind == saved_turn_ind:
encoded_state = state_to_features(g, game_state)
if force_classification:
encoded_state[-1] = int(encoded_state[-1] / 2)
#output_state(encoded_state, r_output_file, ' ')
#output_state(encoded_state, weka_output_file, ',')
#output_state(encoded_state[:-1], librf_output_file, ',')
#librf_labels_file.write('%d\n' % encoded_state[-1])
output_libsvm_state(encoded_state, libsvm_output_file)
def main():
c = utils.get_mongo_connection()
force_classification = True
prefix = 'data/test_small_'
# limit = 10000
r_output_file = open(prefix + 'r_format.data', 'w')
weka_output_file = open(prefix + 'games.arff', 'w')
librf_output_file = open(prefix + 'librf_games.csv', 'w')
librf_labels_file = open(prefix + 'librf_games_labels.txt', 'w')
libsvm_output_file = open(prefix + 'libsvm_games.txt', 'w')
write_r_header(r_output_file)
write_weka_header(weka_output_file, force_classification)
for raw_game in utils.progress_meter(
c.test.games.find(
{'_id': {'$gt': 'game-2010-10'} }
).limit(20000), 100):
g = game.Game(raw_game)
if g.dubious_quality() or len(g.get_player_decks()) != 2:
continue
if force_classification and g.get_player_decks()[0].WinPoints() == 1.0:
print 'skipping tie'
saved_turn_ind = random.randint(0, len(g.get_turns()) - 1)
for ind, game_state in enumerate(g.game_state_iterator()):
# if ind == saved_turn_ind:
encoded_state = state_to_features(g, game_state)
if force_classification:
encoded_state[-1] = int(encoded_state[-1] / 2)
#output_state(encoded_state, r_output_file, ' ')
#output_state(encoded_state, weka_output_file, ',')
#output_state(encoded_state[:-1], librf_output_file, ',')
#librf_labels_file.write('%d\n' % encoded_state[-1])
output_libsvm_state(encoded_state, libsvm_output_file)
def create_user(username=None, password=None, first_name=None, last_name=None, \
is_admin=False):
"""
Creates a new user
:param username: Username of user
:param password: User password
:param first_name: First name of user
:param last_name: Last name of user
:param is_admin: Admin user
"""
mongo = get_mongo_connection()
obj_id = mongo.db.accounts.save(
{
'username': username,
'first_name': first_name,
'last_name': last_name,
'password': hash_text(password),
'is_admin': True,
},
safe=True)
return mongo.db.accounts.find_one(obj_id)
def create_user(username=None, password=None, first_name=None, last_name=None, is_admin=False):
"""
Creates a new user
:param username: Username of user
:param password: User password
:param first_name: First name of user
:param last_name: Last name of user
:param is_admin: Admin user
"""
mongo = get_mongo_connection()
obj_id = mongo.db.accounts.save(
{
"username": username,
"first_name": first_name,
"last_name": last_name,
"password": hash_text(password),
"is_admin": True,
},
safe=True,
)
return mongo.db.accounts.find_one(obj_id)
def main():
c = utils.get_mongo_connection()
force_classification = True
prefix = "data/test_huge_"
# limit = 10000
r_output_file = open(prefix + "r_format.data", "w")
weka_output_file = open(prefix + "games.arff", "w")
vw_output_file = open(prefix + "games.vw.txt", "w")
librf_output_file = open(prefix + "librf_games.csv", "w")
librf_labels_file = open(prefix + "librf_games_labels.txt", "w")
libsvm_output_file = open(prefix + "libsvm_games.txt", "w")
write_r_header(r_output_file)
write_weka_header(weka_output_file, force_classification)
for raw_game in utils.progress_meter(c.test.games.find()):
g = game.Game(raw_game)
if g.dubious_quality() or len(g.get_player_decks()) != 2:
continue
if force_classification and g.get_player_decks()[0].WinPoints() == 1.0:
print "skipping tie"
saved_turn_ind = random.randint(0, len(g.get_turns()) - 1)
for ind, game_state in enumerate(g.game_state_iterator()):
# if ind == saved_turn_ind:
encoded_state = state_to_features(g, game_state)
if force_classification:
encoded_state[-1] = int(encoded_state[-1] / 2)
# output_state(encoded_state, r_output_file, ' ')
# output_state(encoded_state, weka_output_file, ',')
# output_state(encoded_state[:-1], librf_output_file, ',')
# librf_labels_file.write('%d\n' % encoded_state[-1])
output_libsvm_state(encoded_state, libsvm_output_file)
output_vw_state(encoded_state, vw_output_file)
def main():
con = utils.get_mongo_connection()
ensure_all_indexes(con.test)
from utils import get_mongo_connection
from game import Game, PlayerDeck
import card_info
from bolt.io import Dataset, dense2sparse
from bolt.trainer.sgd import SGD, Log
from bolt.model import LinearModel
from collections import defaultdict
import math
import numpy as np
from pymongo import ASCENDING, DESCENDING
con = get_mongo_connection()
DB = con.test
MAX_TURNS = 40
REQUIRED_PLAYERS = 2
CARDS = sorted(card_info._card_info_rows.keys())
CARDS_INDEX = {}
for i, card in enumerate(CARDS):
CARDS_INDEX[card] = i
NCARDS = len(CARDS)
def logit(x):
return 1.0 / (1.0 + math.exp(-x))
def vp_only(deck):
newdeck = {}
for card in deck:
from utils import get_mongo_connection
con = get_mongo_connection()
DB = con.test
games = DB.games
from trueskill.trueskill import db_update_trueskill, get_skill, get_stdev
def results_to_ranks(results):
sorted_results = sorted(results)
return [sorted_results.index(r) for r in results]
def run_trueskill_players():
collection = DB.trueskill_players
collection.remove()
collection.ensure_index('name')
for game in games.find():
if len(game['decks']) >= 2:
players = []
results = []
for deck in game['decks']:
nturns = len(deck['turns'])
if deck['resigned']:
vp = -1000
else:
vp = deck['points']
results.append((-vp, nturns))
players.append(deck['name'])
ranks = results_to_ranks(results)
team_results = [
([player], [1.0], rank)
for player, rank in zip(players, ranks)
