def setUp(self, mock_requests):
mock_tournament_response = Mock(spec=requests.Response)
mock_matches_response = Mock(spec=requests.Response)
mock_participants_response = Mock(spec=requests.Response)
with open(TOURNAMENT_JSON_FILE) as f:
self.tournament_json_dict = json.load(f)
with open(MATCHES_JSON_FILE) as f:
self.matches_json_dict = json.load(f)
with open(PARTICIPANTS_JSON_FILE) as f:
self.participants_json_dict = json.load(f)
mock_tournament_response.status_code = 200
mock_matches_response.status_code = 200
mock_participants_response.status_code = 200
mock_tournament_response.json.return_value = self.tournament_json_dict
mock_matches_response.json.return_value = self.matches_json_dict
mock_participants_response.json.return_value = self.participants_json_dict
expected_tournament_url = scraper.challonge.TOURNAMENT_URL % TOURNAMENT_ID;
expected_matches_url = scraper.challonge.MATCHES_URL % TOURNAMENT_ID;
expected_participants_url = scraper.challonge.PARTICIPANTS_URL % TOURNAMENT_ID;
mock_requests_return_values = {
(expected_tournament_url, TEMPLATE_API_KEY): mock_tournament_response,
(expected_matches_url, TEMPLATE_API_KEY): mock_matches_response,
(expected_participants_url, TEMPLATE_API_KEY): mock_participants_response
}
mock_requests.get.side_effect = lambda url, **kwargs: mock_requests_return_values[(url, kwargs['params']['api_key'])]
self.scraper = ChallongeScraper(TOURNAMENT_ID, TEMPLATE_CONFIG_FILE_PATH)
def test_get_tournament_id_from_url(self):
self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_1), 'faketournament')
self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_2), 'TWL7')
self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_3),
'jerseyjapes-jj109meleesingles')
self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_4), 'tdml46')
self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_5),
'vgbootcamp-f117ijy1')
def import_tournament(type, path, bracket, region, name):
config = Config()
mongo_client = MongoClient(host=config.get_mongo_url())
if type == 'tio':
scraper = TioScraper.from_file(path, bracket)
elif type == 'challonge':
scraper = ChallongeScraper(path)
else:
click.echo("Illegal type")
dao = Dao(region, mongo_client=mongo_client)
player_map = get_player_alias_to_id_map(scraper, dao)
# TODO pass in a map of overrides for specific players
tournament = Tournament.from_scraper(type, scraper, player_map, region)
if name:
tournament.name = name
dao.insert_tournament(tournament)
click.echo("Generating new ranking...")
rankings.generate_ranking(dao)
click.echo("Done!")
def test_get_tournament_id_from_url(self):
self.assertEquals(
ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_1),
'faketournament')
self.assertEquals(
ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_2),
'TWL7')
self.assertEquals(
ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_3),
'jerseyjapes-jj109meleesingles')
self.assertEquals(
ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_4),
'tdml46')
self.assertEquals(
ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_5),
'vgbootcamp-f117ijy1')
def test_get_tournament_id_from_invalid_url(self):
with self.assertRaises(ValueError):
ChallongeScraper.get_tournament_id_from_url(INVALID_TOURNAMENT_URL_1)
with self.assertRaises(ValueError):
ChallongeScraper.get_tournament_id_from_url(INVALID_TOURNAMENT_URL_2)
with self.assertRaises(ValueError):
ChallongeScraper.get_tournament_id_from_url(INVALID_TOURNAMENT_URL_3)
def post(self, region):
dao = Dao(region, mongo_client=mongo_client)
user = get_user_from_request(request, dao)
if not user:
return 'Permission denied', 403
if not is_user_admin_for_region(user, region):
return 'Permission denied', 403
parser = reqparse.RequestParser()
parser.add_argument('type', type=str, location='json')
parser.add_argument('data', type=unicode, location='json')
parser.add_argument('bracket', type=str, location='json')
args = parser.parse_args()
print "in server post, actually got this far!"
if args['data'] is None:
return "data required", 400
the_bytes = bytearray(args['data'], "utf8")
if the_bytes[0] == 0xef:
print "found magic numbers"
return "magic numbers!", 503
type = args['type']
data = args['data']
if type == 'tio':
if args['bracket'] is None:
return "Missing bracket name", 400
data_bytes = bytes(data)
if data_bytes[0] == '\xef':
data = data[:3]
scraper = TioScraper(data, args['bracket'])
elif type == 'challonge':
scraper = ChallongeScraper(data)
else:
return "Unknown type", 400
pending_tournament = PendingTournament.from_scraper(
type, scraper, region)
pending_tournament.alias_to_id_map = alias_service.get_alias_to_id_map_in_list_format(
dao, pending_tournament.players)
new_id = dao.insert_pending_tournament(pending_tournament)
return_dict = {'id': str(new_id)}
return return_dict
def test_get_tournament_id_from_invalid_url(self):
with self.assertRaises(ValueError):
ChallongeScraper.get_tournament_id_from_url(
INVALID_TOURNAMENT_URL_1)
with self.assertRaises(ValueError):
ChallongeScraper.get_tournament_id_from_url(
INVALID_TOURNAMENT_URL_2)
with self.assertRaises(ValueError):
ChallongeScraper.get_tournament_id_from_url(
INVALID_TOURNAMENT_URL_3)
def bulk_import(path, region):
config = parse_config()
username = config.get('database', 'user')
host = config.get('database', 'host')
auth_db = config.get('database', 'auth_db')
password = getpass.getpass()
mongo_client = MongoClient(host='mongodb://%s:%s@%s/%s' %
(username, password, host, auth_db))
dao = Dao(region, mongo_client=mongo_client, new=True)
with open(path) as f:
for line in f:
line = line.strip()
if line:
print line
scraper = ChallongeScraper(line)
import_players(scraper, dao)
import_tournament(scraper, dao)
rankings.generate_ranking(dao)
def post(self, region):
dao = Dao(region, mongo_client=mongo_client)
user = get_user_from_access_token(request.headers, dao)
if not is_user_admin_for_region(user, region):
return 'Permission denied', 403
parser = reqparse.RequestParser()
parser.add_argument('type', type=str, location='json')
parser.add_argument('data', type=unicode, location='json')
parser.add_argument('bracket', type=str, location='json')
args = parser.parse_args()
if args['data'] is None:
return "data required", 400
type = args['type']
data = args['data']
if type == 'tio':
if args['bracket'] is None:
return "Missing bracket name", 400
scraper = TioScraper(data, args['bracket'])
elif type == 'challonge':
scraper = ChallongeScraper(data)
else:
return "Unknown type", 400
pending_tournament = PendingTournament.from_scraper(
type, scraper, region)
pending_tournament.alias_to_id_map = alias_service.get_alias_to_id_map_in_list_format(
dao, pending_tournament.players)
new_id = dao.insert_pending_tournament(pending_tournament)
return_dict = {'id': str(new_id)}
return return_dict
def post(self, region):
parser = reqparse.RequestParser() \
.add_argument('type', type=str, location='json') \
.add_argument('data', type=unicode, location='json') \
.add_argument('bracket', type=str, location='json')
args = parser.parse_args()
if args['data'] is None:
err("Data required. (TournamentSeedResource.post)")
the_bytes = bytearray(args['data'], "utf8")
if the_bytes[0] == 0xef:
err("Magic numbers! (TournamentSeedResource.post)")
type = args['type']
data = args['data']
pending_tournament = None
try:
if type == 'challonge':
scraper = ChallongeScraper(data)
else:
err("Unknown type")
pending_tournament, raw_file = M.PendingTournament.from_scraper(
type, scraper, region)
except Exception as ex:
err('Scraper encountered an error ' + str(ex))
if not pending_tournament or not raw_file:
err('Scraper encountered an error - null')
pending_tournament_json = pending_tournament.dump(
context='web', exclude=('date', 'matches', 'regions', 'type'))
return pending_tournament_json
class TestChallongeScraper(unittest.TestCase):
@patch('scraper.challonge.requests', spec=requests)
def setUp(self, mock_requests):
mock_tournament_response = Mock(spec=requests.Response)
mock_matches_response = Mock(spec=requests.Response)
mock_participants_response = Mock(spec=requests.Response)
with open(TOURNAMENT_JSON_FILE) as f:
self.tournament_json_dict = json.load(f)
with open(MATCHES_JSON_FILE) as f:
self.matches_json_dict = json.load(f)
with open(PARTICIPANTS_JSON_FILE) as f:
self.participants_json_dict = json.load(f)
mock_tournament_response.status_code = 200
mock_matches_response.status_code = 200
mock_participants_response.status_code = 200
mock_tournament_response.json.return_value = self.tournament_json_dict
mock_matches_response.json.return_value = self.matches_json_dict
mock_participants_response.json.return_value = self.participants_json_dict
expected_tournament_url = scraper.challonge.TOURNAMENT_URL % TOURNAMENT_ID;
expected_matches_url = scraper.challonge.MATCHES_URL % TOURNAMENT_ID;
expected_participants_url = scraper.challonge.PARTICIPANTS_URL % TOURNAMENT_ID;
mock_requests_return_values = {
(expected_tournament_url, TEMPLATE_API_KEY): mock_tournament_response,
(expected_matches_url, TEMPLATE_API_KEY): mock_matches_response,
(expected_participants_url, TEMPLATE_API_KEY): mock_participants_response
}
mock_requests.get.side_effect = lambda url, **kwargs: mock_requests_return_values[(url, kwargs['params']['api_key'])]
self.scraper = ChallongeScraper(TOURNAMENT_ID, TEMPLATE_CONFIG_FILE_PATH)
def test_get_raw(self):
raw = self.scraper.get_raw()
self.assertEquals(len(raw.keys()), 3)
self.assertEquals(raw['tournament'], self.tournament_json_dict)
self.assertEquals(raw['matches'], self.matches_json_dict)
self.assertEquals(raw['participants'], self.participants_json_dict)
def test_get_name(self):
self.assertEquals(self.scraper.get_name(), 'SF Game Night 21')
def test_get_date(self):
self.assertEquals(self.scraper.get_date().replace(tzinfo=None), datetime(2014, 10, 14, 20, 39, 30))
def test_get_matches(self):
matches = self.scraper.get_matches()
self.assertEquals(len(matches), 81)
self.assertEquals(matches[0], AliasMatch(winner='Tiamat', loser='Sharkboi'))
self.assertEquals(matches[-1], AliasMatch(winner='Shroomed', loser='GC | Silentwolf'))
self.assertEquals(matches[-2], AliasMatch(winner='GC | Silentwolf', loser='Shroomed'))
self.assertEquals(matches[-3], AliasMatch(winner='GC | Silentwolf', loser='MIOM | SFAT'))
# make sure none of the matches have a None
for m in matches:
self.assertIsNotNone(m.winner)
self.assertIsNotNone(m.loser)
def test_get_player(self):
players = self.scraper.get_players()
self.assertEquals(len(players), 41)
self.assertTrue('Shroomed' in players)
self.assertTrue('MIOM | SFAT' in players)
def post(self, region):
dao = get_dao(region)
auth_user(request, dao)
parser = reqparse.RequestParser() \
.add_argument('type', type=str, location='json') \
.add_argument('data', type=unicode, location='json') \
.add_argument('bracket', type=str, location='json') \
.add_argument('included_phases', type=list, location='json')
args = parser.parse_args()
if args['data'] is None:
err("Tournament data required.")
the_bytes = bytearray(args['data'], "utf8")
if the_bytes[0] == 0xef:
print "found magic numbers"
err("Magic numbers!")
type = args['type']
data = args['data']
included_phases = args['included_phases']
pending_tournament = None
try:
if type == 'tio':
if args['bracket'] is None:
err("Missing bracket name")
data_bytes = bytes(data)
if data_bytes[0] == '\xef':
data = data[:3]
scraper = TioScraper(data, args['bracket'])
elif type == 'challonge':
scraper = ChallongeScraper(data)
elif type == 'smashgg':
scraper = SmashGGScraper(data, included_phases)
else:
err("Unknown tournament type")
pending_tournament, raw_file = M.PendingTournament.from_scraper(
type, scraper, region)
except Exception as ex:
err('Scraper encountered an error: ' + str(ex))
if not pending_tournament or not raw_file:
err('Scraper encountered an error.')
try:
pending_tournament.alias_to_id_map = alias_service.get_alias_to_id_map_in_list_format(
dao, pending_tournament.players)
except Exception as e:
log_exception()
err('Alias service encountered an error')
# If the tournament is too large, don't insert the raw file into the db.
if len(pending_tournament.players) < 1000:
try:
raw_file = dao.insert_raw_file(raw_file)
except Exception as ex:
print ex
err('Dao insert_raw_file encountered an error')
else:
print 'Skipping inserting raw file for tournament because it is too large'
try:
new_id = dao.insert_pending_tournament(pending_tournament)
return_dict = {
'id': str(new_id)
}
return return_dict
except Exception as ex:
err('Dao insert_pending_tournament encountered an error')
err('Unknown error!')
from scraper.challonge import ChallongeScraper
import unittest
import iso8601
TOURNAMENT_URL = "http://challonge.com/TNE_Singles"
scraper = ChallongeScraper(TOURNAMENT_URL)
class IntTestChallongeScraper(unittest.TestCase):
def test_get_raw(self):
raw = scraper.get_raw()
self.assertTrue('tournament' in raw)
self.assertTrue('matches' in raw)
self.assertTrue('participants' in raw)
self.assertTrue('tournament' in raw['tournament'])
self.assertEquals(len(raw['matches']), 126)
self.assertEquals(len(raw['participants']), 64)
def test_get_name(self):
self.assertEquals(scraper.get_name(), "The Next Episode")
def test_get_date(self):
self.assertEquals(
scraper.get_date(),
iso8601.parse_date("2014-03-23 17:28:34.647000-04:00"))
def test_get_matches(self):
self.assertEquals(len(scraper.get_matches()), 126)
def test_get_players(self):
import unittest from dao import Dao, RegionNotFoundException, DuplicateAliasException, InvalidNameException from bson.objectid import ObjectId from model import * from ming import mim import trueskill from datetime import datetime from pymongo.errors import DuplicateKeyError from pymongo import MongoClient import scraper.challonge from scraper.challonge import ChallongeScraper DATABASE_NAME = 'data' PLAYERS_COLLECTION_NAME = 'players' TOURNAMENTS_COLLECTION_NAME = 'tournaments' RANKINGS_COLLECTION_NAME = 'rankings' REGIONS_COLLECTION_NAME = 'regions' USERS_COLLECTION_NAME = 'users' TEST_TOURNEY = 983062 scraper = ChallongeScraper(TEST_TOURNEY) mongo_client = MongoClient() region = Region(7, 'New Zealand') ##Dao.insert_region(region, mongo_client) p = Dao.get_all_regions(mongo_client) mongo_client.close()
