def setUp(self, mock_requests): mock_tournament_response = Mock(spec=requests.Response) mock_matches_response = Mock(spec=requests.Response) mock_participants_response = Mock(spec=requests.Response) with open(TOURNAMENT_JSON_FILE) as f: self.tournament_json_dict = json.load(f) with open(MATCHES_JSON_FILE) as f: self.matches_json_dict = json.load(f) with open(PARTICIPANTS_JSON_FILE) as f: self.participants_json_dict = json.load(f) mock_tournament_response.status_code = 200 mock_matches_response.status_code = 200 mock_participants_response.status_code = 200 mock_tournament_response.json.return_value = self.tournament_json_dict mock_matches_response.json.return_value = self.matches_json_dict mock_participants_response.json.return_value = self.participants_json_dict expected_tournament_url = scraper.challonge.TOURNAMENT_URL % TOURNAMENT_ID; expected_matches_url = scraper.challonge.MATCHES_URL % TOURNAMENT_ID; expected_participants_url = scraper.challonge.PARTICIPANTS_URL % TOURNAMENT_ID; mock_requests_return_values = { (expected_tournament_url, TEMPLATE_API_KEY): mock_tournament_response, (expected_matches_url, TEMPLATE_API_KEY): mock_matches_response, (expected_participants_url, TEMPLATE_API_KEY): mock_participants_response } mock_requests.get.side_effect = lambda url, **kwargs: mock_requests_return_values[(url, kwargs['params']['api_key'])] self.scraper = ChallongeScraper(TOURNAMENT_ID, TEMPLATE_CONFIG_FILE_PATH)
def test_get_tournament_id_from_url(self): self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_1), 'faketournament') self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_2), 'TWL7') self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_3), 'jerseyjapes-jj109meleesingles') self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_4), 'tdml46') self.assertEquals(ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_5), 'vgbootcamp-f117ijy1')
def import_tournament(type, path, bracket, region, name): config = Config() mongo_client = MongoClient(host=config.get_mongo_url()) if type == 'tio': scraper = TioScraper.from_file(path, bracket) elif type == 'challonge': scraper = ChallongeScraper(path) else: click.echo("Illegal type") dao = Dao(region, mongo_client=mongo_client) player_map = get_player_alias_to_id_map(scraper, dao) # TODO pass in a map of overrides for specific players tournament = Tournament.from_scraper(type, scraper, player_map, region) if name: tournament.name = name dao.insert_tournament(tournament) click.echo("Generating new ranking...") rankings.generate_ranking(dao) click.echo("Done!")
def test_get_tournament_id_from_url(self): self.assertEquals( ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_1), 'faketournament') self.assertEquals( ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_2), 'TWL7') self.assertEquals( ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_3), 'jerseyjapes-jj109meleesingles') self.assertEquals( ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_4), 'tdml46') self.assertEquals( ChallongeScraper.get_tournament_id_from_url(TOURNAMENT_URL_5), 'vgbootcamp-f117ijy1')
def test_get_tournament_id_from_invalid_url(self): with self.assertRaises(ValueError): ChallongeScraper.get_tournament_id_from_url(INVALID_TOURNAMENT_URL_1) with self.assertRaises(ValueError): ChallongeScraper.get_tournament_id_from_url(INVALID_TOURNAMENT_URL_2) with self.assertRaises(ValueError): ChallongeScraper.get_tournament_id_from_url(INVALID_TOURNAMENT_URL_3)
def post(self, region): dao = Dao(region, mongo_client=mongo_client) user = get_user_from_request(request, dao) if not user: return 'Permission denied', 403 if not is_user_admin_for_region(user, region): return 'Permission denied', 403 parser = reqparse.RequestParser() parser.add_argument('type', type=str, location='json') parser.add_argument('data', type=unicode, location='json') parser.add_argument('bracket', type=str, location='json') args = parser.parse_args() print "in server post, actually got this far!" if args['data'] is None: return "data required", 400 the_bytes = bytearray(args['data'], "utf8") if the_bytes[0] == 0xef: print "found magic numbers" return "magic numbers!", 503 type = args['type'] data = args['data'] if type == 'tio': if args['bracket'] is None: return "Missing bracket name", 400 data_bytes = bytes(data) if data_bytes[0] == '\xef': data = data[:3] scraper = TioScraper(data, args['bracket']) elif type == 'challonge': scraper = ChallongeScraper(data) else: return "Unknown type", 400 pending_tournament = PendingTournament.from_scraper( type, scraper, region) pending_tournament.alias_to_id_map = alias_service.get_alias_to_id_map_in_list_format( dao, pending_tournament.players) new_id = dao.insert_pending_tournament(pending_tournament) return_dict = {'id': str(new_id)} return return_dict
def test_get_tournament_id_from_invalid_url(self): with self.assertRaises(ValueError): ChallongeScraper.get_tournament_id_from_url( INVALID_TOURNAMENT_URL_1) with self.assertRaises(ValueError): ChallongeScraper.get_tournament_id_from_url( INVALID_TOURNAMENT_URL_2) with self.assertRaises(ValueError): ChallongeScraper.get_tournament_id_from_url( INVALID_TOURNAMENT_URL_3)
def bulk_import(path, region): config = parse_config() username = config.get('database', 'user') host = config.get('database', 'host') auth_db = config.get('database', 'auth_db') password = getpass.getpass() mongo_client = MongoClient(host='mongodb://%s:%s@%s/%s' % (username, password, host, auth_db)) dao = Dao(region, mongo_client=mongo_client, new=True) with open(path) as f: for line in f: line = line.strip() if line: print line scraper = ChallongeScraper(line) import_players(scraper, dao) import_tournament(scraper, dao) rankings.generate_ranking(dao)
def post(self, region): dao = Dao(region, mongo_client=mongo_client) user = get_user_from_access_token(request.headers, dao) if not is_user_admin_for_region(user, region): return 'Permission denied', 403 parser = reqparse.RequestParser() parser.add_argument('type', type=str, location='json') parser.add_argument('data', type=unicode, location='json') parser.add_argument('bracket', type=str, location='json') args = parser.parse_args() if args['data'] is None: return "data required", 400 type = args['type'] data = args['data'] if type == 'tio': if args['bracket'] is None: return "Missing bracket name", 400 scraper = TioScraper(data, args['bracket']) elif type == 'challonge': scraper = ChallongeScraper(data) else: return "Unknown type", 400 pending_tournament = PendingTournament.from_scraper( type, scraper, region) pending_tournament.alias_to_id_map = alias_service.get_alias_to_id_map_in_list_format( dao, pending_tournament.players) new_id = dao.insert_pending_tournament(pending_tournament) return_dict = {'id': str(new_id)} return return_dict
def post(self, region): parser = reqparse.RequestParser() \ .add_argument('type', type=str, location='json') \ .add_argument('data', type=unicode, location='json') \ .add_argument('bracket', type=str, location='json') args = parser.parse_args() if args['data'] is None: err("Data required. (TournamentSeedResource.post)") the_bytes = bytearray(args['data'], "utf8") if the_bytes[0] == 0xef: err("Magic numbers! (TournamentSeedResource.post)") type = args['type'] data = args['data'] pending_tournament = None try: if type == 'challonge': scraper = ChallongeScraper(data) else: err("Unknown type") pending_tournament, raw_file = M.PendingTournament.from_scraper( type, scraper, region) except Exception as ex: err('Scraper encountered an error ' + str(ex)) if not pending_tournament or not raw_file: err('Scraper encountered an error - null') pending_tournament_json = pending_tournament.dump( context='web', exclude=('date', 'matches', 'regions', 'type')) return pending_tournament_json
class TestChallongeScraper(unittest.TestCase): @patch('scraper.challonge.requests', spec=requests) def setUp(self, mock_requests): mock_tournament_response = Mock(spec=requests.Response) mock_matches_response = Mock(spec=requests.Response) mock_participants_response = Mock(spec=requests.Response) with open(TOURNAMENT_JSON_FILE) as f: self.tournament_json_dict = json.load(f) with open(MATCHES_JSON_FILE) as f: self.matches_json_dict = json.load(f) with open(PARTICIPANTS_JSON_FILE) as f: self.participants_json_dict = json.load(f) mock_tournament_response.status_code = 200 mock_matches_response.status_code = 200 mock_participants_response.status_code = 200 mock_tournament_response.json.return_value = self.tournament_json_dict mock_matches_response.json.return_value = self.matches_json_dict mock_participants_response.json.return_value = self.participants_json_dict expected_tournament_url = scraper.challonge.TOURNAMENT_URL % TOURNAMENT_ID; expected_matches_url = scraper.challonge.MATCHES_URL % TOURNAMENT_ID; expected_participants_url = scraper.challonge.PARTICIPANTS_URL % TOURNAMENT_ID; mock_requests_return_values = { (expected_tournament_url, TEMPLATE_API_KEY): mock_tournament_response, (expected_matches_url, TEMPLATE_API_KEY): mock_matches_response, (expected_participants_url, TEMPLATE_API_KEY): mock_participants_response } mock_requests.get.side_effect = lambda url, **kwargs: mock_requests_return_values[(url, kwargs['params']['api_key'])] self.scraper = ChallongeScraper(TOURNAMENT_ID, TEMPLATE_CONFIG_FILE_PATH) def test_get_raw(self): raw = self.scraper.get_raw() self.assertEquals(len(raw.keys()), 3) self.assertEquals(raw['tournament'], self.tournament_json_dict) self.assertEquals(raw['matches'], self.matches_json_dict) self.assertEquals(raw['participants'], self.participants_json_dict) def test_get_name(self): self.assertEquals(self.scraper.get_name(), 'SF Game Night 21') def test_get_date(self): self.assertEquals(self.scraper.get_date().replace(tzinfo=None), datetime(2014, 10, 14, 20, 39, 30)) def test_get_matches(self): matches = self.scraper.get_matches() self.assertEquals(len(matches), 81) self.assertEquals(matches[0], AliasMatch(winner='Tiamat', loser='Sharkboi')) self.assertEquals(matches[-1], AliasMatch(winner='Shroomed', loser='GC | Silentwolf')) self.assertEquals(matches[-2], AliasMatch(winner='GC | Silentwolf', loser='Shroomed')) self.assertEquals(matches[-3], AliasMatch(winner='GC | Silentwolf', loser='MIOM | SFAT')) # make sure none of the matches have a None for m in matches: self.assertIsNotNone(m.winner) self.assertIsNotNone(m.loser) def test_get_player(self): players = self.scraper.get_players() self.assertEquals(len(players), 41) self.assertTrue('Shroomed' in players) self.assertTrue('MIOM | SFAT' in players)
def post(self, region): dao = get_dao(region) auth_user(request, dao) parser = reqparse.RequestParser() \ .add_argument('type', type=str, location='json') \ .add_argument('data', type=unicode, location='json') \ .add_argument('bracket', type=str, location='json') \ .add_argument('included_phases', type=list, location='json') args = parser.parse_args() if args['data'] is None: err("Tournament data required.") the_bytes = bytearray(args['data'], "utf8") if the_bytes[0] == 0xef: print "found magic numbers" err("Magic numbers!") type = args['type'] data = args['data'] included_phases = args['included_phases'] pending_tournament = None try: if type == 'tio': if args['bracket'] is None: err("Missing bracket name") data_bytes = bytes(data) if data_bytes[0] == '\xef': data = data[:3] scraper = TioScraper(data, args['bracket']) elif type == 'challonge': scraper = ChallongeScraper(data) elif type == 'smashgg': scraper = SmashGGScraper(data, included_phases) else: err("Unknown tournament type") pending_tournament, raw_file = M.PendingTournament.from_scraper( type, scraper, region) except Exception as ex: err('Scraper encountered an error: ' + str(ex)) if not pending_tournament or not raw_file: err('Scraper encountered an error.') try: pending_tournament.alias_to_id_map = alias_service.get_alias_to_id_map_in_list_format( dao, pending_tournament.players) except Exception as e: log_exception() err('Alias service encountered an error') # If the tournament is too large, don't insert the raw file into the db. if len(pending_tournament.players) < 1000: try: raw_file = dao.insert_raw_file(raw_file) except Exception as ex: print ex err('Dao insert_raw_file encountered an error') else: print 'Skipping inserting raw file for tournament because it is too large' try: new_id = dao.insert_pending_tournament(pending_tournament) return_dict = { 'id': str(new_id) } return return_dict except Exception as ex: err('Dao insert_pending_tournament encountered an error') err('Unknown error!')
from scraper.challonge import ChallongeScraper import unittest import iso8601 TOURNAMENT_URL = "http://challonge.com/TNE_Singles" scraper = ChallongeScraper(TOURNAMENT_URL) class IntTestChallongeScraper(unittest.TestCase): def test_get_raw(self): raw = scraper.get_raw() self.assertTrue('tournament' in raw) self.assertTrue('matches' in raw) self.assertTrue('participants' in raw) self.assertTrue('tournament' in raw['tournament']) self.assertEquals(len(raw['matches']), 126) self.assertEquals(len(raw['participants']), 64) def test_get_name(self): self.assertEquals(scraper.get_name(), "The Next Episode") def test_get_date(self): self.assertEquals( scraper.get_date(), iso8601.parse_date("2014-03-23 17:28:34.647000-04:00")) def test_get_matches(self): self.assertEquals(len(scraper.get_matches()), 126) def test_get_players(self):
import unittest from dao import Dao, RegionNotFoundException, DuplicateAliasException, InvalidNameException from bson.objectid import ObjectId from model import * from ming import mim import trueskill from datetime import datetime from pymongo.errors import DuplicateKeyError from pymongo import MongoClient import scraper.challonge from scraper.challonge import ChallongeScraper DATABASE_NAME = 'data' PLAYERS_COLLECTION_NAME = 'players' TOURNAMENTS_COLLECTION_NAME = 'tournaments' RANKINGS_COLLECTION_NAME = 'rankings' REGIONS_COLLECTION_NAME = 'regions' USERS_COLLECTION_NAME = 'users' TEST_TOURNEY = 983062 scraper = ChallongeScraper(TEST_TOURNEY) mongo_client = MongoClient() region = Region(7, 'New Zealand') ##Dao.insert_region(region, mongo_client) p = Dao.get_all_regions(mongo_client) mongo_client.close()