def add_awards_from_csv(cls, awards_csv): events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning("Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append(Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] )) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] return new_awards
def test_parse_regional_awards(self): with open("test_data/pre_2002_regional_awards.csv", "r") as f: awards = CSVAwardsParser.parse(f.read()) # Check number of parsed awards num_awards = 0 for award in awards: num_awards += len(award["recipient_json_list"]) self.assertEqual(num_awards, 885) self.assertEqual(len(awards), 601) awards = convert_to_comparable(awards) award = { "year": 1995, "event_short": "nh", "name_str": "Team Spirit Award", "award_type_enum": AwardType.SPIRIT, "team_number_list": [213], "recipient_json_list": [{"team_number": 213, "awardee": None}], } self.assertTrue(award in awards) award = { "year": 2000, "event_short": "mi", "name_str": "Best Offensive Round", "award_type_enum": AwardType.BEST_OFFENSIVE_ROUND, "team_number_list": [1, 240], "recipient_json_list": [{"team_number": 1, "awardee": None}, {"team_number": 240, "awardee": None}], } self.assertTrue(award in awards)
def add_awards_from_csv(cls, awards_csv): events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning( "Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append( Award(id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list'] ], recipient_json_list=award['recipient_json_list'])) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] return new_awards
def test_parse_cmp_awards(self): with open("test_data/pre_2007_cmp_awards.csv", "r") as f: awards = CSVAwardsParser.parse(f.read()) # Check number of parsed awards num_awards = 0 for award in awards: num_awards += len(award["recipient_json_list"]) self.assertEqual(num_awards, 560) self.assertEqual(len(awards), 330) awards = convert_to_comparable(awards) award = { "year": 2003, "event_short": "cmp", "name_str": "Championship Finalist", "award_type_enum": AwardType.FINALIST, "team_number_list": [25, 343, 494], "recipient_json_list": [ {"team_number": 25, "awardee": None}, {"team_number": 343, "awardee": None}, {"team_number": 494, "awardee": None}, ], } self.assertTrue(award in awards) award = { "year": 2002, "event_short": "arc", "name_str": "#1 Seed", "award_type_enum": AwardType.NUM_1_SEED, "team_number_list": [121], "recipient_json_list": [{"team_number": 121, "awardee": None}], } self.assertTrue(award in awards) award = { "year": 1992, "event_short": "cmp", "name_str": "Play of the Day", "award_type_enum": AwardType.PLAY_OF_THE_DAY, "team_number_list": [131], "recipient_json_list": [{"team_number": 131, "awardee": None}], } self.assertTrue(award in awards)
def test_parse_regional_awards(self): with open('test_data/pre_2002_regional_awards.csv', 'r') as f: awards = CSVAwardsParser.parse(f.read()) # Check number of parsed awards num_awards = 0 for award in awards: num_awards += len(award['recipient_json_list']) self.assertEqual(num_awards, 885) self.assertEqual(len(awards), 601) awards = convert_to_comparable(awards) award = { 'year': 1995, 'event_short': 'nh', 'name_str': "Team Spirit Award", 'award_type_enum': AwardType.SPIRIT, 'team_number_list': [213], 'recipient_json_list': [{ 'team_number': 213, 'awardee': None }], } self.assertTrue(award in awards) award = { 'year': 2000, 'event_short': 'mi', 'name_str': "Best Offensive Round", 'award_type_enum': AwardType.BEST_OFFENSIVE_ROUND, 'team_number_list': [1, 240], 'recipient_json_list': [{ 'team_number': 1, 'awardee': None }, { 'team_number': 240, 'awardee': None }], } self.assertTrue(award in awards)
def post(self): self._require_admin() awards_csv = self.request.get('awards_csv') events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning( "Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append( Award(id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list'] ], recipient_json_list=award['recipient_json_list'])) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] self.template_values = { 'awards': new_awards, } path = os.path.join(os.path.dirname(__file__), '../../templates/admin/awards_add.html') self.response.out.write(template.render(path, self.template_values))
def post(self): self._require_admin() awards_csv = self.request.get('awards_csv') events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning("Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append(Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] )) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] self.template_values = { 'awards': new_awards, } path = os.path.join(os.path.dirname(__file__), '../../templates/admin/awards_add.html') self.response.out.write(template.render(path, self.template_values))
def get(self, event_key): if tba_config.CONFIG["env"] == "prod": # disable in prod for now logging.error("Tried to restore {} from CSV in prod! No can do.".format(event_key)) return event = Event.get_by_id(event_key) # alliances result = urlfetch.fetch(self.ALLIANCES_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.ALLIANCES_URL.format(event.year, event_key, event_key))) else: data = result.content.replace('frc', '') alliance_selections = CSVAllianceSelectionsParser.parse(data) event_details = EventDetails( id=event_key, alliance_selections=alliance_selections ) EventDetailsManipulator.createOrUpdate(event_details) # awards result = urlfetch.fetch(self.AWARDS_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.AWARDS_URL.format(event.year, event_key, event_key))) else: # convert into expected input format data = StringIO.StringIO() writer = csv.writer(data, delimiter=',') for row in csv.reader(StringIO.StringIO(result.content), delimiter=','): writer.writerow([event.year, event.event_short, row[1], row[2].replace('frc', ''), row[3]]) awards = [] for award in CSVAwardsParser.parse(data.getvalue()): awards.append(Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] )) AwardManipulator.createOrUpdate(awards) # matches result = urlfetch.fetch(self.MATCHES_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.MATCHES_URL.format(event.year, event_key, event_key))) else: data = result.content.replace('frc', '').replace('{}_'.format(event_key), '') match_dicts, _ = OffseasonMatchesParser.parse(data) matches = [ Match( id=Match.renderKeyName( event.key.id(), match.get("comp_level", None), match.get("set_number", 0), match.get("match_number", 0)), event=event.key, year=event.year, set_number=match.get("set_number", 0), match_number=match.get("match_number", 0), comp_level=match.get("comp_level", None), team_key_names=match.get("team_key_names", None), alliances_json=match.get("alliances_json", None) ) for match in match_dicts] MatchManipulator.createOrUpdate(matches) # rankings result = urlfetch.fetch(self.RANKINGS_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.RANKINGS_URL.format(event.year, event_key, event_key))) else: # convert into expected input format rankings = list(csv.reader(StringIO.StringIO(result.content), delimiter=',')) event_details = EventDetails( id=event_key, rankings=rankings ) EventDetailsManipulator.createOrUpdate(event_details) self.response.out.write("Done restoring {}!".format(event_key))
def test_parse_cmp_awards(self): with open('test_data/pre_2007_cmp_awards.csv', 'r') as f: awards = CSVAwardsParser.parse(f.read()) # Check number of parsed awards num_awards = 0 for award in awards: num_awards += len(award['recipient_json_list']) self.assertEqual(num_awards, 560) self.assertEqual(len(awards), 330) awards = convert_to_comparable(awards) award = { 'year': 2003, 'event_short': 'cmp', 'name_str': "Championship Finalist", 'award_type_enum': AwardType.FINALIST, 'team_number_list': [25, 343, 494], 'recipient_json_list': [{ 'team_number': 25, 'awardee': None }, { 'team_number': 343, 'awardee': None }, { 'team_number': 494, 'awardee': None }], } self.assertTrue(award in awards) award = { 'year': 2002, 'event_short': 'arc', 'name_str': "#1 Seed", 'award_type_enum': AwardType.NUM_1_SEED, 'team_number_list': [121], 'recipient_json_list': [{ 'team_number': 121, 'awardee': None }], } self.assertTrue(award in awards) award = { 'year': 1992, 'event_short': 'cmp', 'name_str': "Play of the Day", 'award_type_enum': AwardType.PLAY_OF_THE_DAY, 'team_number_list': [131], 'recipient_json_list': [{ 'team_number': 131, 'awardee': None }], } self.assertTrue(award in awards)