def getEventAwards(self, event): """ Works reliably for regional events from 2002-present and championship events from 2007-present """ if event.year < 2002 or (event.event_type_enum in EventType.CMP_EVENT_TYPES and event.year < 2007): # award pages malformatted logging.warning("Skipping awards parsing for event: {}".format(event.key_name)) return [] url = self.EVENT_AWARDS_URL_PATTERN % (event.year, self.EVENT_SHORT_EXCEPTIONS.get(event.event_short, event.event_short)) awards, _ = self.parse(url, self.YEAR_AWARD_PARSER.get(event.year, self.DEFAULT_AWARD_PARSER)) return [Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] ) for award in awards]
def test_awards_update(self): self.awards_auth.put() awards = [{'name_str': 'Winner', 'team_key': 'frc254'}, {'name_str': 'Winner', 'team_key': 'frc604'}, {'name_str': 'Volunteer Blahblah', 'team_key': 'frc1', 'awardee': 'Bob Bobby'}] request_body = json.dumps(awards) request_path = '/api/trusted/v1/event/2014casj/awards/update' sig = md5.new('{}{}{}'.format('321tEsTsEcReT', request_path, request_body)).hexdigest() response = self.testapp.post(request_path, request_body, headers={'X-TBA-Auth-Id': 'tEsT_id_4', 'X-TBA-Auth-Sig': sig}, expect_errors=True) self.assertEqual(response.status_code, 200) db_awards = Award.query(Award.event == self.event.key).fetch(None) self.assertEqual(len(db_awards), 2) self.assertTrue('2014casj_1' in [a.key.id() for a in db_awards]) self.assertTrue('2014casj_5' in [a.key.id() for a in db_awards]) awards = [{'name_str': 'Winner', 'team_key': 'frc254'}, {'name_str': 'Winner', 'team_key': 'frc604'}] request_body = json.dumps(awards) sig = md5.new('{}{}{}'.format('321tEsTsEcReT', request_path, request_body)).hexdigest() response = self.testapp.post(request_path, request_body, headers={'X-TBA-Auth-Id': 'tEsT_id_4', 'X-TBA-Auth-Sig': sig}, expect_errors=True) self.assertEqual(response.status_code, 200) db_awards = Award.query(Award.event == self.event.key).fetch(None) self.assertEqual(len(db_awards), 1) self.assertTrue('2014casj_1' in [a.key.id() for a in db_awards])
def _process_request(self, request, event_key): event = Event.get_by_id(event_key) awards = [] for award in JSONAwardsParser.parse(request.body, event_key): awards.append( Award(id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ ndb.Key(Team, team_key) for team_key in award['team_key_list'] ], recipient_json_list=award['recipient_json_list'])) # it's easier to clear all awards and add new ones than try to find the difference old_award_keys = Award.query(Award.event == event.key).fetch( None, keys_only=True) AwardManipulator.delete_keys(old_award_keys) if event.remap_teams: EventHelper.remapteams_awards(awards, event.remap_teams) AwardManipulator.createOrUpdate(awards) self.response.out.write( json.dumps({'Success': "Awards successfully updated"}))
def doAwardInsights(self, year): """ Calculate award insights for a given year. Returns a list of Insights. """ # Get all Blue Banner, Division Finalist, and Championship Finalist awards blue_banner_award_keys_future = Award.query( Award.year == year, Award.award_type_enum.IN(AwardType.BLUE_BANNER_AWARDS), Award.event_type_enum.IN({EventType.REGIONAL, EventType.DISTRICT, EventType.DISTRICT_CMP, EventType.CMP_DIVISION, EventType.CMP_FINALS}) ).fetch_async(10000, keys_only=True) cmp_finalist_award_keys_future = Award.query( Award.year == year, Award.award_type_enum == AwardType.FINALIST, Award.event_type_enum.IN({EventType.CMP_DIVISION, EventType.CMP_FINALS}) ).fetch_async(10000, keys_only=True) award_futures = ndb.get_multi_async( set(blue_banner_award_keys_future.get_result()).union( set(cmp_finalist_award_keys_future.get_result())) ) insights = [] insights += self._calculateBlueBanners(award_futures, year) insights += self._calculateChampionshipStats(award_futures, year) insights += self._calculateRegionalStats(award_futures, year) insights += self._calculateSuccessfulElimTeamups(award_futures, year) return insights
def doAwardInsights(self, year): """ Calculate award insights for a given year. Returns a list of Insights. """ # Get all Blue Banner, Division Finalist, and Championship Finalist awards blue_banner_award_keys_future = Award.query( Award.year == year, Award.award_type_enum.IN(AwardType.BLUE_BANNER_AWARDS), Award.event_type_enum.IN({ EventType.REGIONAL, EventType.DISTRICT, EventType.DISTRICT_CMP_DIVISION, EventType.DISTRICT_CMP, EventType.CMP_DIVISION, EventType.CMP_FINALS, EventType.FOC })).fetch_async(10000, keys_only=True) cmp_finalist_award_keys_future = Award.query( Award.year == year, Award.award_type_enum == AwardType.FINALIST, Award.event_type_enum.IN( {EventType.CMP_DIVISION, EventType.CMP_FINALS})).fetch_async(10000, keys_only=True) award_futures = ndb.get_multi_async( set(blue_banner_award_keys_future.get_result()).union( set(cmp_finalist_award_keys_future.get_result()))) insights = [] insights += self._calculateBlueBanners(award_futures, year) insights += self._calculateChampionshipStats(award_futures, year) insights += self._calculateRegionalStats(award_futures, year) insights += self._calculateSuccessfulElimTeamups(award_futures, year) return insights
def add_awards_from_csv(cls, awards_csv): events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning( "Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append( Award(id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list'] ], recipient_json_list=award['recipient_json_list'])) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] return new_awards
def _process_request(self, request, event_key): event = Event.get_by_id(event_key) awards = [] for award in JSONAwardsParser.parse(request.body, event_key): awards.append(Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, team_key) for team_key in award['team_key_list']], recipient_json_list=award['recipient_json_list'] )) # it's easier to clear all awards and add new ones than try to find the difference old_award_keys = Award.query(Award.event == event.key).fetch(None, keys_only=True) AwardManipulator.delete_keys(old_award_keys) if event.remap_teams: EventHelper.remapteams_awards(awards, event.remap_teams) AwardManipulator.createOrUpdate(awards) self.response.out.write(json.dumps({'Success': "Awards successfully updated"}))
def getAwards(self, keys, year=None): awards = [] for key in keys: if year == None: awards += Award.query(Award.name == key) else: awards += Award.query(Award.name == key, Award.year == year) return awards
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() ndb.get_context().clear_cache( ) # Prevent data from leaking between tests self.testbed.init_taskqueue_stub(root_path=".") self.event = Event( id="2013casj", event_short="casj", year=2013, event_type_enum=EventType.REGIONAL, ) self.old_award = Award( id=Award.render_key_name(self.event.key_name, AwardType.WINNER), name_str="Regional Winner", award_type_enum=AwardType.WINNER, year=2013, event=self.event.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc111'), ndb.Key(Team, 'frc234')], recipient_json_list=[ json.dumps({ 'team_number': 111, 'awardee': None }), json.dumps({ 'team_number': 234, 'awardee': None }) ], ) self.new_award = Award( id="2013casj_1", name_str="Regional Champion", award_type_enum=AwardType.WINNER, year=2013, event=self.event.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc359')], recipient_json_list=[ json.dumps({ 'team_number': 359, 'awardee': None }) ], )
def parse(self, response): awards_by_type = {} for award in response['Awards']: team_number = award['teamNumber'] if self.valid_team_nums is not None and team_number not in self.valid_team_nums: continue award_type_enum = AwardHelper.parse_award_type(award['name']) if award_type_enum is None: continue recipient_json = json.dumps({ 'team_number': team_number, 'awardee': award['person'], }) if award_type_enum in awards_by_type: if team_number is not None: awards_by_type[award_type_enum]['team_number_list'].append( team_number) awards_by_type[award_type_enum]['recipient_json_list'].append( recipient_json) else: awards_by_type[award_type_enum] = { 'name_str': award['name'], 'award_type_enum': award_type_enum, 'team_number_list': [team_number] if team_number is not None else [], 'recipient_json_list': [recipient_json], } awards = [] for award in awards_by_type.values(): awards.append( Award(id=Award.render_key_name(self.event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=self.event.year, event=self.event.key, event_type_enum=self.event.event_type_enum, team_list=[ ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list'] ], recipient_json_list=award['recipient_json_list'])) return awards
def _render(self, *args, **kw): awards_future = Award.query( Award.award_type_enum == AwardType.CHAIRMANS, Award.event_type_enum == EventType.CMP_FINALS).fetch_async() teams_by_year = defaultdict(list) for award in awards_future.get_result(): for team_key in award.team_list: teams_by_year[award.year].append(( team_key.get_async(), award.event.get_async(), award, media_query.TeamTagMediasQuery( team_key.id(), MediaTag.CHAIRMANS_VIDEO).fetch_async(), media_query.TeamTagMediasQuery( team_key.id(), MediaTag.CHAIRMANS_PRESENTATION).fetch_async(), media_query.TeamTagMediasQuery( team_key.id(), MediaTag.CHAIRMANS_ESSAY).fetch_async(), )) teams_by_year = sorted(teams_by_year.items(), key=lambda (k, v): -k) for _, tea in teams_by_year: tea.sort(key=lambda x: x[1].get_result().start_date) self.template_values.update({ 'teams_by_year': teams_by_year, }) return jinja2_engine.render('hof.html', self.template_values)
def _render(self, team_key, event_key): award_keys_future = Award.query(Award.team_list == ndb.Key(Team, self.team_key), Award.event == ndb.Key(Event, event_key)).fetch_async(None, keys_only=True) awards = ndb.get_multi(award_keys_future.get_result()) awards_dicts = [ModelToDict.awardConverter(award) for award in AwardHelper.organizeAwards(awards)] return json.dumps(awards_dicts, ensure_ascii=True)
def _render(self, *args, **kw): awards_future = Award.query( Award.award_type_enum==AwardType.CHAIRMANS, Award.event_type_enum==EventType.CMP_FINALS).fetch_async() teams_by_year = defaultdict(list) for award in awards_future.get_result(): for team_key in award.team_list: teams_by_year[award.year].append(( team_key.get_async(), award.event.get_async(), award, media_query.TeamTagMediasQuery(team_key.id(), MediaTag.CHAIRMANS_VIDEO).fetch_async(), media_query.TeamTagMediasQuery(team_key.id(), MediaTag.CHAIRMANS_PRESENTATION).fetch_async(), media_query.TeamTagMediasQuery(team_key.id(), MediaTag.CHAIRMANS_ESSAY).fetch_async(), )) teams_by_year = sorted(teams_by_year.items(), key=lambda (k, v): -k) for _, tea in teams_by_year: tea.sort(key=lambda x: x[1].get_result().start_date) self.template_values.update({ 'teams_by_year': teams_by_year, }) return jinja2_engine.render('hof.html', self.template_values)
def _query_async(self): team_key = self._query_args[0] event_key = self._query_args[1] awards = yield Award.query(Award.team_list == ndb.Key( Team, team_key), Award.event == ndb.Key(Event, event_key)).fetch_async() raise ndb.Return(awards)
def update(self, event_key): """ Updates EventTeams for an event. Returns a tuple of (teams, event_teams, event_team_keys_to_delete) An EventTeam is valid iff the team: a) played a match at the event, b) the team received an award at the event, c) or the event has not yet occurred. """ event = Event.get_by_id(event_key) # Add teams from Matches and Awards team_ids = set() match_key_futures = Match.query( Match.event == event.key).fetch_async(1000, keys_only=True) award_key_futures = Award.query( Award.event == event.key).fetch_async(1000, keys_only=True) match_futures = ndb.get_multi_async(match_key_futures.get_result()) award_futures = ndb.get_multi_async(award_key_futures.get_result()) for match_future in match_futures: match = match_future.get_result() for team in match.team_key_names: team_ids.add(team) for award_future in award_futures: award = award_future.get_result() for team_key in award.team_list: team_ids.add(team_key.id()) # Create or update EventTeams teams = [Team(id=team_id, team_number=int(team_id[3:])) for team_id in team_ids] if teams: event_teams = [EventTeam(id=event_key + "_" + team.key.id(), event=event.key, team=team.key, year=event.year) for team in teams] else: event_teams = None # Delete EventTeams for teams who did not participate in the event # Only runs if event is over existing_event_teams_keys = EventTeam.query( EventTeam.event == event.key).fetch(1000, keys_only=True) existing_event_teams = ndb.get_multi(existing_event_teams_keys) existing_team_ids = set() for et in existing_event_teams: existing_team_ids.add(et.team.id()) et_keys_to_delete = set() if event.end_date is not None and event.end_date < datetime.datetime.now(): for team_id in existing_team_ids.difference(team_ids): et_key_name = "{}_{}".format(event.key_name, team_id) et_keys_to_delete.add(ndb.Key(EventTeam, et_key_name)) ndb.delete_multi(et_keys_to_delete) return teams, event_teams, et_keys_to_delete
def _render(self, team_number): team = Team.get_by_id("frc" + team_number) if not team: return self.redirect("/error/404") event_team_keys_future = EventTeam.query(EventTeam.team == team.key).fetch_async(1000, keys_only=True) award_keys_future = Award.query(Award.team == team.key).fetch_async(1000, keys_only=True) event_teams_futures = ndb.get_multi_async(event_team_keys_future.get_result()) awards_futures = ndb.get_multi_async(award_keys_future.get_result()) event_keys = [event_team_future.get_result().event for event_team_future in event_teams_futures] events_futures = ndb.get_multi_async(event_keys) awards_by_event = {} for award_future in awards_futures: award = award_future.get_result() if award.event.id() not in awards_by_event: awards_by_event[award.event.id()] = [award] else: awards_by_event[award.event.id()].append(award) event_awards = [] current_event = None matches_upcoming = None short_cache = False for event_future in events_futures: event = event_future.get_result() if event.now: current_event = event team_matches_future = Match.query(Match.event == event.key, Match.team_key_names == team.key_name)\ .fetch_async(500, keys_only=True) matches = ndb.get_multi(team_matches_future.get_result()) matches_upcoming = MatchHelper.upcomingMatches(matches) if event.within_a_day: short_cache = True if event.key_name in awards_by_event: sorted_awards = AwardHelper.organizeAwards(awards_by_event[event.key_name])['list'] else: sorted_awards = [] event_awards.append((event, sorted_awards)) event_awards = sorted(event_awards, key=lambda (e, _): e.start_date if e.start_date else datetime.datetime(e.year, 12, 31)) years = sorted(set([et.get_result().year for et in event_teams_futures if et.get_result().year != None])) template_values = {'team': team, 'event_awards': event_awards, 'years': years, 'current_event': current_event, 'matches_upcoming': matches_upcoming} if short_cache: self._cache_expiration = self.SHORT_CACHE_EXPIRATION path = os.path.join(os.path.dirname(__file__), '../templates/team_history.html') return template.render(path, template_values)
def post(self): event_key = self.request.get('event_key') awards_json = self.request.get('awards_json') awards = json.loads(awards_json) event = Event.get_by_id(event_key) def _getTeamKey(award): team = Team.get_by_id('frc' + str(award.get('team_number', None))) if team is not None: return team.key else: return None awards = [Award( id = Award.renderKeyName(event.key_name, award.get('name')), name = award.get('name', None), team = _getTeamKey(award), awardee = award.get('awardee', None), year = event.year, official_name = award.get('official_name', None), event = event.key) for award in awards] AwardManipulator.createOrUpdate(awards) self.redirect('/admin/event/{}'.format(event_key))
def _query_async(self): team_key = self._query_args[0] year = self._query_args[1] awards = yield Award.query( Award.team_list == ndb.Key(Team, team_key), Award.year == year).fetch_async() raise ndb.Return(awards)
def add_awards_from_csv(cls, awards_csv): events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning("Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append(Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] )) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] return new_awards
def _query_async(self): team_key = self._query_args[0] event_key = self._query_args[1] awards = yield Award.query( Award.team_list == ndb.Key(Team, team_key), Award.event == ndb.Key(Event, event_key)).fetch_async() raise ndb.Return(awards)
def get(self, event_key): # Fetch for later event_future = Event.get_by_id_async(event_key) matches_future = match_query.EventMatchesQuery(event_key).fetch_async() # Rebuild event teams taskqueue.add( url='/tasks/math/do/eventteam_update/' + event_key, method='GET') # Create Winner/Finalist awards for offseason events awards = [] event = event_future.get_result() if event.event_type_enum in {EventType.OFFSEASON, EventType.FOC}: matches = MatchHelper.organizeMatches(matches_future.get_result()) bracket = MatchHelper.generateBracket(matches, event, event.alliance_selections) if 'f' in bracket: winning_alliance = '{}_alliance'.format(bracket['f'][1]['winning_alliance']) if winning_alliance == 'red_alliance': losing_alliance = 'blue_alliance' else: losing_alliance = 'red_alliance' awards.append(Award( id=Award.render_key_name(event.key_name, AwardType.WINNER), name_str="Winner", award_type_enum=AwardType.WINNER, year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team)) for team in bracket['f'][1][winning_alliance] if team.isdigit()], recipient_json_list=[json.dumps({'team_number': team, 'awardee': None}) for team in bracket['f'][1][winning_alliance]], )) awards.append(Award( id=Award.render_key_name(event.key_name, AwardType.FINALIST), name_str="Finalist", award_type_enum=AwardType.FINALIST, year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team)) for team in bracket['f'][1][losing_alliance] if team.isdigit()], recipient_json_list=[json.dumps({'team_number': team, 'awardee': None}) for team in bracket['f'][1][losing_alliance]], )) AwardManipulator.createOrUpdate(awards) self.response.out.write("Finished post-event tasks for {}. Created awards: {}".format(event_key, awards))
def post(self): # create a new award print('Award API: create a new award') requestData = request.get_data().decode('utf8') awardData = json.loads(requestData) newAward = Award() newAward.awardTitle = awardData['p_award_title'] newAward.awardDescription = awardData['p_award_description'] newAward.awardPoints = awardData['p_award_points'] newAward.awardAuthor = awardData['p_award_author'] newAward.awardCreatedDate = awardData['p_award_created_date'] newAward.awardUpdatedDate = awardData['p_award_updated_date'] newAward.awardCreatedBy = awardData['p_award_created_by'] newAward.awardUpdatedBy = awardData['p_award_updated_by'] add_award(newAward) # TODO: Finish this section, then do "def put()" return jsonify({"Saved": True})
def parse(cls, data): """ Parse CSV that contains awards Format is as follows: year, event_short, award_name_str, team_number (can be blank), awardee (can be blank) Example: 2000,mi,Regional Finalist,45, """ awards_by_key = {} csv_data = list(csv.reader(StringIO.StringIO(data), delimiter=',', skipinitialspace=True)) for award in csv_data: year = int(award[0]) event_short = award[1] name_str = award[2] team_number = award[3] awardee = award[4] if team_number == '': team_number = None else: team_number = int(team_number) if awardee == '': awardee = None # an award must have either an awardee or a team_number if awardee is None and team_number is None: continue if team_number is not None: team_number_list = [team_number] else: team_number_list = [] recipient_json = json.dumps({ 'team_number': team_number, 'awardee': awardee, }) award_type_enum = AwardHelper.parse_award_type(name_str) if award_type_enum is None: # If we can't figure it out, fall back to OTHER (good for offseason events) award_type_enum = AwardType.OTHER award_key_name = Award.render_key_name('{}{}'.format(year, event_short), award_type_enum) if award_key_name in awards_by_key: if team_number is not None: awards_by_key[award_key_name]['team_number_list'].append(team_number) awards_by_key[award_key_name]['recipient_json_list'].append(recipient_json) else: awards_by_key[award_key_name] = { 'year': year, 'event_short': event_short, 'name_str': name_str, 'award_type_enum': award_type_enum, 'team_number_list': team_number_list, 'recipient_json_list': [recipient_json], } return awards_by_key.values()
def _query_async(self): team_key = self._query_args[0] event_type_enum = self._query_args[1] award_type_enum = self._query_args[2] awards = yield Award.query( Award.team_list == ndb.Key(Team, team_key), Award.event_type_enum == event_type_enum, Award.award_type_enum == award_type_enum).fetch_async() raise ndb.Return(awards)
def get(self, award_key): self._require_admin() award = Award.get_by_id(award_key) self.template_values.update({"award": award}) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/award_edit.html') self.response.out.write(template.render(path, self.template_values))
def get(self, award_key): award = Award.get_by_id(award_key) template_values = { "award": award } path = os.path.join(os.path.dirname(__file__), '../../templates/admin/award_edit.html') self.response.out.write(template.render(path, template_values))
def get(self): self._require_admin() award_count = Award.query().count() self.template_values.update({"award_count": award_count}) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/award_dashboard.html') self.response.out.write(template.render(path, self.template_values))
def get(self): award_count = Award.query().count() template_values = { "award_count": award_count } path = os.path.join(os.path.dirname(__file__), '../../templates/admin/award_dashboard.html') self.response.out.write(template.render(path, template_values))
def test_remap_awards(self): awards = [ Award(recipient_json_list=[ """{"team_number": 2, "awardee": null}""" ]), ] EventHelper.remapteams_awards(awards, self.remap_teams) self.assertEqual(awards[0].recipient_list[0]['team_number'], '200') self.assertEqual(awards[0].team_list, [ndb.Key('Team', 'frc200')])
def get(self): self._require_admin() award_count = Award.query().count() self.template_values.update({ "award_count": award_count }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/award_dashboard.html') self.response.out.write(template.render(path, self.template_values))
def post(self): award_key = self.request.get('award_key') if not award_key: self.abort(400) award = Award.get_by_id(award_key) if not award: self.abort(404) AwardManipulator.delete(award) self.redirect('/admin/awards')
def get(self, award_key): self._require_admin() award = Award.get_by_id(award_key) self.template_values.update({ "award": award }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/award_edit.html') self.response.out.write(template.render(path, self.template_values))
def post(self): self._require_admin() awards_csv = self.request.get('awards_csv') events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning( "Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append( Award(id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list'] ], recipient_json_list=award['recipient_json_list'])) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] self.template_values = { 'awards': new_awards, } path = os.path.join(os.path.dirname(__file__), '../../templates/admin/awards_add.html') self.response.out.write(template.render(path, self.template_values))
def test_team_awards(self): # Insert some Teams frc1 = Team( id='frc1', team_number=1 ) frc1.put() frc2 = Team( id='frc2', team_number=2 ) frc2.put() frc3 = Team( id='frc3', team_number=3 ) frc3.put() # Insert some Awards for some Teams award = Award( id=Award.render_key_name(self.event_ends_today.key_name, AwardType.INDUSTRIAL_DESIGN), name_str='Industrial Design Award sponsored by General Motors', award_type_enum=AwardType.INDUSTRIAL_DESIGN, event=self.event_ends_today.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc1')], year=2020 ) award.put() winner_award = Award( id=Award.render_key_name(self.event_ends_today.key_name, AwardType.WINNER), name_str='Regional Event Winner', award_type_enum=AwardType.WINNER, event=self.event_ends_today.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc2'), ndb.Key(Team, 'frc1')], year=2020 ) winner_award.put() self.assertItemsEqual(self.event_ends_today.team_awards().keys(), [frc1.key, frc2.key])
def store_award(data, event): award = Award(id=Award.render_key_name(data['event_key'], data['award_type'])) award.event = ndb.Key(Event, data['event_key']) award.award_type_enum = data['award_type'] award.year = data['year'] award.name_str = data['name'] award.event_type_enum = event.event_type_enum recipient_list_fixed = [] team_keys = [] for recipient in data['recipient_list']: if recipient['team_key']: team_keys.append(ndb.Key(Team, recipient['team_key'])) recipient_list_fixed.append(json.dumps({ 'awardee': recipient['awardee'], 'team_number': int(recipient['team_key'][3:]) if recipient['team_key'] else None, })) award.recipient_json_list = recipient_list_fixed return AwardManipulator.createOrUpdate(award)
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() ndb.get_context().clear_cache() # Prevent data from leaking between tests self.testbed.init_taskqueue_stub(root_path=".") self.event = Event( id="2013casj", event_short="casj", year=2013, event_type_enum=EventType.REGIONAL, ) self.old_award = Award( id=Award.render_key_name(self.event.key_name, AwardType.WINNER), name_str="Regional Winner", award_type_enum=AwardType.WINNER, year=2013, event=self.event.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc111'), ndb.Key(Team, 'frc234')], recipient_json_list=[json.dumps({'team_number': 111, 'awardee': None}), json.dumps({'team_number': 234, 'awardee': None})], ) self.new_award = Award( id="2013casj_1", name_str="Regional Champion", award_type_enum=AwardType.WINNER, year=2013, event=self.event.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc359')], recipient_json_list=[json.dumps({'team_number': 359, 'awardee': None})], )
def parse(self, awards_json, event_key): """ Parse JSON that contains a list of awards where each award is a dict of: name_str: String of award name. ex: "Tournament Winner" or "Dean's List Finalist" team_key: String in the format "frcXXX" for the team that won the award. Can be null. awardee: String corresponding to the name of an individual that won the award. Can be null. """ try: awards = json.loads(awards_json) except: raise ParserInputException("Invalid JSON. Please check input.") awards_by_key = {} for award in awards: if type(award) is not dict: raise ParserInputException("Awards must be dicts.") name_str = award.get('name_str', None) team_key = award.get('team_key', None) awardee = award.get('awardee', None) if name_str is None: raise ParserInputException("Award must have a 'name_str'") if team_key and not re.match(r'frc\d+', str(team_key)): raise ParserInputException("Bad team_key: '{}'. Must follow format 'frcXXX' or be null.".format(team_key)) award_type_enum = AwardHelper.parse_award_type(name_str) if award_type_enum is None: raise ParserInputException("Cannot determine award type from: '{}'. Please contact a www.thebluealliance.com admin.".format(name_str)) recipient_json = json.dumps({ 'team_number': int(team_key[3:]) if team_key else None, 'awardee': awardee, }) award_key_name = Award.render_key_name(event_key, award_type_enum) if award_key_name in awards_by_key: if team_key is not None: awards_by_key[award_key_name]['team_key_list'].append(team_key) awards_by_key[award_key_name]['recipient_json_list'].append(recipient_json) else: awards_by_key[award_key_name] = { 'name_str': name_str, 'award_type_enum': award_type_enum, 'team_key_list': [team_key] if team_key else [], 'recipient_json_list': [recipient_json], } return awards_by_key.values()
def parse(self, response): awards_by_type = {} for award in response['Awards']: team_number = award['teamNumber'] if self.valid_team_nums is not None and team_number not in self.valid_team_nums: continue award_type_enum = AwardHelper.parse_award_type(award['name']) if award_type_enum is None: continue recipient_json = json.dumps({ 'team_number': team_number, 'awardee': award['person'], }) if award_type_enum in awards_by_type: if team_number is not None: awards_by_type[award_type_enum]['team_number_list'].append(team_number) awards_by_type[award_type_enum]['recipient_json_list'].append(recipient_json) else: awards_by_type[award_type_enum] = { 'name_str': award['name'], 'award_type_enum': award_type_enum, 'team_number_list': [team_number] if team_number is not None else [], 'recipient_json_list': [recipient_json], } awards = [] for award in awards_by_type.values(): awards.append(Award( id=Award.render_key_name(self.event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=self.event.year, event=self.event.key, event_type_enum=self.event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] )) return awards
def getEventAwards(self, event): def _getTeamKey(award): team = Team.get_by_id('frc' + str(award.get('team_number', None))) if team is not None: return team.key else: return None url = self.EVENT_AWARDS_URL_PATTERN % (event.year, self.EVENT_SHORT_EXCEPTIONS.get(event.event_short, event.event_short)) awards, _ = self.parse(url, UsfirstEventAwardsParser) return [Award( id=Award.renderKeyName(event.key_name, award.get('name')), name=award.get('name', None), team=_getTeamKey(award), awardee=award.get('awardee', None), year=event.year, official_name=award.get('official_name', None), event=event.key) for award in awards]
def post(self): self._require_admin() awards_csv = self.request.get('awards_csv') events = {} # for reducing datastore fetches of events and teams awards = [] for award in CSVAwardsParser.parse(awards_csv): event_key_name = '{}{}'.format(award['year'], award['event_short']) if event_key_name in events: event = events[event_key_name] else: event = Event.get_by_id(event_key_name) if event is None: logging.warning("Event: {} doesn't exist!".format(event_key_name)) continue events[event_key_name] = event awards.append(Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] )) new_awards = AwardManipulator.createOrUpdate(awards) if type(new_awards) != list: new_awards = [new_awards] self.template_values = { 'awards': new_awards, } path = os.path.join(os.path.dirname(__file__), '../../templates/admin/awards_add.html') self.response.out.write(template.render(path, self.template_values))
def post(self, award_key): self._require_admin() event_key_name = self.request.get('event_key_name') recipient_json_list = [] team_list = [] for recipient in json.loads(self.request.get('recipient_list_json')): recipient_json_list.append(json.dumps(recipient)) if recipient['team_number'] is not None: team_list.append( ndb.Key(Team, 'frc{}'.format(recipient['team_number']))) award = Award( id=award_key, name_str=self.request.get('name_str'), award_type_enum=int(self.request.get('award_type_enum')), event=ndb.Key(Event, event_key_name), event_type_enum=int(self.request.get('event_type_enum')), team_list=team_list, recipient_json_list=recipient_json_list, ) award = AwardManipulator.createOrUpdate(award, auto_union=False) self.redirect("/admin/event/" + event_key_name)
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() ndb.get_context().clear_cache() self.testbed.init_taskqueue_stub(root_path='.') self.event = Event( id='2020miket', event_type_enum=EventType.DISTRICT, short_name='Kettering University #1', name='FIM District Kettering University Event #1', event_short='miket', year=2020 ) self.team = Team( id='frc7332', team_number=7332 ) self.award = Award( id=Award.render_key_name(self.event.key_name, AwardType.INDUSTRIAL_DESIGN), name_str='Industrial Design Award sponsored by General Motors', award_type_enum=AwardType.INDUSTRIAL_DESIGN, event=self.event.key, event_type_enum=EventType.DISTRICT, year=2020 ) self.winner_award = Award( id=Award.render_key_name(self.event.key_name, AwardType.WINNER), name_str='District Event Winner', award_type_enum=AwardType.WINNER, event=self.event.key, event_type_enum=EventType.DISTRICT, year=2020 )
def get(self, event_key): if tba_config.CONFIG["env"] == "prod": # disable in prod for now logging.error("Tried to restore {} from CSV in prod! No can do.".format(event_key)) return event = Event.get_by_id(event_key) # alliances result = urlfetch.fetch(self.ALLIANCES_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.ALLIANCES_URL.format(event.year, event_key, event_key))) else: data = result.content.replace('frc', '') alliance_selections = CSVAllianceSelectionsParser.parse(data) event_details = EventDetails( id=event_key, alliance_selections=alliance_selections ) EventDetailsManipulator.createOrUpdate(event_details) # awards result = urlfetch.fetch(self.AWARDS_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.AWARDS_URL.format(event.year, event_key, event_key))) else: # convert into expected input format data = StringIO.StringIO() writer = csv.writer(data, delimiter=',') for row in csv.reader(StringIO.StringIO(result.content), delimiter=','): writer.writerow([event.year, event.event_short, row[1], row[2].replace('frc', ''), row[3]]) awards = [] for award in CSVAwardsParser.parse(data.getvalue()): awards.append(Award( id=Award.render_key_name(event.key_name, award['award_type_enum']), name_str=award['name_str'], award_type_enum=award['award_type_enum'], year=event.year, event=event.key, event_type_enum=event.event_type_enum, team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']], recipient_json_list=award['recipient_json_list'] )) AwardManipulator.createOrUpdate(awards) # matches result = urlfetch.fetch(self.MATCHES_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.MATCHES_URL.format(event.year, event_key, event_key))) else: data = result.content.replace('frc', '').replace('{}_'.format(event_key), '') match_dicts, _ = OffseasonMatchesParser.parse(data) matches = [ Match( id=Match.renderKeyName( event.key.id(), match.get("comp_level", None), match.get("set_number", 0), match.get("match_number", 0)), event=event.key, year=event.year, set_number=match.get("set_number", 0), match_number=match.get("match_number", 0), comp_level=match.get("comp_level", None), team_key_names=match.get("team_key_names", None), alliances_json=match.get("alliances_json", None) ) for match in match_dicts] MatchManipulator.createOrUpdate(matches) # rankings result = urlfetch.fetch(self.RANKINGS_URL.format(event.year, event_key, event_key)) if result.status_code != 200: logging.warning('Unable to retreive url: ' + (self.RANKINGS_URL.format(event.year, event_key, event_key))) else: # convert into expected input format rankings = list(csv.reader(StringIO.StringIO(result.content), delimiter=',')) event_details = EventDetails( id=event_key, rankings=rankings ) EventDetailsManipulator.createOrUpdate(event_details) self.response.out.write("Done restoring {}!".format(event_key))
def test_createOrUpdate_no_auto_union(self): AwardManipulator.createOrUpdate(self.old_award) self.assertOldAward(Award.get_by_id("2013casj_1")) AwardManipulator.createOrUpdate(self.new_award, auto_union=False) self.assertMergedAward(Award.get_by_id("2013casj_1"), False)
def _query_async(self): event_key = self._query_args[0] awards = yield Award.query(Award.event == ndb.Key(Event, event_key)).fetch_async() raise ndb.Return(awards)
class TestAwardManipulator(unittest2.TestCase): def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub() self.event = Event( id="2013casj", event_short="casj", year=2013, event_type_enum=EventType.REGIONAL, ) self.old_award = Award( id=Award.render_key_name(self.event.key_name, AwardType.WINNER), name_str="Regional Winner", award_type_enum=AwardType.WINNER, year=2013, event=self.event.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc111'), ndb.Key(Team, 'frc234')], recipient_json_list=[ json.dumps({ 'team_number': 111, 'awardee': None }), json.dumps({ 'team_number': 234, 'awardee': None }) ], ) self.new_award = Award( id="2013casj_1", name_str="Regional Champion", award_type_enum=AwardType.WINNER, year=2013, event=self.event.key, event_type_enum=EventType.REGIONAL, team_list=[ndb.Key(Team, 'frc359')], recipient_json_list=[ json.dumps({ 'team_number': 359, 'awardee': None }) ], ) def tearDown(self): self.testbed.deactivate() def assertMergedAward(self, award, is_auto_union): self.assertEqual(award.name_str, "Regional Champion") self.assertEqual(award.award_type_enum, AwardType.WINNER) self.assertEqual(award.year, 2013) self.assertEqual(award.event, self.event.key) self.assertEqual(award.event_type_enum, EventType.REGIONAL) if is_auto_union: self.assertEqual( set(award.team_list), { ndb.Key(Team, 'frc111'), ndb.Key(Team, 'frc234'), ndb.Key(Team, 'frc359') }) self.assertEqual(len(award.recipient_json_list), 3) for r in award.recipient_json_list: self.assertTrue( json.loads(r) in [{ 'team_number': 111, 'awardee': None }, { 'team_number': 234, 'awardee': None }, { 'team_number': 359, 'awardee': None }]) else: self.assertEqual(set(award.team_list), {ndb.Key(Team, 'frc359')}) self.assertEqual(len(award.recipient_json_list), 1) for r in award.recipient_json_list: self.assertTrue( json.loads(r) in [{ 'team_number': 359, 'awardee': None }]) def assertOldAward(self, award): self.assertEqual(award.name_str, "Regional Winner") self.assertEqual(award.award_type_enum, AwardType.WINNER) self.assertEqual(award.year, 2013) self.assertEqual(award.event, self.event.key) self.assertEqual(award.event_type_enum, EventType.REGIONAL) self.assertEqual(set(award.team_list), {ndb.Key(Team, 'frc111'), ndb.Key(Team, 'frc234')}) self.assertEqual(len(award.recipient_json_list), 2) for r in award.recipient_json_list: self.assertTrue( json.loads(r) in [{ 'team_number': 111, 'awardee': None }, { 'team_number': 234, 'awardee': None }]) def test_createOrUpdate(self): AwardManipulator.createOrUpdate(self.old_award) self.assertOldAward(Award.get_by_id("2013casj_1")) AwardManipulator.createOrUpdate(self.new_award) self.assertMergedAward(Award.get_by_id("2013casj_1"), True) def test_findOrSpawn(self): self.old_award.put() self.assertMergedAward(AwardManipulator.findOrSpawn(self.new_award), True) def test_updateMerge(self): self.assertMergedAward( AwardManipulator.updateMerge(self.new_award, self.old_award), True) def test_createOrUpdate_no_auto_union(self): AwardManipulator.createOrUpdate(self.old_award) self.assertOldAward(Award.get_by_id("2013casj_1")) AwardManipulator.createOrUpdate(self.new_award, auto_union=False) self.assertMergedAward(Award.get_by_id("2013casj_1"), False) def test_findOrSpawn_no_auto_union(self): self.old_award.put() self.assertMergedAward( AwardManipulator.findOrSpawn(self.new_award, auto_union=False), False) def test_updateMerge_no_auto_union(self): self.assertMergedAward( AwardManipulator.updateMerge(self.new_award, self.old_award, auto_union=False), False)
def _query_async(self): event_key = self._query_args[0] awards = yield Award.query( Award.event == ndb.Key(Event, event_key)).fetch_async() raise ndb.Return(awards)
def test_createOrUpdate(self): AwardManipulator.createOrUpdate(self.old_award) self.assertOldAward(Award.get_by_id("2013casj_1")) AwardManipulator.createOrUpdate(self.new_award) self.assertMergedAward(Award.get_by_id("2013casj_1"), True)
def _query_async(self): team_key = self._query_args[0] year = self._query_args[1] awards = yield Award.query(Award.team_list == ndb.Key(Team, team_key), Award.year == year).fetch_async() raise ndb.Return(awards)
def get_awards_async(): award_keys = yield Award.query(Award.year == year, Award.team_list == team.key).fetch_async(500, keys_only=True) awards = yield ndb.get_multi_async(award_keys) raise ndb.Return(awards)
def parse(self, data): """ Parse CSV that contains awards Format is as follows: year, event_short, award_name_str, team_number (can be blank), awardee (can be blank) Example: 2000,mi,Regional Finalist,45, """ awards_by_key = {} csv_data = list( csv.reader(StringIO.StringIO(data), delimiter=',', skipinitialspace=True)) for award in csv_data: year = int(award[0]) event_short = award[1] name_str = award[2] team_number = award[3] awardee = award[4] if team_number == '': team_number = None else: team_number = int(team_number) if awardee == '': awardee = None # an award must have either an awardee or a team_number if awardee is None and team_number is None: continue if team_number is not None: team_number_list = [team_number] else: team_number_list = [] recipient_json = json.dumps({ 'team_number': team_number, 'awardee': awardee, }) award_type_enum = AwardHelper.parse_award_type(name_str) if award_type_enum is None: continue award_key_name = Award.render_key_name( '{}{}'.format(year, event_short), award_type_enum) if award_key_name in awards_by_key: if team_number is not None: awards_by_key[award_key_name]['team_number_list'].append( team_number) awards_by_key[award_key_name]['recipient_json_list'].append( recipient_json) else: awards_by_key[award_key_name] = { 'year': year, 'event_short': event_short, 'name_str': name_str, 'award_type_enum': award_type_enum, 'team_number_list': team_number_list, 'recipient_json_list': [recipient_json], } return awards_by_key.values()