def test_eventteams_update(self): self.teams_auth.put() team_list = ['frc254', 'frc971', 'frc604'] request_body = json.dumps(team_list) request_path = '/api/trusted/v1/event/2014casj/team_list/update' sig = md5.new('{}{}{}'.format('321tEsTsEcReT', request_path, request_body)).hexdigest() response = self.testapp.post(request_path, request_body, headers={'X-TBA-Auth-Id': 'tEsT_id_0', 'X-TBA-Auth-Sig': sig}, expect_errors=True) self.assertEqual(response.status_code, 200) db_eventteams = EventTeam.query(EventTeam.event == self.event.key).fetch(None) self.assertEqual(len(db_eventteams), 3) self.assertTrue('2014casj_frc254' in [et.key.id() for et in db_eventteams]) self.assertTrue('2014casj_frc971' in [et.key.id() for et in db_eventteams]) self.assertTrue('2014casj_frc604' in [et.key.id() for et in db_eventteams]) team_list = ['frc254', 'frc100'] request_body = json.dumps(team_list) sig = md5.new('{}{}{}'.format('321tEsTsEcReT', request_path, request_body)).hexdigest() response = self.testapp.post(request_path, request_body, headers={'X-TBA-Auth-Id': 'tEsT_id_0', 'X-TBA-Auth-Sig': sig}, expect_errors=True) self.assertEqual(response.status_code, 200) db_eventteams = EventTeam.query(EventTeam.event == self.event.key).fetch(None) self.assertEqual(len(db_eventteams), 2) self.assertTrue('2014casj_frc254' in [et.key.id() for et in db_eventteams]) self.assertTrue('2014casj_frc100' in [et.key.id() for et in db_eventteams])
def get_events_and_matches_async(): if return_valid_years: event_team_keys_query = EventTeam.query(EventTeam.team == team.key) else: event_team_keys_query = EventTeam.query(EventTeam.team == team.key, EventTeam.year == year) event_team_keys = yield event_team_keys_query.fetch_async(1000, keys_only=True) event_teams = yield ndb.get_multi_async(event_team_keys) event_keys = [] for event_team in event_teams: if return_valid_years: valid_years.add(event_team.year) # valid_years is a "global" variable (defined below). Doing this removes the complexity of having to propagate the years up through the tasklet call chain. if not return_valid_years or event_team.year == year: event_keys.append(event_team.event) events, matches = yield ndb.get_multi_async(event_keys), get_matches_async(event_keys) raise ndb.Return((events, matches))
def team_updated(affected_refs): team_keys = _filter(affected_refs['key']) event_team_keys_future = EventTeam.query(EventTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True) district_team_keys_future = DistrictTeam.query(DistrictTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True) queries_and_keys = [] for team_key in team_keys: queries_and_keys.append((TeamQuery(team_key.id()))) page_num = _get_team_page_num(team_key.id()) queries_and_keys.append((TeamListQuery(page_num))) for et_key in event_team_keys_future.get_result(): year = int(et_key.id()[:4]) event_key = et_key.id().split('_')[0] page_num = _get_team_page_num(et_key.id().split('_')[1]) queries_and_keys.append((TeamListYearQuery(year, page_num))) queries_and_keys.append((EventTeamsQuery(event_key))) queries_and_keys.append((EventEventTeamsQuery(event_key))) for dt_key in district_team_keys_future.get_result(): district_key = dt_key.id().split('_')[0] queries_and_keys.append((DistrictTeamsQuery(district_key))) return queries_and_keys
def get_event_details_cache_keys_and_controllers(cls, affected_refs): """ Gets cache keys and controllers that references this EventDetails TODO: Currently inefficient and also clears event APIv2 endpoints, since alliances are served under the event. APIv3 will break alliances out. """ event_details_keys = affected_refs['key'] event_keys = set() years = set() event_district_abbrevs = set() for event_details_key in event_details_keys: event_key = ndb.Key(Event, event_details_key.id()) event_keys.add(event_key) event = event_key.get() years.add(event.year) event_district_abbrevs.add(event.event_district_abbrev) event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True) team_keys = set() for et_key in event_team_keys_future.get_result(): team_key_name = et_key.id().split('_')[1] team_keys.add(ndb.Key(Team, team_key_name)) return cls._get_events_cache_keys_and_controllers(event_keys) + \ cls._get_event_district_points_cache_keys_and_controllers(event_keys) + \ cls._get_eventlist_cache_keys_and_controllers(years) + \ cls._get_team_events_cache_keys_and_controllers(team_keys, years) + \ cls._get_districtlist_cache_keys_and_controllers(years) + \ cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years) + \ cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years)
def addTeamDetails(self, team_dict, year): """ Consume a Team dict, and return it with a year's Events filtered and Matches added """ # TODO Matches should live under Events - gregmarra 1 feb 2011 # TODO Filter Events by year - gregmarra 1 feb 2011 memcache_key = "api_team_details_%s_%s" % (team_dict["key"], year) matches_list = memcache.get(memcache_key) if matches_list is None: matches = list() team = Team.get_by_id(team_dict["key"]) for e in [a.event.get() for a in EventTeam.query(EventTeam.team == team.key).fetch(1000) if a.year == year]: match_list = Match.query(Match.event == event.key, Match.team_key_names == team.key_name).fetch(500) matches.extend(match_list) matches_list = list() for match in matches: match_dict = dict() match_dict["key"] = match.key_name match_dict["event"] = match.event match_dict["comp_level"] = match.comp_level match_dict["set_number"] = match.set_number match_dict["match_number"] = match.match_number match_dict["team_keys"] = match.team_key_names match_dict["alliances"] = json.loads(match.alliances_json) matches_list.append(match_dict) #TODO: Reduce caching time before 2013 season. 2592000 is one month -gregmarra if tba_config.CONFIG["memcache"]: memcache.set(memcache_key, matches_list, 2592000) team_dict["matches"] = matches_list return team_dict
def get_event_cache_keys_and_controllers(cls, affected_refs): """ Gets cache keys and controllers that references this event """ event_keys = affected_refs["key"] years = affected_refs["year"] event_district_abbrevs = affected_refs["event_district_abbrev"] event_team_keys_future = EventTeam.query( EventTeam.event.IN([event_key for event_key in event_keys]) ).fetch_async(None, keys_only=True) team_keys = set() for et_key in event_team_keys_future.get_result(): team_key_name = et_key.id().split("_")[1] team_keys.add(ndb.Key(Team, team_key_name)) return ( cls._get_events_cache_keys_and_controllers(event_keys) + cls._get_event_district_points_cache_keys_and_controllers(event_keys) + cls._get_eventlist_cache_keys_and_controllers(years) + cls._get_team_events_cache_keys_and_controllers(team_keys, years) + cls._get_districtlist_cache_keys_and_controllers(years) + cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years) + cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years) )
def update(self, event_key): """ Updates EventTeams for an event. Returns a tuple of (teams, event_teams, event_team_keys_to_delete) An EventTeam is valid iff the team: a) played a match at the event, b) the team received an award at the event, c) or the event has not yet occurred. """ event = Event.get_by_id(event_key) # Add teams from Matches and Awards team_ids = set() match_key_futures = Match.query( Match.event == event.key).fetch_async(1000, keys_only=True) award_key_futures = Award.query( Award.event == event.key).fetch_async(1000, keys_only=True) match_futures = ndb.get_multi_async(match_key_futures.get_result()) award_futures = ndb.get_multi_async(award_key_futures.get_result()) for match_future in match_futures: match = match_future.get_result() for team in match.team_key_names: team_ids.add(team) for award_future in award_futures: award = award_future.get_result() for team_key in award.team_list: team_ids.add(team_key.id()) # Create or update EventTeams teams = [Team(id=team_id, team_number=int(team_id[3:])) for team_id in team_ids] if teams: event_teams = [EventTeam(id=event_key + "_" + team.key.id(), event=event.key, team=team.key, year=event.year) for team in teams] else: event_teams = None # Delete EventTeams for teams who did not participate in the event # Only runs if event is over existing_event_teams_keys = EventTeam.query( EventTeam.event == event.key).fetch(1000, keys_only=True) existing_event_teams = ndb.get_multi(existing_event_teams_keys) existing_team_ids = set() for et in existing_event_teams: existing_team_ids.add(et.team.id()) et_keys_to_delete = set() if event.end_date is not None and event.end_date < datetime.datetime.now(): for team_id in existing_team_ids.difference(team_ids): et_key_name = "{}_{}".format(event.key_name, team_id) et_keys_to_delete.add(ndb.Key(EventTeam, et_key_name)) ndb.delete_multi(et_keys_to_delete) return teams, event_teams, et_keys_to_delete
def event_updated(affected_refs): event_keys = filter(None, affected_refs["key"]) years = filter(None, affected_refs["year"]) event_district_keys = filter(None, affected_refs["event_district_key"]) event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async( None, keys_only=True ) queries_and_keys = [] for event_key in event_keys: queries_and_keys.append((EventQuery(event_key.id()))) for year in years: queries_and_keys.append((EventListQuery(year))) for event_district_key in event_district_keys: queries_and_keys.append((DistrictEventsQuery(event_district_key))) for et_key in event_team_keys_future.get_result(): team_key = et_key.id().split("_")[1] year = int(et_key.id()[:4]) queries_and_keys.append((TeamEventsQuery(team_key))) queries_and_keys.append((TeamYearEventsQuery(team_key, year))) return queries_and_keys
def event_updated(affected_refs): event_keys = _filter(affected_refs['key']) years = _filter(affected_refs['year']) event_district_keys = _filter(affected_refs['district_key']) event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True) events_future = ndb.get_multi_async(event_keys) queries_and_keys = [] for event_key in event_keys: queries_and_keys.append((EventQuery(event_key.id()))) queries_and_keys.append(EventDivisionsQuery(event_key.id())) for year in years: queries_and_keys.append((EventListQuery(year))) for event_district_key in event_district_keys: queries_and_keys.append((DistrictEventsQuery(event_district_key.id()))) if event_keys: for et_key in event_team_keys_future.get_result(): team_key = et_key.id().split('_')[1] year = int(et_key.id()[:4]) queries_and_keys.append((TeamEventsQuery(team_key))) queries_and_keys.append((TeamYearEventsQuery(team_key, year))) queries_and_keys.append((TeamYearEventTeamsQuery(team_key, year))) events_with_parents = filter(lambda e: e.get_result() is not None and e.get_result().parent_event is not None, events_future) parent_keys = set([e.get_result().parent_event for e in events_with_parents]) for parent_key in parent_keys: queries_and_keys.append((EventDivisionsQuery(parent_key.id()))) return queries_and_keys
def getTeamInfo(self, team_key): """ Return a Team dict with basic information. """ memcache_key = "api_team_info_%s" % team_key team_dict = memcache.get(memcache_key) if team_dict is None: team = Team.get_by_id(team_key) if team is not None: team_dict = dict() team_dict["key"] = team.key_name team_dict["team_number"] = team.team_number team_dict["name"] = team.name team_dict["nickname"] = team.nickname team_dict["website"] = team.website team_dict["location"] = team.location event_teams = EventTeam.query(EventTeam.team == team.key,\ EventTeam.year == datetime.now().year)\ .fetch(1000, projection=[EventTeam.event]) team_dict["events"] = [event_team.event.id() for event_team in event_teams] try: team_dict["location"] = team.location team_dict["locality"] = team.locality team_dict["region"] = team.region team_dict["country_name"] = team.country_name except Exception, e: logging.warning("Failed to include Address for api_team_info_%s: %s" % (team_key, e)) #TODO: Reduce caching time before 2013 season. 2592000 is one month -gregmarra if tba_config.CONFIG["memcache"]: memcache.set(memcache_key, team_dict, 2592000) else: raise IndexError
def _render(self, team_number): team = Team.get_by_id("frc" + team_number) if not team: return self.redirect("/error/404") event_team_keys_future = EventTeam.query(EventTeam.team == team.key).fetch_async(1000, keys_only=True) award_keys_future = Award.query(Award.team == team.key).fetch_async(1000, keys_only=True) event_teams_futures = ndb.get_multi_async(event_team_keys_future.get_result()) awards_futures = ndb.get_multi_async(award_keys_future.get_result()) event_keys = [event_team_future.get_result().event for event_team_future in event_teams_futures] events_futures = ndb.get_multi_async(event_keys) awards_by_event = {} for award_future in awards_futures: award = award_future.get_result() if award.event.id() not in awards_by_event: awards_by_event[award.event.id()] = [award] else: awards_by_event[award.event.id()].append(award) event_awards = [] current_event = None matches_upcoming = None short_cache = False for event_future in events_futures: event = event_future.get_result() if event.now: current_event = event team_matches_future = Match.query(Match.event == event.key, Match.team_key_names == team.key_name)\ .fetch_async(500, keys_only=True) matches = ndb.get_multi(team_matches_future.get_result()) matches_upcoming = MatchHelper.upcomingMatches(matches) if event.within_a_day: short_cache = True if event.key_name in awards_by_event: sorted_awards = AwardHelper.organizeAwards(awards_by_event[event.key_name])['list'] else: sorted_awards = [] event_awards.append((event, sorted_awards)) event_awards = sorted(event_awards, key=lambda (e, _): e.start_date if e.start_date else datetime.datetime(e.year, 12, 31)) years = sorted(set([et.get_result().year for et in event_teams_futures if et.get_result().year != None])) template_values = {'team': team, 'event_awards': event_awards, 'years': years, 'current_event': current_event, 'matches_upcoming': matches_upcoming} if short_cache: self._cache_expiration = self.SHORT_CACHE_EXPIRATION path = os.path.join(os.path.dirname(__file__), '../templates/team_history.html') return template.render(path, template_values)
def media_updated(affected_refs): reference_keys = _filter(affected_refs['references']) years = _filter(affected_refs['year']) media_tags = _filter(affected_refs['media_tag_enum']) team_keys = filter(lambda x: x.kind() == 'Team', reference_keys) event_team_keys_future = EventTeam.query(EventTeam.team.IN(team_keys)).fetch_async(None, keys_only=True) if team_keys else None queries_and_keys = [] for reference_key in reference_keys: if reference_key.kind() == 'Team': for year in years: queries_and_keys.append((TeamYearMediaQuery(reference_key.id(), year))) for media_tag in media_tags: queries_and_keys.append((TeamYearTagMediasQuery(reference_key.id(), media_tag, year))) for media_tag in media_tags: queries_and_keys.append((TeamTagMediasQuery(reference_key.id(), media_tag))) queries_and_keys.append((TeamSocialMediaQuery(reference_key.id()))) if reference_key.kind() == 'Event': queries_and_keys.append((EventMediasQuery(reference_key.id()))) if event_team_keys_future: for event_team_key in event_team_keys_future.get_result(): event_key = event_team_key.id().split('_')[0] year = int(event_key[:4]) if year in years: queries_and_keys.append(EventTeamsMediasQuery(event_key)) queries_and_keys.append(EventTeamsPreferredMediasQuery(event_key)) return queries_and_keys
def get(self, team_number): self._require_admin() team = Team.get_by_id("frc" + team_number) event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500) team_medias = Media.query(Media.references == team.key).fetch(500) robots = Robot.query(Robot.team == team.key).fetch() district_teams = DistrictTeam.query(DistrictTeam.team == team.key).fetch() team_medias_by_year = {} for media in team_medias: if media.year in team_medias_by_year: team_medias_by_year[media.year].append(media) else: team_medias_by_year[media.year] = [media] self.template_values.update({ 'event_teams': event_teams, 'team': team, 'team_medias_by_year': team_medias_by_year, 'robots': robots, 'district_teams': district_teams, }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html') self.response.out.write(template.render(path, self.template_values))
def get(self, team_number): self._require_admin() team = Team.get_by_id("frc" + team_number) if not team: self.abort(404) event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500) team_medias = Media.query(Media.references == team.key).fetch(500) robots = Robot.query(Robot.team == team.key).fetch() district_teams = DistrictTeam.query(DistrictTeam.team == team.key).fetch() years_participated = sorted(TeamParticipationQuery(team.key_name).fetch()) team_medias_by_year = {} for media in team_medias: if media.year in team_medias_by_year: team_medias_by_year[media.year].append(media) else: team_medias_by_year[media.year] = [media] media_years = sorted(team_medias_by_year.keys(), reverse=True) self.template_values.update({ 'event_teams': event_teams, 'team': team, 'team_media_years': media_years, 'team_medias_by_year': team_medias_by_year, 'robots': robots, 'district_teams': district_teams, 'years_participated': years_participated, }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html') self.response.out.write(template.render(path, self.template_values))
def prepTeams(self): # TODO there is a way to do this with yields such that this would be a # generator function that would yield, and if two sets of ndb fetches # went by would cleanly do itself without forcing a fetch. # -gregmarra 20121007 from models.event_team import EventTeam if self._teams_future is None: self._event_teams_future = EventTeam.query(EventTeam.event == self.key).fetch_async(500)
def _render(self, team_key): event_team_keys = EventTeam.query(EventTeam.team == ndb.Key(Team, team_key)).fetch(None, keys_only=True) years_participated = set() for event_team_key in event_team_keys: years_participated.add(int(event_team_key.id()[:4])) years_participated = sorted(list(years_participated)) return json.dumps(years_participated, ensure_ascii=True)
def _query_async(self): team_key = self._query_args[0] year = self._query_args[1] event_teams = yield EventTeam.query( EventTeam.team == ndb.Key(Team, team_key), EventTeam.year == year).fetch_async() event_keys = map(lambda event_team: event_team.event, event_teams) events = yield ndb.get_multi_async(event_keys) raise ndb.Return(events)
def getTeamData(self, event): #reader = csv.reader(open(file,"rb")) event_teams = EventTeam.query(EventTeam.event == event.key).fetch(500) team_keys = [event_team.team for event_team in event_teams] for num, team_key in enumerate(team_keys): OprHelper.teamdata.append([]) OprHelper.teamdata[num].append(num) # teamid OprHelper.teamdata[num].append(int(team_key[3:])) # teamnumber
def get(self, event_key): self._require_admin() event = Event.get_by_id(event_key) if not event: self.abort(404) return existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True)) EventTeamManipulator.delete_keys(existing_event_team_keys) self.response.out.write("Deleted {} EventTeams from {}".format(len(existing_event_team_keys), event_key))
def getAwards(self, event): awards = [] if event.event_type_enum == EventType.CMP_DIVISION and event.year >= 2015: # 8 subdivisions from 2015+ have awards listed under 4 divisions event_team_keys = EventTeam.query(EventTeam.event == event.key).fetch(keys_only=True) valid_team_nums = set([int(etk.id().split('_')[1][3:]) for etk in event_team_keys]) awards += self._parse(self.FMS_API_AWARDS_URL_PATTERN % (event.year, self._get_event_short(self.SUBDIV_TO_DIV[event.event_short])), FMSAPIAwardsParser(event, valid_team_nums)) awards += self._parse(self.FMS_API_AWARDS_URL_PATTERN % (event.year, self._get_event_short(event.event_short)), FMSAPIAwardsParser(event)) return awards
def get(self, year, first_eid): df = DatafeedUsfirst() df_legacy = DatafeedUsfirstLegacy() event = df.getEventDetails(first_eid) if not event: logging.warning("getEventDetails with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy.".format(first_eid)) event = df_legacy.getEventDetails(int(year), first_eid) if self.request.get('event_district_enum'): event.event_district_enum = int(self.request.get('event_district_enum')) event = EventManipulator.createOrUpdate(event) teams = df.getEventTeams(int(year), first_eid) if not teams: logging.warning("getEventTeams with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy.".format(first_eid)) teams = df_legacy.getEventTeams(int(year), first_eid) if not teams: logging.warning("getEventTeams with DatafeedUsfirstLegacy for event id {} failed.".format(first_eid)) teams = [] teams = TeamManipulator.createOrUpdate(teams) if teams: if type(teams) is not list: teams = [teams] event_teams = [EventTeam( id=event.key.id() + "_" + team.key.id(), event=event.key, team=team.key, year=event.year) for team in teams] # Delete eventteams of teams that unregister from an event if event.future: existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True)) event_team_keys = set([et.key for et in event_teams]) et_keys_to_delete = existing_event_team_keys.difference(event_team_keys) EventTeamManipulator.delete_keys(et_keys_to_delete) event_teams = EventTeamManipulator.createOrUpdate(event_teams) if type(event_teams) is not list: event_teams = [event_teams] else: event_teams = [] template_values = { 'event': event, 'event_teams': event_teams, } path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html') self.response.out.write(template.render(path, template_values))
def get(self, event_key): event = Event.get_by_id(event_key) event_teams = EventTeam.query(EventTeam.event==event.key).fetch() for event_team in event_teams: status = EventTeamStatusHelper.generate_team_at_event_status(event_team.team.id(), event) event_team.status = status FirebasePusher.update_event_team_status(event_key, event_team.team.id(), status) EventTeamManipulator.createOrUpdate(event_teams) if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue self.response.out.write("Finished calculating event team statuses for: {}".format(event_key))
def _render(self, team_key, year=None): self._set_team(team_key) event_team_keys = EventTeam.query(EventTeam.team == self.team.key, EventTeam.year == self.year).fetch(1000, keys_only=True) event_teams = ndb.get_multi(event_team_keys) event_keys = [event_team.event for event_team in event_teams] events = ndb.get_multi(event_keys) events = [ModelToDict.eventConverter(event) for event in events] return json.dumps(events, ensure_ascii=True)
def _query_async(self): event_key = self._query_args[0] year = int(event_key[:4]) event_team_keys = yield EventTeam.query(EventTeam.event == ndb.Key(Event, event_key)).fetch_async(keys_only=True) if not event_team_keys: raise ndb.Return([]) team_keys = map(lambda event_team_key: ndb.Key(Team, event_team_key.id().split('_')[1]), event_team_keys) medias = yield Media.query( Media.preferred_references.IN(team_keys), Media.year == year).fetch_async() raise ndb.Return(medias)
def test_repair(self): event_team = EventTeam.get_by_id("2011ct_frc177") self.assertEqual(event_team.year, None) broken_event_teams = EventTeam.query(EventTeam.year == None).fetch() self.assertGreater(len(broken_event_teams), 0) fixed_event_teams = EventTeamRepairer.repair(broken_event_teams) fixed_event_teams = EventTeamManipulator.createOrUpdate(fixed_event_teams) event_team = EventTeam.get_by_id("2011ct_frc177") self.assertEqual(event_team.year, 2011)
def get(self, event_key): df = DatafeedFMSAPI('v2.0') event = Event.get_by_id(event_key) models = df.getEventTeams(event_key) teams = [] district_teams = [] robots = [] for group in models: # models is a list of tuples (team, districtTeam, robot) if isinstance(group[0], Team): teams.append(group[0]) if isinstance(group[1], DistrictTeam): district_teams.append(group[1]) if isinstance(group[2], Robot): robots.append(group[2]) # Write new models teams = TeamManipulator.createOrUpdate(teams) district_teams = DistrictTeamManipulator.createOrUpdate(district_teams) robots = RobotManipulator.createOrUpdate(robots) if not teams: # No teams found registered for this event teams = [] # Build EventTeams event_teams = [EventTeam( id=event.key_name + "_" + team.key_name, event=event.key, team=team.key, year=event.year) for team in teams] # Delete eventteams of teams that are no longer registered if event.future: existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True)) event_team_keys = set([et.key for et in event_teams]) et_keys_to_delete = existing_event_team_keys.difference(event_team_keys) EventTeamManipulator.delete_keys(et_keys_to_delete) event_teams = EventTeamManipulator.createOrUpdate(event_teams) if type(event_teams) is not list: event_teams = [event_teams] template_values = { 'event': event, 'event_teams': event_teams, } path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html') self.response.out.write(template.render(path, template_values))
def get(self, team_number): self._require_admin() team = Team.get_by_id("frc" + team_number) event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500) self.template_values.update({ 'event_teams': event_teams, 'team': team, }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html') self.response.out.write(template.render(path, self.template_values))
def _render(self, district_abbrev, year=None, explicit_year=False): district_type = DistrictType.abbrevs[district_abbrev] event_keys = Event.query(Event.year == year, Event.event_district_enum == district_type).fetch(None, keys_only=True) if not event_keys: self.abort(404) # needed for valid_years all_cmp_event_keys_future = Event.query(Event.event_district_enum == district_type, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True) # needed for valid_districts district_cmp_keys_future = Event.query(Event.year == year, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True) # to compute valid_districts event_futures = ndb.get_multi_async(event_keys) event_team_keys_future = EventTeam.query(EventTeam.event.IN(event_keys)).fetch_async(None, keys_only=True) if year == 2014: # TODO: only 2014 has accurate rankings calculations team_futures = ndb.get_multi_async(set([ndb.Key(Team, et_key.id().split('_')[1]) for et_key in event_team_keys_future.get_result()])) events = [event_future.get_result() for event_future in event_futures] EventHelper.sort_events(events) district_cmp_futures = ndb.get_multi_async(district_cmp_keys_future.get_result()) if year == 2014: # TODO: only 2014 has accurate rankings calculations team_totals = DistrictHelper.calculate_rankings(events, team_futures, year) else: team_totals = None valid_districts = set() for district_cmp_future in district_cmp_futures: district_cmp = district_cmp_future.get_result() cmp_dis_type = district_cmp.event_district_enum if cmp_dis_type is None: logging.warning("District event {} has unknown district type!".format(district_cmp.key.id())) else: valid_districts.add((DistrictType.type_names[cmp_dis_type], DistrictType.type_abbrevs[cmp_dis_type])) valid_districts = sorted(valid_districts, key=lambda (name, _): name) self.template_values.update({ 'explicit_year': explicit_year, 'year': year, 'valid_years': sorted(set([int(event_key.id()[:4]) for event_key in all_cmp_event_keys_future.get_result()])), 'valid_districts': valid_districts, 'district_name': DistrictType.type_names[district_type], 'district_abbrev': district_abbrev, 'events': events, 'team_totals': team_totals, }) path = os.path.join(os.path.dirname(__file__), '../templates/district_details.html') return template.render(path, self.template_values)
def _query_async(self): year = self._query_args[0] page_num = self._query_args[1] event_team_keys_future = EventTeam.query(EventTeam.year == year).fetch_async(keys_only=True) teams_future = TeamListQuery(page_num).fetch_async() year_team_keys = set() for et_key in event_team_keys_future.get_result(): team_key = et_key.id().split('_')[1] year_team_keys.add(team_key) teams = filter(lambda team: team.key.id() in year_team_keys, teams_future.get_result()) raise ndb.Return(teams)
def _render(self, district_abbrev, year=None): self._set_district(district_abbrev) if self.year < 2009: return json.dumps([], ensure_ascii=True) event_keys = Event.query(Event.year == self.year, Event.event_district_enum == self.district).fetch( None, keys_only=True ) if not event_keys: return json.dumps([], ensure_ascii=True) events = ndb.get_multi(event_keys) event_futures = ndb.get_multi_async(event_keys) event_team_keys_future = EventTeam.query(EventTeam.event.IN(event_keys)).fetch_async(None, keys_only=True) team_futures = ndb.get_multi_async( set([ndb.Key(Team, et_key.id().split("_")[1]) for et_key in event_team_keys_future.get_result()]) ) events = [event_future.get_result() for event_future in event_futures] EventHelper.sort_events(events) team_totals = DistrictHelper.calculate_rankings(events, team_futures, self.year) rankings = [] current_rank = 1 for key, points in team_totals: point_detail = {} point_detail["rank"] = current_rank point_detail["team_key"] = key point_detail["event_points"] = {} for event in points["event_points"]: event_key = event[0].key_name point_detail["event_points"][event_key] = event[1] event_details = Event.get_by_id(event_key) point_detail["event_points"][event[0].key_name]["district_cmp"] = ( True if event_details.event_type_enum == EventType.DISTRICT_CMP else False ) if "rookie_bonus" in points: point_detail["rookie_bonus"] = points["rookie_bonus"] else: point_detail["rookie_bonus"] = 0 point_detail["point_total"] = points["point_total"] rankings.append(point_detail) current_rank += 1 return json.dumps(rankings)
def event_updated(affected_refs): event_keys = _filter(affected_refs['key']) years = _filter(affected_refs['year']) event_district_keys = _filter(affected_refs['district_key']) event_team_keys_future = EventTeam.query( EventTeam.event.IN([event_key for event_key in event_keys ])).fetch_async(None, keys_only=True) events_future = ndb.get_multi_async(event_keys) queries_and_keys = [] for event_key in event_keys: queries_and_keys.append((EventQuery(event_key.id()))) queries_and_keys.append(EventDivisionsQuery(event_key.id())) for year in years: queries_and_keys.append((EventListQuery(year))) for event_district_key in event_district_keys: queries_and_keys.append((DistrictEventsQuery(event_district_key.id()))) if event_keys: for et_key in event_team_keys_future.get_result(): team_key = et_key.id().split('_')[1] year = int(et_key.id()[:4]) queries_and_keys.append((TeamEventsQuery(team_key))) queries_and_keys.append((TeamYearEventsQuery(team_key, year))) queries_and_keys.append((TeamYearEventTeamsQuery(team_key, year))) events_with_parents = filter( lambda e: e.get_result() is not None and e.get_result().parent_event is not None, events_future) parent_keys = set( [e.get_result().parent_event for e in events_with_parents]) for parent_key in parent_keys: queries_and_keys.append((EventDivisionsQuery(parent_key.id()))) return queries_and_keys
def get(self, team_number): self._require_admin() team = Team.get_by_id("frc" + team_number) event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500) team_medias = Media.query(Media.references == team.key).fetch(500) team_medias_by_year = {} for media in team_medias: if media.year in team_medias_by_year: team_medias_by_year[media.year].append(media) else: team_medias_by_year[media.year] = [media] self.template_values.update({ 'event_teams': event_teams, 'team': team, 'team_medias_by_year': team_medias_by_year, }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html') self.response.out.write(template.render(path, self.template_values))
def media_updated(affected_refs): reference_keys = filter(None, affected_refs['references']) years = filter(None, affected_refs['year']) team_keys = filter(lambda x: x.kind() == 'Team', reference_keys) event_team_keys_future = EventTeam.query( EventTeam.team.IN(team_keys)).fetch_async(None, keys_only=True) queries_and_keys = [] for year in years: for reference_key in reference_keys: if reference_key.kind() == 'Team': queries_and_keys.append( (TeamYearMediaQuery(reference_key.id(), year))) for event_team_key in event_team_keys_future.get_result(): event_key = event_team_key.id().split('_')[0] year = int(event_key[:4]) if year in years: queries_and_keys.append(EventTeamsMediasQuery(event_key)) queries_and_keys.append(EventTeamsPreferredMediasQuery(event_key)) return queries_and_keys
def event_updated(affected_refs): event_keys = filter(None, affected_refs['key']) years = filter(None, affected_refs['year']) event_district_keys = filter(None, affected_refs['event_district_key']) event_team_keys_future = EventTeam.query( EventTeam.event.IN([event_key for event_key in event_keys ])).fetch_async(None, keys_only=True) queries_and_keys = [] for year in years: queries_and_keys.append((EventListQuery(year))) for event_district_key in event_district_keys: queries_and_keys.append((DistrictEventsQuery(event_district_key))) for et_key in event_team_keys_future.get_result(): team_key = et_key.id().split('_')[1] year = int(et_key.id()[:4]) queries_and_keys.append((TeamEventsQuery(team_key))) queries_and_keys.append((TeamYearEventsQuery(team_key, year))) return queries_and_keys
def team_updated(affected_refs): team_keys = filter(None, affected_refs['key']) event_team_keys_future = EventTeam.query(EventTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True) district_team_keys_future = DistrictTeam.query(DistrictTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True) queries_and_keys = [] for team_key in team_keys: page_num = _get_team_page_num(team_key.id()) queries_and_keys.append((TeamListQuery(page_num))) for et_key in event_team_keys_future.get_result(): year = int(et_key.id()[:4]) event_key = et_key.id().split('_')[0] page_num = _get_team_page_num(et_key.id().split('_')[1]) queries_and_keys.append((TeamListYearQuery(year, page_num))) queries_and_keys.append((EventTeamsQuery(event_key))) for dt_key in district_team_keys_future.get_result(): district_key = dt_key.id().split('_')[0] queries_and_keys.append((DistrictTeamsQuery(district_key))) return queries_and_keys
def get_event_cache_keys_and_controllers(cls, affected_refs): """ Gets cache keys and controllers that references this event """ event_keys = affected_refs['key'] years = affected_refs['year'] event_district_abbrevs = map(lambda x: x.id()[4:], filter(None, affected_refs['district_key'])) event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True) team_keys = set() for et_key in event_team_keys_future.get_result(): team_key_name = et_key.id().split('_')[1] team_keys.add(ndb.Key(Team, team_key_name)) return cls._get_events_cache_keys_and_controllers(event_keys) + \ cls._get_event_district_points_cache_keys_and_controllers(event_keys) + \ cls._get_eventlist_cache_keys_and_controllers(years) + \ cls._get_team_events_cache_keys_and_controllers(team_keys, years) + \ cls._get_districtlist_cache_keys_and_controllers(years) + \ cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years) + \ cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years) + \ cls._queries_to_cache_keys_and_controllers(get_affected_queries.event_updated(affected_refs))
def get_event_details_cache_keys_and_controllers(cls, affected_refs): """ Gets cache keys and controllers that references this EventDetails TODO: Currently inefficient and also clears event APIv2 endpoints, since alliances are served under the event. APIv3 will break alliances out. """ event_details_keys = affected_refs['key'] event_keys = set() years = set() event_district_abbrevs = set() for event_details_key in event_details_keys: event_key = ndb.Key(Event, event_details_key.id()) event_keys.add(event_key) event = event_key.get() years.add(event.year) event_district_abbrevs.add(event.event_district_abbrev) event_team_keys_future = EventTeam.query( EventTeam.event.IN([event_key for event_key in event_keys ])).fetch_async(None, keys_only=True) team_keys = set() for et_key in event_team_keys_future.get_result(): team_key_name = et_key.id().split('_')[1] team_keys.add(ndb.Key(Team, team_key_name)) return cls._get_events_cache_keys_and_controllers(event_keys) + \ cls._get_event_district_points_cache_keys_and_controllers(event_keys) + \ cls._get_eventlist_cache_keys_and_controllers(years) + \ cls._get_team_events_cache_keys_and_controllers(team_keys, years) + \ cls._get_districtlist_cache_keys_and_controllers(years) + \ cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years) + \ cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years) + \ cls._queries_to_cache_keys_and_controllers(get_affected_queries.event_details_updated(affected_refs))
def get_team_cache_keys_and_controllers(cls, affected_refs): """ Gets cache keys and controllers that references this team """ team_keys = affected_refs['key'] event_team_keys_future = EventTeam.query(EventTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True) district_team_keys_future = DistrictTeam.query(DistrictTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True) event_keys = set() for et_key in event_team_keys_future.get_result(): event_key_name = et_key.id().split('_')[0] event_keys.add(ndb.Key(Event, event_key_name)) district_keys = set() for dt_key in district_team_keys_future.get_result(): district_key_name = dt_key.id().split('_')[0] district_keys.add(ndb.Key(District, district_key_name)) return cls._get_teams_cache_keys_and_controllers(team_keys) + \ cls._get_eventteams_cache_keys_and_controllers(event_keys) + \ cls._get_teamlist_cache_keys_and_controllers(team_keys) + \ cls._get_districtteams_cache_keys_and_controllers(district_keys, team_keys) + \ cls._queries_to_cache_keys_and_controllers(get_affected_queries.team_updated(affected_refs))
def get(self, event_key): df = DatafeedFMSAPI('v2.0') df2 = DatafeedFIRSTElasticSearch() event = Event.get_by_id(event_key) # Update event updated_event = df2.getEventDetails(event) if updated_event: event = EventManipulator.createOrUpdate(updated_event) models = df.getEventTeams(event_key) teams = [] district_teams = [] robots = [] for group in models: # models is a list of tuples (team, districtTeam, robot) if isinstance(group[0], Team): teams.append(group[0]) if isinstance(group[1], DistrictTeam): district_teams.append(group[1]) if isinstance(group[2], Robot): robots.append(group[2]) # Merge teams teams = TeamManipulator.mergeModels(teams, df2.getEventTeams(event)) # Write new models if teams: teams = TeamManipulator.createOrUpdate(teams) district_teams = DistrictTeamManipulator.createOrUpdate(district_teams) robots = RobotManipulator.createOrUpdate(robots) if not teams: # No teams found registered for this event teams = [] if type(teams) is not list: teams = [teams] # Build EventTeams event_teams = [EventTeam( id=event.key_name + "_" + team.key_name, event=event.key, team=team.key, year=event.year) for team in teams] # Delete eventteams of teams that are no longer registered if event_teams != []: existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True)) event_team_keys = set([et.key for et in event_teams]) et_keys_to_delete = existing_event_team_keys.difference(event_team_keys) EventTeamManipulator.delete_keys(et_keys_to_delete) event_teams = EventTeamManipulator.createOrUpdate(event_teams) if type(event_teams) is not list: event_teams = [event_teams] template_values = { 'event': event, 'event_teams': event_teams, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html') self.response.out.write(template.render(path, template_values))
def _render(self, *args, **kw): week_events = EventHelper.getWeekEvents() special_webcasts = FirebasePusher.get_special_webcasts() events = {} for event in week_events: events[event.key.id()] = event # Calculate popular teams # Get cached team keys event_team_keys = memcache.get_multi(events.keys(), namespace='event-team-keys') # Get uncached team keys to_query = set(events.keys()).difference(event_team_keys.keys()) event_teams_futures = [ (event_key, EventTeam.query( EventTeam.event == ndb.Key(Event, event_key)).fetch_async( projection=[EventTeam.team])) for event_key in to_query ] # Merge cached and uncached team keys for event_key, event_teams in event_teams_futures: event_team_keys[event_key] = [ et.team.id() for et in event_teams.get_result() ] memcache.set_multi(event_team_keys, 60 * 60 * 24, namespace='event-team-keys') team_keys = [] team_events = {} for event_key, event_team_keys in event_team_keys.items(): team_keys += event_team_keys for team_key in event_team_keys: team_events[team_key] = events[event_key] # Get cached counts team_favorite_counts = memcache.get_multi( team_keys, namespace='team-favorite-counts') # Get uncached counts to_count = set(team_keys).difference(team_favorite_counts.keys()) count_futures = [ (team_key, Favorite.query(Favorite.model_key == team_key).count_async()) for team_key in to_count ] # Merge cached and uncached counts for team_key, count_future in count_futures: team_favorite_counts[team_key] = count_future.get_result() memcache.set_multi(team_favorite_counts, 60 * 60 * 24, namespace='team-favorite-counts') # Sort to get top popular teams popular_team_keys = [] for team_key, _ in sorted(team_favorite_counts.items(), key=lambda tc: -tc[1])[:25]: popular_team_keys.append(ndb.Key(Team, team_key)) popular_teams = sorted(ndb.get_multi(popular_team_keys), key=lambda team: team.team_number) popular_teams_events = [] for team in popular_teams: popular_teams_events.append((team, team_events[team.key.id()])) self.template_values.update({ "events": week_events, "any_webcast_online": any(w.get('status') == 'online' for w in special_webcasts), "special_webcasts": special_webcasts, "popular_teams_events": popular_teams_events, }) path = os.path.join(os.path.dirname(__file__), '../templates/index_competitionseason.html') return template.render(path, self.template_values)
def get(self, event_key): df = DatafeedFMSAPI('v2.0') df2 = DatafeedFIRSTElasticSearch() event = Event.get_by_id(event_key) # Update event fmsapi_events, fmsapi_districts = df.getEventDetails(event_key) elasticsearch_events = df2.getEventDetails(event) updated_event = EventManipulator.mergeModels(fmsapi_events, elasticsearch_events) if updated_event: event = EventManipulator.createOrUpdate(updated_event) DistrictManipulator.createOrUpdate(fmsapi_districts) models = df.getEventTeams(event_key) teams = [] district_teams = [] robots = [] for group in models: # models is a list of tuples (team, districtTeam, robot) if isinstance(group[0], Team): teams.append(group[0]) if isinstance(group[1], DistrictTeam): district_teams.append(group[1]) if isinstance(group[2], Robot): robots.append(group[2]) # Merge teams teams = TeamManipulator.mergeModels(teams, df2.getEventTeams(event)) # Write new models if teams and event.year == tba_config.MAX_YEAR: # Only update from latest year teams = TeamManipulator.createOrUpdate(teams) district_teams = DistrictTeamManipulator.createOrUpdate(district_teams) robots = RobotManipulator.createOrUpdate(robots) if not teams: # No teams found registered for this event teams = [] if type(teams) is not list: teams = [teams] # Build EventTeams cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') events_without_eventteams = cmp_hack_sitevar.contents.get('skip_eventteams', []) \ if cmp_hack_sitevar else [] skip_eventteams = event_key in events_without_eventteams event_teams = [ EventTeam(id=event.key_name + "_" + team.key_name, event=event.key, team=team.key, year=event.year) for team in teams ] if not skip_eventteams else [] # Delete eventteams of teams that are no longer registered if event_teams and not skip_eventteams: existing_event_team_keys = set( EventTeam.query(EventTeam.event == event.key).fetch( 1000, keys_only=True)) event_team_keys = set([et.key for et in event_teams]) et_keys_to_delete = existing_event_team_keys.difference( event_team_keys) EventTeamManipulator.delete_keys(et_keys_to_delete) event_teams = EventTeamManipulator.createOrUpdate(event_teams) if type(event_teams) is not list: event_teams = [event_teams] if event.year in {2018, 2019, 2020}: avatars, keys_to_delete = df.getEventTeamAvatars(event.key_name) if avatars: MediaManipulator.createOrUpdate(avatars) MediaManipulator.delete_keys(keys_to_delete) template_values = { 'event': event, 'event_teams': event_teams, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join( os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html') self.response.out.write(template.render(path, template_values))
def render_team_history(cls, handler, team, is_canonical): event_team_keys_future = EventTeam.query( EventTeam.team == team.key).fetch_async(1000, keys_only=True) award_keys_future = Award.query( Award.team_list == team.key).fetch_async(1000, keys_only=True) event_teams_futures = ndb.get_multi_async( event_team_keys_future.get_result()) awards_futures = ndb.get_multi_async(award_keys_future.get_result()) event_keys = [ event_team_future.get_result().event for event_team_future in event_teams_futures ] events_futures = ndb.get_multi_async(event_keys) awards_by_event = {} for award_future in awards_futures: award = award_future.get_result() if award.event.id() not in awards_by_event: awards_by_event[award.event.id()] = [award] else: awards_by_event[award.event.id()].append(award) event_awards = [] current_event = None matches_upcoming = None short_cache = False for event_future in events_futures: event = event_future.get_result() if event.now: current_event = event team_matches_future = Match.query(Match.event == event.key, Match.team_key_names == team.key_name)\ .fetch_async(500, keys_only=True) matches = ndb.get_multi(team_matches_future.get_result()) matches_upcoming = MatchHelper.upcomingMatches(matches) if event.within_a_day: short_cache = True if event.key_name in awards_by_event: sorted_awards = AwardHelper.organizeAwards( awards_by_event[event.key_name]) else: sorted_awards = [] event_awards.append((event, sorted_awards)) event_awards = sorted( event_awards, key=lambda (e, _): e.start_date if e.start_date else datetime.datetime(e.year, 12, 31)) years = sorted( set([ et.get_result().year for et in event_teams_futures if et.get_result().year is not None ])) handler.template_values.update({ 'is_canonical': is_canonical, 'team': team, 'event_awards': event_awards, 'years': years, 'current_event': current_event, 'matches_upcoming': matches_upcoming }) if short_cache: handler._cache_expiration = handler.SHORT_CACHE_EXPIRATION path = os.path.join(os.path.dirname(__file__), '../templates/team_history.html') return template.render(path, handler.template_values)
def getPopularTeamsEvents(self, events): events_by_key = {} for event in events: events_by_key[event.key.id()] = event # Calculate popular teams # Get cached team keys event_team_keys = memcache.get_multi(events_by_key.keys(), namespace='event-team-keys') # Get uncached team keys to_query = set(events_by_key.keys()).difference(event_team_keys.keys()) event_teams_futures = [ (event_key, EventTeam.query( EventTeam.event == ndb.Key(Event, event_key)).fetch_async( projection=[EventTeam.team])) for event_key in to_query ] # Merge cached and uncached team keys for event_key, event_teams in event_teams_futures: event_team_keys[event_key] = [ et.team.id() for et in event_teams.get_result() ] memcache.set_multi(event_team_keys, 60 * 60 * 24, namespace='event-team-keys') team_keys = [] team_events = {} for event_key, event_team_keys in event_team_keys.items(): team_keys += event_team_keys for team_key in event_team_keys: team_events[team_key] = events_by_key[event_key] # Get cached counts team_favorite_counts = memcache.get_multi( team_keys, namespace='team-favorite-counts') # Get uncached counts to_count = set(team_keys).difference(team_favorite_counts.keys()) count_futures = [ (team_key, Favorite.query(Favorite.model_key == team_key).count_async()) for team_key in to_count ] # Merge cached and uncached counts for team_key, count_future in count_futures: team_favorite_counts[team_key] = count_future.get_result() memcache.set_multi(team_favorite_counts, 60 * 60 * 24, namespace='team-favorite-counts') # Sort to get top popular teams popular_team_keys = [] for team_key, _ in sorted(team_favorite_counts.items(), key=lambda tc: -tc[1])[:25]: popular_team_keys.append(ndb.Key(Team, team_key)) popular_teams = sorted(ndb.get_multi(popular_team_keys), key=lambda team: team.team_number) popular_teams_events = [] for team in popular_teams: popular_teams_events.append((team, team_events[team.key.id()])) return popular_teams_events
def _query_async(self): team_key = self._query_args[0] event_teams = yield EventTeam.query(EventTeam.team == ndb.Key(Team, team_key)).fetch_async() event_keys = map(lambda event_team: event_team.event, event_teams) events = yield ndb.get_multi_async(event_keys) raise ndb.Return(events)
def _render(self, district_abbrev, year=None, explicit_year=False): district_type = DistrictType.abbrevs[district_abbrev] event_keys = Event.query(Event.year == year, Event.event_district_enum == district_type).fetch(None, keys_only=True) if not event_keys: self.abort(404) # needed for district teams district_key = '{}{}'.format(year, district_abbrev) district_teams_future = DistrictTeamsQuery(district_key).fetch_async() # needed for valid_years all_cmp_event_keys_future = Event.query(Event.event_district_enum == district_type, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True) # needed for valid_districts district_cmp_keys_future = Event.query(Event.year == year, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True) # to compute valid_districts # Needed for active team statuses live_events = EventHelper.getWeekEvents() live_eventteams_futures = [] for event in live_events: live_eventteams_futures.append(EventTeamsQuery(event.key_name).fetch_async()) event_futures = ndb.get_multi_async(event_keys) event_team_keys_future = EventTeam.query(EventTeam.event.IN(event_keys)).fetch_async(None, keys_only=True) team_futures = ndb.get_multi_async(set([ndb.Key(Team, et_key.id().split('_')[1]) for et_key in event_team_keys_future.get_result()])) events = [event_future.get_result() for event_future in event_futures] EventHelper.sort_events(events) district_cmp_futures = ndb.get_multi_async(district_cmp_keys_future.get_result()) team_totals = DistrictHelper.calculate_rankings(events, team_futures, year) valid_districts = set() for district_cmp_future in district_cmp_futures: district_cmp = district_cmp_future.get_result() cmp_dis_type = district_cmp.event_district_enum if cmp_dis_type is None: logging.warning("District event {} has unknown district type!".format(district_cmp.key.id())) else: valid_districts.add((DistrictType.type_names[cmp_dis_type], DistrictType.type_abbrevs[cmp_dis_type])) valid_districts = sorted(valid_districts, key=lambda (name, _): name) teams = TeamHelper.sortTeams(district_teams_future.get_result()) num_teams = len(teams) middle_value = num_teams / 2 if num_teams % 2 != 0: middle_value += 1 teams_a, teams_b = teams[:middle_value], teams[middle_value:] # Currently Competing Team Status live_events_with_teams = EventTeamStatusHelper.buildEventTeamStatus(live_events, live_eventteams_futures, teams) live_events_with_teams.sort(key=lambda x: x[0].name) self.template_values.update({ 'explicit_year': explicit_year, 'year': year, 'valid_years': sorted(set([int(event_key.id()[:4]) for event_key in all_cmp_event_keys_future.get_result()])), 'valid_districts': valid_districts, 'district_name': DistrictType.type_names[district_type], 'district_abbrev': district_abbrev, 'events': events, 'team_totals': team_totals, 'teams_a': teams_a, 'teams_b': teams_b, 'live_events_with_teams': live_events_with_teams, }) path = os.path.join(os.path.dirname(__file__), '../templates/district_details.html') return template.render(path, self.template_values)
def _render(self): year, location, range_limit, search_type, page = self._get_params() num_results = 0 results = [] distances = [] if location: lat_lon, _ = LocationHelper.get_lat_lon(location, geocode=True) if lat_lon: lat, lon = lat_lon dist_expr = 'distance(location, geopoint({}, {}))'.format( lat, lon) if search_type == 'teams': query_string = '{} < {}'.format( dist_expr, range_limit * self.METERS_PER_MILE) else: query_string = '{} < {} AND year={}'.format( dist_expr, range_limit * self.METERS_PER_MILE, year) offset = self.PAGE_SIZE * page query = search.Query( query_string=query_string, options=search.QueryOptions( limit=self.PAGE_SIZE, offset=offset, sort_options=search.SortOptions(expressions=[ search.SortExpression( expression=dist_expr, direction=search.SortExpression.ASCENDING) ]), returned_expressions=[ search.FieldExpression(name='distance', expression=dist_expr) ], )) if search_type == 'teams': search_index = search.Index(name="teamLocation") else: search_index = search.Index(name="eventLocation") docs = search_index.search(query) num_results = docs.number_found distances = {} keys = [] event_team_count_futures = {} for result in docs.results: distances[result.doc_id] = result.expressions[ 0].value / self.METERS_PER_MILE if search_type == 'teams': event_team_count_futures[ result.doc_id] = EventTeam.query( EventTeam.team == ndb.Key( 'Team', result.doc_id), EventTeam.year == year).count_async( limit=1, keys_only=True) keys.append(ndb.Key('Team', result.doc_id)) else: keys.append(ndb.Key('Event', result.doc_id)) result_futures = ndb.get_multi_async(keys) if search_type == 'teams': results = [] for result_future, team_key in zip(result_futures, keys): if event_team_count_futures[ team_key.id()].get_result() != 0: results.append(result_future.get_result()) else: results = [ result_future.get_result() for result_future in result_futures ] self.template_values.update({ 'valid_years': self.VALID_YEARS, 'valid_ranges': self.VALID_RANGES, 'page_size': self.PAGE_SIZE, 'page': page, 'year': year, 'location': location, 'range_limit': range_limit, 'search_type': search_type, 'num_results': num_results, 'results': results, 'distances': distances, }) return jinja2_engine.render('nearby.html', self.template_values)
def get(self, year, first_eid): df = DatafeedUsfirst() df_legacy = DatafeedUsfirstLegacy() event = df.getEventDetails(first_eid) if not event: logging.warning( "getEventDetails with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy." .format(first_eid)) event = df_legacy.getEventDetails(int(year), first_eid) if self.request.get('event_district_enum'): event.event_district_enum = int( self.request.get('event_district_enum')) event = EventManipulator.createOrUpdate(event) teams = df.getEventTeams(int(year), first_eid) if not teams: logging.warning( "getEventTeams with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy." .format(first_eid)) teams = df_legacy.getEventTeams(int(year), first_eid) if not teams: logging.warning( "getEventTeams with DatafeedUsfirstLegacy for event id {} failed." .format(first_eid)) teams = [] teams = TeamManipulator.createOrUpdate(teams) if teams: if type(teams) is not list: teams = [teams] event_teams = [ EventTeam(id=event.key.id() + "_" + team.key.id(), event=event.key, team=team.key, year=event.year) for team in teams ] # Delete eventteams of teams that unregister from an event if event.future: existing_event_team_keys = set( EventTeam.query(EventTeam.event == event.key).fetch( 1000, keys_only=True)) event_team_keys = set([et.key for et in event_teams]) et_keys_to_delete = existing_event_team_keys.difference( event_team_keys) EventTeamManipulator.delete_keys(et_keys_to_delete) event_teams = EventTeamManipulator.createOrUpdate(event_teams) if type(event_teams) is not list: event_teams = [event_teams] else: event_teams = [] template_values = { 'event': event, 'event_teams': event_teams, } path = os.path.join( os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html') self.response.out.write(template.render(path, template_values))
def _query_async(self): event_key = self._query_args[0] event_teams = yield EventTeam.query( EventTeam.event == ndb.Key(Event, event_key)).fetch_async() raise ndb.Return(event_teams)
def update(self, event_key): """ Updates EventTeams for an event. Returns a tuple of (teams, event_teams, event_team_keys_to_delete) An EventTeam is valid iff the team: a) played a match at the event, b) the team received an award at the event, c) the event has not yet occurred, d) or the event is not from the current year. (This is to make sure we don't delete old data we may no longer be able to scrape) """ event = Event.get_by_id(event_key) cur_year = datetime.datetime.now().year # Add teams from Matches and Awards team_ids = set() match_key_futures = Match.query( Match.event == event.key).fetch_async(1000, keys_only=True) award_key_futures = Award.query( Award.event == event.key).fetch_async(1000, keys_only=True) match_futures = ndb.get_multi_async(match_key_futures.get_result()) award_futures = ndb.get_multi_async(award_key_futures.get_result()) for match_future in match_futures: match = match_future.get_result() for team in match.team_key_names: team_ids.add(team) for award_future in award_futures: award = award_future.get_result() for team_key in award.team_list: team_ids.add(team_key.id()) # Create or update EventTeams teams = [Team(id=team_id, team_number=int(team_id[3:])) for team_id in team_ids] if teams: event_teams = [EventTeam(id=event_key + "_" + team.key.id(), event=event.key, team=team.key, year=event.year) for team in teams] else: event_teams = None # Delete EventTeams for teams who did not participate in the event # Only runs if event is over existing_event_teams_keys = EventTeam.query( EventTeam.event == event.key).fetch(1000, keys_only=True) existing_event_teams = ndb.get_multi(existing_event_teams_keys) existing_team_ids = set() for et in existing_event_teams: existing_team_ids.add(et.team.id()) et_keys_to_delete = set() if event.year == cur_year and event.end_date is not None and event.end_date < datetime.datetime.now(): for team_id in existing_team_ids.difference(team_ids): et_key_name = "{}_{}".format(event.key_name, team_id) et_keys_to_delete.add(ndb.Key(EventTeam, et_key_name)) return teams, event_teams, et_keys_to_delete